From 1f94a0c62589837ca2a4f55a044633d8258a3343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 19:29:47 +0300 Subject: [PATCH 1/6] noneDB 4.0.0-rc.6 Storage format 4: a store record, one directory per database, fail-closed opening, generations, JSONL segments with key-index checkpoints, paged field indexes, shard filters and packed R-trees. Every derived file is checked by a crc, and damage to one never changes an answer. noneDB.php stays one file, built from src/. The separate tool noneDB-migrate.php, built from migrate/, converts databases of every earlier version. Also: database-level locking, rollback of failed writes, crash recovery, key rotation, online backup and restore, metrics, durable mode on Linux, bounded memory over many segments. PHP 8.1 or newer. The test suites are rebuilt. --- .gitattributes | 17 + .github/workflows/tests.yml | 119 +- .gitignore | 29 +- .nonedb.example | 1 - CHANGES.md | 219 +- README.md | 325 +- build/baseline/noneDB-3.3dev.php | 13992 +++++++++ composer.json | 9 +- docs/API.md | 747 +- docs/ARCHITECTURE.md | 200 + docs/BACKUP_RESTORE.md | 159 + docs/BENCHMARKS.md | 350 - docs/CONFIGURATION.md | 698 +- docs/DURABILITY.md | 153 + docs/MIGRATION.md | 393 + docs/OPERATIONS.md | 229 + docs/PERFORMANCE.md | 361 + docs/QUERY.md | 524 +- docs/SPATIAL.md | 277 +- docs/STORAGE_FORMAT.md | 770 + docs/index.html | 29 +- examples/chaining.php | 8 +- examples/database-management.php | 8 +- examples/delete.php | 8 +- examples/find.php | 4 +- examples/performance.php | 25 +- examples/query-methods.php | 2 +- examples/sharding.php | 66 +- examples/update.php | 6 +- migrate/Backup/Backup.php | 54 + migrate/Cli.php | 161 + migrate/Converter/Writer.php | 118 + migrate/Files.php | 201 + migrate/Journal/Journal.php | 53 + migrate/Publisher/Migration.php | 619 + migrate/Readers/Discovery.php | 102 + migrate/Readers/Source.php | 658 + migrate/Restore/Restore.php | 171 + migrate/Stop.php | 20 + migrate/Validator/Verifier.php | 129 + migrate/footer.php | 10 + migrate/header.php | 13 + migrate/manifest.php | 20 + noneDB-migrate.php | 2251 ++ noneDB.php | 24481 ++++++++++++---- phpunit.xml | 22 +- src/Internal/Cache/CacheManager.php | 456 + src/Internal/Cache/Runtime.php | 122 + src/Internal/Concurrency/LockManager.php | 868 + src/Internal/Concurrency/Snapshot.php | 129 + src/Internal/Config/Configuration.php | 693 + src/Internal/Core/Crud.php | 264 + src/Internal/Core/DatabaseOps.php | 371 + src/Internal/Core/Kernel.php | 331 + src/Internal/Diagnostics/Integrity.php | 384 + src/Internal/Diagnostics/Metrics.php | 182 + src/Internal/Durability/FileIO.php | 512 + src/Internal/Durability/Sync.php | 239 + src/Internal/Index/FieldIndex.php | 1961 ++ src/Internal/Index/FieldIndexApi.php | 252 + src/Internal/Index/PagedFile.php | 289 + src/Internal/Index/SecondaryFiles.php | 437 + src/Internal/Index/ShardFilters.php | 168 + src/Internal/Index/Spatial/GeoJson.php | 875 + src/Internal/Index/Spatial/Geometry.php | 250 + src/Internal/Index/Spatial/RTree.php | 936 + .../Index/Spatial/SpatialIndexApi.php | 198 + src/Internal/Maintenance/BackupRestore.php | 627 + src/Internal/Maintenance/Compaction.php | 250 + src/Internal/Maintenance/KeyMaintenance.php | 570 + src/Internal/Maintenance/Maintenance.php | 366 + src/Internal/Query/Aggregation.php | 161 + src/Internal/Query/FindFilter.php | 225 + src/Internal/Query/Planner.php | 384 + src/Internal/Query/QueryMethods.php | 186 + src/Internal/Query/SpatialQueries.php | 797 + src/Internal/Query/Utility.php | 200 + src/Internal/Query/Values.php | 235 + src/Internal/Recovery/Recovery.php | 238 + src/Internal/Recovery/RecoveryLog.php | 144 + src/Internal/Recovery/RollbackScope.php | 286 + src/Internal/Sharding/LayoutSwitch.php | 187 + src/Internal/Sharding/ShardInfo.php | 105 + src/Internal/Sharding/ShardMeta.php | 246 + src/Internal/Sharding/ShardRouter.php | 193 + src/Internal/Storage/Buffer.php | 424 + src/Internal/Storage/Checkpoints.php | 889 + src/Internal/Storage/KeyIndex.php | 579 + src/Internal/Storage/Layout.php | 736 + src/Internal/Storage/RecordIO.php | 487 + src/Internal/Storage/Segments.php | 195 + src/Internal/Storage/StoreRecord.php | 508 + src/Internal/Storage/Writer.php | 204 + src/header.php | 13 + src/manifest.php | 87 + src/noneDB/Facade.php | 907 + src/noneDBQuery/Aggregates.php | 93 + src/noneDBQuery/Chain.php | 271 + src/noneDBQuery/Core.php | 69 + src/noneDBQuery/Execution.php | 926 + src/noneDBQuery/Filters.php | 301 + src/noneDBQuery/SpatialChain.php | 74 + src/noneDBQuery/Writes.php | 83 + tests/Concurrency/BackupConcurrencyTest.php | 231 + .../ConcurrencyRegressionTest.php | 48 +- .../ConcurrencyTest.php | 112 +- tests/Concurrency/LockEpochTest.php | 231 + tests/Concurrency/LockStateRegressionTest.php | 351 + tests/Crash/BackupRestoreCrashTest.php | 258 + .../CrashRecoveryTest.php | 16 +- tests/Crash/DatabaseLifecycleTest.php | 351 + tests/Crash/DerivedFileDamageTest.php | 404 + tests/Crash/DurabilityContractTest.php | 833 + tests/Crash/DurabilityLinuxTest.php | 117 + tests/Crash/FaultMatrixTest.php | 486 + tests/Crash/GenerationSwitchTest.php | 237 + tests/Crash/KeyMaintenanceTest.php | 641 + tests/Crash/PowerLossSimulationTest.php | 210 + tests/Crash/RollbackScopeTest.php | 762 + tests/Crash/TornWriteRecoveryTest.php | 512 + .../DifferentialFuzzTest.php | 90 +- .../Differential/GeometryConsistencyTest.php | 146 + .../PagedIndexDifferentialTest.php | 178 + tests/Differential/QueryConsistencyTest.php | 221 + tests/Docs/doctest.php | 107 + tests/Feature/AggregationTest.php | 326 - tests/Feature/ChainingTest.php | 558 - tests/Feature/CheckDBTest.php | 192 - tests/Feature/ComparisonOperatorTest.php | 768 - tests/Feature/ConfigurationTest.php | 360 - tests/Feature/CreateDBTest.php | 158 - tests/Feature/GetDBsTest.php | 268 - tests/Feature/NewChainingMethodsTest.php | 1626 - tests/Feature/NewMethodsShardedTest.php | 387 - tests/Feature/QueryDocumentationTest.php | 886 - tests/Feature/QueryMethodsTest.php | 260 - tests/Feature/RemoveFieldsTest.php | 582 - tests/Feature/TypeEdgeCasesTest.php | 1096 - tests/Integration/AggregationTest.php | 443 + tests/Integration/AppendContractTest.php | 97 + tests/Integration/BackupRestoreTest.php | 961 + tests/Integration/BoundedMemoryTest.php | 441 + tests/Integration/CRUDWorkflowTest.php | 75 +- tests/Integration/CacheLimitTest.php | 524 + .../ChainingEdgeCasesTest.php | 397 +- tests/Integration/ChainingTest.php | 793 + tests/Integration/CheckDBTest.php | 233 + tests/Integration/ComparisonOperatorTest.php | 756 + tests/Integration/ConfigurationTest.php | 554 + tests/Integration/CreateDBTest.php | 233 + tests/{Feature => Integration}/DeleteTest.php | 119 +- tests/Integration/DeltaLogTest.php | 257 + tests/Integration/DurabilityConfigTest.php | 245 + tests/Integration/EdgeCasesTest.php | 128 +- .../Integration/FieldIndexCheckpointTest.php | 307 + tests/Integration/FieldIndexFormatTest.php | 1613 + .../FieldIndexRegressionTest.php | 82 +- .../FieldIndexTest.php | 146 +- tests/Integration/FileSecurityTest.php | 244 + tests/{Feature => Integration}/FindTest.php | 179 +- tests/Integration/GetDBsTest.php | 318 + tests/Integration/GlobalIndexTest.php | 205 + tests/Integration/HardeningRegressionTest.php | 541 + tests/Integration/HeldDataTest.php | 503 + tests/Integration/IndexHardeningTest.php | 328 + tests/{Feature => Integration}/InsertTest.php | 120 +- tests/Integration/IntegrityIndexTest.php | 189 + tests/Integration/KeyIndexFormatTest.php | 370 + tests/Integration/MetricsTest.php | 369 + tests/Integration/NewChainingMethodsTest.php | 2162 ++ .../NewMethodsEdgeCasesTest.php | 302 +- tests/Integration/NewMethodsShardedTest.php | 616 + tests/Integration/OperationsToolsTest.php | 411 + .../PublicApiCoverageTest.php | 82 +- tests/Integration/QueryDocumentationTest.php | 1772 ++ tests/Integration/QueryEngineTest.php | 279 + tests/Integration/QueryHardeningTest.php | 137 + tests/Integration/QueryMethodsTest.php | 308 + .../QueryRegressionTest.php | 126 +- tests/Integration/RTreeTest.php | 262 + tests/Integration/RemoveFieldsTest.php | 752 + .../SecurityConfigRegressionTest.php | 288 +- .../ShardedFieldIndexTest.php | 266 +- .../{Feature => Integration}/ShardingTest.php | 247 +- .../SpatialIndexTest.php | 96 +- .../SpatialOperatorCombinationTest.php | 126 +- .../SpatialRealWorldTest.php | 129 +- .../SpatialRegressionTest.php | 82 +- tests/Integration/SpecIndexTest.php | 1964 ++ tests/Integration/SpecOperationsTest.php | 1948 ++ tests/Integration/SpecQueryTest.php | 1764 ++ tests/Integration/SpecSpatialTest.php | 2051 ++ tests/Integration/SpecStorageTest.php | 1602 + .../StorageRegressionTest.php | 133 +- tests/Integration/StoreFormatTest.php | 359 + tests/Integration/StrictOptionsTest.php | 142 + tests/Integration/TypeEdgeCasesTest.php | 1455 + tests/{Feature => Integration}/UpdateTest.php | 108 +- .../UtilityMethodsTest.php | 131 +- tests/InternalMethod.php | 31 + tests/InternalValue.php | 30 + tests/Migration/MigrationFixtureTest.php | 169 + tests/Migration/MigrationProcessTest.php | 753 + tests/Migration/MigrationRulesTest.php | 259 + tests/Migration/MigrationTestCase.php | 166 + tests/Performance/durability_cost.php | 81 + tests/Performance/large.php | 337 + .../{ => Performance}/operation_benchmark.php | 11 +- .../{ => Performance}/operator_benchmark.php | 17 +- tests/Performance/perf_gate.php | 936 + .../performance_benchmark.php | 19 +- .../{ => Performance}/sleekdb_comparison.php | 19 +- tests/{ => Performance}/spatial_benchmark.php | 23 +- tests/Performance/sqlite_comparison.php | 340 + tests/Regression/LockStateRegressionTest.php | 193 - tests/{Regression => }/RegressionTestCase.php | 66 +- .../noneDB_critical_scope_regression.php | 68 + tests/Smoke/noneDB_regression_smoke.php | 204 + tests/Support/Testing.php | 180 + tests/Unit/ArchitectureTest.php | 223 + tests/Unit/BuildTest.php | 228 + tests/Unit/DatabaseLocatorTest.php | 230 + tests/Unit/FileSizeConvertTest.php | 22 +- tests/Unit/GeoJSONValidationTest.php | 194 +- tests/Unit/GeometryOperationsTest.php | 276 +- tests/Unit/GeometryValidationTest.php | 488 + tests/Unit/HashDBNameTest.php | 133 - tests/Unit/LimitTest.php | 52 +- tests/Unit/ShardFiltersTest.php | 110 + tests/Unit/SourceWriteGuardTest.php | 141 + tests/bootstrap.php | 35 +- tests/buffer_test.php | 124 - tests/concurrent_insert_test.php | 184 - tests/concurrent_stress_test.php | 122 - tests/noneDBTestCase.php | 830 +- tests/sleekdb_vs_nonedb_benchmark.php | 420 - tools/build.php | 292 + tools/make-migration-fixtures.php | 451 + tools/soak.php | 1076 + 239 files changed, 109006 insertions(+), 17380 deletions(-) create mode 100644 .gitattributes create mode 100644 build/baseline/noneDB-3.3dev.php create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/BACKUP_RESTORE.md delete mode 100644 docs/BENCHMARKS.md create mode 100644 docs/DURABILITY.md create mode 100644 docs/MIGRATION.md create mode 100644 docs/OPERATIONS.md create mode 100644 docs/PERFORMANCE.md create mode 100644 docs/STORAGE_FORMAT.md create mode 100644 migrate/Backup/Backup.php create mode 100644 migrate/Cli.php create mode 100644 migrate/Converter/Writer.php create mode 100644 migrate/Files.php create mode 100644 migrate/Journal/Journal.php create mode 100644 migrate/Publisher/Migration.php create mode 100644 migrate/Readers/Discovery.php create mode 100644 migrate/Readers/Source.php create mode 100644 migrate/Restore/Restore.php create mode 100644 migrate/Stop.php create mode 100644 migrate/Validator/Verifier.php create mode 100644 migrate/footer.php create mode 100644 migrate/header.php create mode 100644 migrate/manifest.php create mode 100644 noneDB-migrate.php create mode 100644 src/Internal/Cache/CacheManager.php create mode 100644 src/Internal/Cache/Runtime.php create mode 100644 src/Internal/Concurrency/LockManager.php create mode 100644 src/Internal/Concurrency/Snapshot.php create mode 100644 src/Internal/Config/Configuration.php create mode 100644 src/Internal/Core/Crud.php create mode 100644 src/Internal/Core/DatabaseOps.php create mode 100644 src/Internal/Core/Kernel.php create mode 100644 src/Internal/Diagnostics/Integrity.php create mode 100644 src/Internal/Diagnostics/Metrics.php create mode 100644 src/Internal/Durability/FileIO.php create mode 100644 src/Internal/Durability/Sync.php create mode 100644 src/Internal/Index/FieldIndex.php create mode 100644 src/Internal/Index/FieldIndexApi.php create mode 100644 src/Internal/Index/PagedFile.php create mode 100644 src/Internal/Index/SecondaryFiles.php create mode 100644 src/Internal/Index/ShardFilters.php create mode 100644 src/Internal/Index/Spatial/GeoJson.php create mode 100644 src/Internal/Index/Spatial/Geometry.php create mode 100644 src/Internal/Index/Spatial/RTree.php create mode 100644 src/Internal/Index/Spatial/SpatialIndexApi.php create mode 100644 src/Internal/Maintenance/BackupRestore.php create mode 100644 src/Internal/Maintenance/Compaction.php create mode 100644 src/Internal/Maintenance/KeyMaintenance.php create mode 100644 src/Internal/Maintenance/Maintenance.php create mode 100644 src/Internal/Query/Aggregation.php create mode 100644 src/Internal/Query/FindFilter.php create mode 100644 src/Internal/Query/Planner.php create mode 100644 src/Internal/Query/QueryMethods.php create mode 100644 src/Internal/Query/SpatialQueries.php create mode 100644 src/Internal/Query/Utility.php create mode 100644 src/Internal/Query/Values.php create mode 100644 src/Internal/Recovery/Recovery.php create mode 100644 src/Internal/Recovery/RecoveryLog.php create mode 100644 src/Internal/Recovery/RollbackScope.php create mode 100644 src/Internal/Sharding/LayoutSwitch.php create mode 100644 src/Internal/Sharding/ShardInfo.php create mode 100644 src/Internal/Sharding/ShardMeta.php create mode 100644 src/Internal/Sharding/ShardRouter.php create mode 100644 src/Internal/Storage/Buffer.php create mode 100644 src/Internal/Storage/Checkpoints.php create mode 100644 src/Internal/Storage/KeyIndex.php create mode 100644 src/Internal/Storage/Layout.php create mode 100644 src/Internal/Storage/RecordIO.php create mode 100644 src/Internal/Storage/Segments.php create mode 100644 src/Internal/Storage/StoreRecord.php create mode 100644 src/Internal/Storage/Writer.php create mode 100644 src/header.php create mode 100644 src/manifest.php create mode 100644 src/noneDB/Facade.php create mode 100644 src/noneDBQuery/Aggregates.php create mode 100644 src/noneDBQuery/Chain.php create mode 100644 src/noneDBQuery/Core.php create mode 100644 src/noneDBQuery/Execution.php create mode 100644 src/noneDBQuery/Filters.php create mode 100644 src/noneDBQuery/SpatialChain.php create mode 100644 src/noneDBQuery/Writes.php create mode 100644 tests/Concurrency/BackupConcurrencyTest.php rename tests/{Regression => Concurrency}/ConcurrencyRegressionTest.php (70%) rename tests/{Integration => Concurrency}/ConcurrencyTest.php (60%) create mode 100644 tests/Concurrency/LockEpochTest.php create mode 100644 tests/Concurrency/LockStateRegressionTest.php create mode 100644 tests/Crash/BackupRestoreCrashTest.php rename tests/{Regression => Crash}/CrashRecoveryTest.php (91%) create mode 100644 tests/Crash/DatabaseLifecycleTest.php create mode 100644 tests/Crash/DerivedFileDamageTest.php create mode 100644 tests/Crash/DurabilityContractTest.php create mode 100644 tests/Crash/DurabilityLinuxTest.php create mode 100644 tests/Crash/FaultMatrixTest.php create mode 100644 tests/Crash/GenerationSwitchTest.php create mode 100644 tests/Crash/KeyMaintenanceTest.php create mode 100644 tests/Crash/PowerLossSimulationTest.php create mode 100644 tests/Crash/RollbackScopeTest.php create mode 100644 tests/Crash/TornWriteRecoveryTest.php rename tests/{Regression => Differential}/DifferentialFuzzTest.php (80%) create mode 100644 tests/Differential/GeometryConsistencyTest.php create mode 100644 tests/Differential/PagedIndexDifferentialTest.php create mode 100644 tests/Differential/QueryConsistencyTest.php create mode 100644 tests/Docs/doctest.php delete mode 100644 tests/Feature/AggregationTest.php delete mode 100644 tests/Feature/ChainingTest.php delete mode 100644 tests/Feature/CheckDBTest.php delete mode 100644 tests/Feature/ComparisonOperatorTest.php delete mode 100644 tests/Feature/ConfigurationTest.php delete mode 100644 tests/Feature/CreateDBTest.php delete mode 100644 tests/Feature/GetDBsTest.php delete mode 100644 tests/Feature/NewChainingMethodsTest.php delete mode 100644 tests/Feature/NewMethodsShardedTest.php delete mode 100644 tests/Feature/QueryDocumentationTest.php delete mode 100644 tests/Feature/QueryMethodsTest.php delete mode 100644 tests/Feature/RemoveFieldsTest.php delete mode 100644 tests/Feature/TypeEdgeCasesTest.php create mode 100644 tests/Integration/AggregationTest.php create mode 100644 tests/Integration/AppendContractTest.php create mode 100644 tests/Integration/BackupRestoreTest.php create mode 100644 tests/Integration/BoundedMemoryTest.php create mode 100644 tests/Integration/CacheLimitTest.php rename tests/{Feature => Integration}/ChainingEdgeCasesTest.php (63%) create mode 100644 tests/Integration/ChainingTest.php create mode 100644 tests/Integration/CheckDBTest.php create mode 100644 tests/Integration/ComparisonOperatorTest.php create mode 100644 tests/Integration/ConfigurationTest.php create mode 100644 tests/Integration/CreateDBTest.php rename tests/{Feature => Integration}/DeleteTest.php (56%) create mode 100644 tests/Integration/DeltaLogTest.php create mode 100644 tests/Integration/DurabilityConfigTest.php create mode 100644 tests/Integration/FieldIndexCheckpointTest.php create mode 100644 tests/Integration/FieldIndexFormatTest.php rename tests/{Regression => Integration}/FieldIndexRegressionTest.php (64%) rename tests/{Feature => Integration}/FieldIndexTest.php (62%) create mode 100644 tests/Integration/FileSecurityTest.php rename tests/{Feature => Integration}/FindTest.php (53%) create mode 100644 tests/Integration/GetDBsTest.php create mode 100644 tests/Integration/GlobalIndexTest.php create mode 100644 tests/Integration/HardeningRegressionTest.php create mode 100644 tests/Integration/HeldDataTest.php create mode 100644 tests/Integration/IndexHardeningTest.php rename tests/{Feature => Integration}/InsertTest.php (66%) create mode 100644 tests/Integration/IntegrityIndexTest.php create mode 100644 tests/Integration/KeyIndexFormatTest.php create mode 100644 tests/Integration/MetricsTest.php create mode 100644 tests/Integration/NewChainingMethodsTest.php rename tests/{Feature => Integration}/NewMethodsEdgeCasesTest.php (62%) create mode 100644 tests/Integration/NewMethodsShardedTest.php create mode 100644 tests/Integration/OperationsToolsTest.php rename tests/{Regression => Integration}/PublicApiCoverageTest.php (55%) create mode 100644 tests/Integration/QueryDocumentationTest.php create mode 100644 tests/Integration/QueryEngineTest.php create mode 100644 tests/Integration/QueryHardeningTest.php create mode 100644 tests/Integration/QueryMethodsTest.php rename tests/{Regression => Integration}/QueryRegressionTest.php (73%) create mode 100644 tests/Integration/RTreeTest.php create mode 100644 tests/Integration/RemoveFieldsTest.php rename tests/{Regression => Integration}/SecurityConfigRegressionTest.php (55%) rename tests/{Feature => Integration}/ShardedFieldIndexTest.php (51%) rename tests/{Feature => Integration}/ShardingTest.php (59%) rename tests/{Feature => Integration}/SpatialIndexTest.php (70%) rename tests/{Feature => Integration}/SpatialOperatorCombinationTest.php (73%) rename tests/{Feature => Integration}/SpatialRealWorldTest.php (69%) rename tests/{Regression => Integration}/SpatialRegressionTest.php (80%) create mode 100644 tests/Integration/SpecIndexTest.php create mode 100644 tests/Integration/SpecOperationsTest.php create mode 100644 tests/Integration/SpecQueryTest.php create mode 100644 tests/Integration/SpecSpatialTest.php create mode 100644 tests/Integration/SpecStorageTest.php rename tests/{Regression => Integration}/StorageRegressionTest.php (67%) create mode 100644 tests/Integration/StoreFormatTest.php create mode 100644 tests/Integration/StrictOptionsTest.php create mode 100644 tests/Integration/TypeEdgeCasesTest.php rename tests/{Feature => Integration}/UpdateTest.php (60%) rename tests/{Feature => Integration}/UtilityMethodsTest.php (54%) create mode 100644 tests/InternalMethod.php create mode 100644 tests/InternalValue.php create mode 100644 tests/Migration/MigrationFixtureTest.php create mode 100644 tests/Migration/MigrationProcessTest.php create mode 100644 tests/Migration/MigrationRulesTest.php create mode 100644 tests/Migration/MigrationTestCase.php create mode 100644 tests/Performance/durability_cost.php create mode 100644 tests/Performance/large.php rename tests/{ => Performance}/operation_benchmark.php (92%) rename tests/{ => Performance}/operator_benchmark.php (95%) create mode 100644 tests/Performance/perf_gate.php rename tests/{ => Performance}/performance_benchmark.php (95%) rename tests/{ => Performance}/sleekdb_comparison.php (95%) rename tests/{ => Performance}/spatial_benchmark.php (81%) create mode 100644 tests/Performance/sqlite_comparison.php delete mode 100644 tests/Regression/LockStateRegressionTest.php rename tests/{Regression => }/RegressionTestCase.php (59%) create mode 100644 tests/Smoke/noneDB_critical_scope_regression.php create mode 100644 tests/Smoke/noneDB_regression_smoke.php create mode 100644 tests/Support/Testing.php create mode 100644 tests/Unit/ArchitectureTest.php create mode 100644 tests/Unit/BuildTest.php create mode 100644 tests/Unit/DatabaseLocatorTest.php create mode 100644 tests/Unit/GeometryValidationTest.php delete mode 100644 tests/Unit/HashDBNameTest.php create mode 100644 tests/Unit/ShardFiltersTest.php create mode 100644 tests/Unit/SourceWriteGuardTest.php delete mode 100644 tests/buffer_test.php delete mode 100644 tests/concurrent_insert_test.php delete mode 100644 tests/concurrent_stress_test.php delete mode 100644 tests/sleekdb_vs_nonedb_benchmark.php create mode 100644 tools/build.php create mode 100644 tools/make-migration-fixtures.php create mode 100644 tools/soak.php diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0ef9a92 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +# Text is LF on every platform (tools/build.php refuses CR) +* text=auto eol=lf + +# noneDB.php is built from src/ by tools/build.php; packages ship only the built files +/src export-ignore +/migrate export-ignore +/tools export-ignore +/tests export-ignore +/docs export-ignore +/examples export-ignore +/build export-ignore +/.github export-ignore +/.gitattributes export-ignore +/.gitignore export-ignore +/phpunit.xml export-ignore +/CHANGES.md export-ignore +noneDB.php linguist-generated=true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 61e8acd..1f0a884 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,24 +7,74 @@ on: branches: [main, master, dev] jobs: + fixtures: + # The databases the earlier releases write themselves: the Migration suite reads them, git keeps none + name: Migration fixtures + runs-on: ubuntu-latest + + steps: + - name: Checkout code with its history (the earlier releases come from git) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup PHP 7.4 (releases before 3.2) + uses: shivammathur/setup-php@v2 + with: + php-version: '7.4' + extensions: json, mbstring, posix + coverage: none + + - name: Keep its path + run: echo "NONEDB_PHP74=$(php -r 'echo PHP_BINARY;')" >> $GITHUB_ENV + + - name: Setup PHP 8.2 + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + extensions: json, mbstring, posix + coverage: none + + - name: Make the fixtures + run: php tools/make-migration-fixtures.php + + - name: Pack them (dot files included, bytes as written) + run: tar -czf migration-fixtures.tgz -C tests/Migration fixtures + + - name: Hand them to the other jobs + uses: actions/upload-artifact@v4 + with: + name: migration-fixtures + path: migration-fixtures.tgz + retention-days: 1 + test: name: PHP ${{ matrix.php-version }} + needs: fixtures runs-on: ubuntu-latest strategy: fail-fast: false matrix: - php-version: ['7.4', '8.0', '8.1', '8.2', '8.3', '8.4', '8.5'] + php-version: ['8.1', '8.2', '8.3', '8.4', '8.5'] steps: - name: Checkout code uses: actions/checkout@v4 + - name: Get the migration fixtures + uses: actions/download-artifact@v4 + with: + name: migration-fixtures + + - name: Unpack them + run: tar -xzf migration-fixtures.tgz -C tests/Migration + - name: Setup PHP ${{ matrix.php-version }} uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-version }} - extensions: json, mbstring + extensions: json, mbstring, posix, pdo_sqlite tools: composer:v2 coverage: none @@ -46,9 +96,23 @@ jobs: - name: Install dependencies run: composer install --prefer-dist --no-progress --no-interaction - - name: Run PHPUnit tests (Unit, Feature, Integration, Regression) + - name: Check that noneDB.php is the build of src/ + run: php tools/build.php --check + + - name: Run PHPUnit (Unit, Integration, Crash, Concurrency, Differential, Migration) run: vendor/bin/phpunit --testdox --colors=always + - name: Run the examples of the documentation against noneDB.php + run: php tests/Docs/doctest.php + + - name: Run the SQLite comparison once (a smoke test, no timing is checked) + run: php tests/Performance/sqlite_comparison.php 100 --runs=1 + + - name: Run the standalone smoke checks + run: | + php tests/Smoke/noneDB_regression_smoke.php + php tests/Smoke/noneDB_critical_scope_regression.php + - name: Run the examples (each must finish without warnings) run: | for f in examples/*.php; do @@ -60,3 +124,52 @@ jobs: - name: Cleanup test databases if: always() run: rm -rf tests/test_db examples/db + + windows: + # Windows (file locking, rename retries): informational until it has been seen green + name: PHP ${{ matrix.php-version }} (Windows) + needs: fixtures + runs-on: windows-latest + continue-on-error: true + + strategy: + fail-fast: false + matrix: + php-version: ['8.2', '8.4'] + + defaults: + run: + shell: bash + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get the migration fixtures + uses: actions/download-artifact@v4 + with: + name: migration-fixtures + + - name: Unpack them + run: tar -xzf migration-fixtures.tgz -C tests/Migration + + - name: Setup PHP ${{ matrix.php-version }} + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-version }} + extensions: json, mbstring + tools: composer:v2 + coverage: none + + - name: Install dependencies + run: composer install --prefer-dist --no-progress --no-interaction + + - name: Check that noneDB.php is the build of src/ + run: php tools/build.php --check + + - name: Run PHPUnit tests + run: vendor/bin/phpunit --colors=always + + - name: Cleanup test databases + if: always() + run: rm -rf tests/test_db examples/db diff --git a/.gitignore b/.gitignore index a3c39e3..e1b622e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,19 +3,28 @@ composer.phar composer.lock -# Database files +# Database files (noneDB 4 store; files of earlier versions until they are migrated) +/db/*.ndb4/ +/db/STORE +/db/STORE.2 +/db/STORE.LOCK +/db/STORE*.tmp* +/db/.nonedb-maintenance/ +/db/.nonedb-retained/ /db/*.nonedb /db/*.nonedbinfo /db/*.nonedb.* # Test artifacts /tests/test_db/ +/tests/build/ .phpunit.result.cache .phpunit.cache/ # IDE .idea/ .vscode/ +.claude/ *.sublime-* # OS @@ -38,3 +47,21 @@ CLAUDE.md # Example databases /examples/db/ + +# Evidence stays on this machine; git keeps only the frozen baseline (the gate and the 3.3-dev fixtures need it) +/build/* +!/build/baseline/ + +*.nonedb +*.jidx +*.meta +*.backup +*.nonedbinfo + +# Databases the earlier releases wrote: tools/make-migration-fixtures.php makes them from git (CI too) +/tests/Migration/fixtures/ + +# Working notes, a tool of the 3.3 gates nothing uses any more +/v4.0.0.plan.md +/docs/v4/ +/tools/gate-evidence.php diff --git a/.nonedb.example b/.nonedb.example index 743e163..77d409c 100644 --- a/.nonedb.example +++ b/.nonedb.example @@ -4,7 +4,6 @@ "autoCreateDB": true, "shardingEnabled": true, "shardSize": 10000, - "autoMigrate": true, "autoCompactThreshold": 0.3, "lockTimeout": 5, "lockRetryDelay": 10000 diff --git a/CHANGES.md b/CHANGES.md index ed1a443..964e6fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,227 @@ # noneDB Changelog +## v4.0.0 (unreleased) + +### A new engine and a new storage format + +noneDB 4 keeps what noneDB is: one PHP file, no server, no extension, plain JSON lines on disk. Underneath, the +engine is new. It has its own storage format (format 4) and opens a database only when it can prove what the +database is. Writes are rolled back when they fail, and crash recovery never loses a write it acknowledged. Durable +mode survives a power loss. Backups are consistent, metrics are cheap, and the costs of operations are bounded. + +noneDB 4 does not read the files of noneDB 1.x–3.x, and has no compatibility layer for them. A separate tool, +`noneDB-migrate.php`, converts them once, verified and reversible ([docs/MIGRATION.md](docs/MIGRATION.md)). + +Requirements: PHP 8.1 or later, 64-bit (tested on 8.1–8.5); the `json` extension, and `mbstring` for Unicode case +folding in `like()`. + +What "production-ready" covers: embedded use on one machine with a local file system. It does not cover +multi-record ACID transactions, or network file systems (NFS, SMB, a volume shared by several containers). + +--- + +### Upgrading from 1.x–3.x (read first) + +1. **Migrate the databases.** Stop every process that uses the directory, then run the tool with the key the + databases were written with: + ```bash + NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --dry-run + NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --backup-dir=/var/backups/nonedb --confirm-stopped --verify + ``` + The tool reads 1.4, 2.0–2.3, 3.0–3.2 and the unreleased 3.3 development state. Each version's records come + through as that version returned them, with their keys. Records a version had deleted stay deleted, even where + its data file still holds them. The old files are backed up first, then kept until `--cleanup`; `--restore` + brings them back. A database whose files don't determine its records is reported and left alone: nothing is + guessed. +2. **Until then the engine refuses the old files.** While `dbDir` holds files that are not noneDB 4 objects, no + database is created and none is answered as missing (`UNSUPPORTED_FORMAT`). An old database never looks empty. + noneDB 4 databases in the same directory keep working. +3. **PHP 8.1+.** PHP 7.4 and 8.0 are no longer supported. +4. **Removed methods:** `enableIndexing()`, `isIndexingEnabled()`, `flushSpatialIndexes()`, `clearDistanceCache()`, + `clearCentroidCache()` and `noneDBQuery::compareValues()`. Index use follows `fieldIndexEnabled` and + `spatialIndexEnabled`; spatial indexes are written by the operations themselves; there are no distance caches. +5. **Changed:** + - `migrate()` is now `convertToSharded()`. It reports `"converted"` (else `"already_sharded"`, + `"database_not_found"`, `"conversion_failed"`). + - The `autoMigrate` setting is gone: `shardingEnabled` alone decides whether a database switches to shards + by itself once it has assigned `shardSize` keys. Remove it from your `.nonedb` before you migrate: noneDB 4 + refuses settings it doesn't know (item 7), and so does the migration tool's pre-check, for a `.nonedb` the engine + finds itself. + - `migrateKey()` is a journaled store operation with its own rules (see "Keys" below). Its fourth argument + takes options (`abort`). +6. **Error codes are new:** `RuntimeException` codes, with constants of `noneDB` (`LOCK_FAILED`, + `UNSUPPORTED_FORMAT`, `FORMAT_DAMAGED`, `KEY_NOT_REGISTERED`, …; docs/API.md "Errors"). A failure after a + commit point, or an undo that failed, is `OUTCOME_UNKNOWN` (3301). +7. **Settings are checked:** + - A key that is not a setting is an `InvalidArgumentException`, in the array and in a `.nonedb` file. The + message names it and suggests the setting it was probably meant to be, so a misspelled `durability` never + means performance mode. + - So is a setting given as `null`. + - The options of a method are checked the same way: `createSpatialIndex()`, `withinDistance()`, `nearest()`, + `restore()` and the key maintenance methods. + - `durability`, `cacheMemoryLimit`, `filePermissions` and `directoryPermissions` are new. + - Bool settings accept only `true/false`, `1/0` and `"1"/"0"/"true"/"false"`. + - `lockTimeout` takes fractions of a second (docs/CONFIGURATION.md). +8. **Keys:** a store accepts only the keys registered for it. The first key creates the store; another key gets + `KEY_NOT_REGISTERED` (never an empty database) until `registerKey()` adds it. + +--- + +### Storage format 4 (docs/STORAGE_FORMAT.md, normative) + +- **A store record** (`STORE`, `STORE.2`: two copies, a sequence number and a checksum) lists the keys that may open + the directory. +- **One directory per database**, `{locator}.ndb4/`. The locator is an HMAC of the name under the key: a name + can't be guessed without the key. + - `FORMAT` is the database's identity: its `databaseId` (a UUID), its name and its shard size. It doesn't + depend on the key. + - `LOCK` holds the database lock and a 128-byte state record (generation, epoch, flags). + - `CURRENT` names the active generation, `generations/N/`. Changing a database's layout, or restoring it, builds + a new generation and publishes it by one atomic replacement of `CURRENT`. +- **Fail closed.** A database whose identity or active generation can't be proven fails with `FORMAT_DAMAGED`; + nothing is guessed and nothing is removed. A file is repaired only with proof (the identity copy in the + generation's metadata), and the repair is logged. +- **Data** stays JSON lines, with the key last in every line. Updates and deletes append; an unterminated last + line is never a record. The next writer moves it to a verified quarantine file and cuts it. +- **Indexes are checkpoints of the data** and are rebuilt from it when they don't fit: + - key indexes (`.kix`): 4 KiB blocks, each with its crc; dense or sparse; + - field indexes (`.fx`): a static B+tree of ~4 KiB pages; + - the global field index (`.gfx`) of a sharded database: one Bloom filter per shard, stored bit-sliced; + - R-trees (`.rx`), and each shard's bounding box (`.grx`). + + A write appends to the data and leaves the index files alone. Checkpoints are written at the end of a session, + past a threshold. + +### Writes, durability and recovery (docs/DURABILITY.md) + +- **Every write is one operation with one outcome:** + - `committed`; + - `rolled_back`: a failure before the commit point; every append of the operation is cut back; + - `unknown`: a failure after the commit point, `OUTCOME_UNKNOWN`. The next access recovers. + + `getDurability()['lastOperation']` reports it. +- **`durability`:** `'performance'` (the default: what the operating system has accepted survives a crash of the + process) or `'durable'` (Linux; every write is synced before it returns, so a power loss doesn't lose it). + `'durable'` elsewhere is `UNSUPPORTED_DURABILITY` at construction. A sync that fails before the commit point is + `DURABILITY_FAILED`, rolled back. +- **Locks:** a database lock across processes for every operation (shared to read, exclusive to write), and + `lockTimeout` for every wait (`LOCK_FAILED`). +- **Crash recovery** applies complete lines and quarantines the rest; the recovery log (`getRecoveryLog()`) says + what was done. A SIGKILL at any point of any operation leaves the state before or after it: tested at every + step of every state machine (creation, drop, generation switch, compaction, key maintenance, migration, restore). +- **`checkIntegrity($db)`** reads every line and compares indexes and counters with the data. It writes nothing. + +### Keys (STORAGE_FORMAT.md §9) + +- `registerKey($dir, $registeredKey, $newKey)`: lets a second key open the store. +- `migrateKey($dir, $oldKey, $newKey)`: renames every database of the old key to its locator under the new one. + - **Journaled, one commit point.** An interruption leaves the store closed (`MAINTENANCE_IN_PROGRESS`) until the + same call runs again: before the commit it continues, or rolls back with `['abort' => true]`; after it, it + only finishes. + - **Busy:** a database in use makes it `MAINTENANCE_BUSY`, and nothing is written. + - **No stale answers:** a process still running with the old key gets `MAINTENANCE_IN_PROGRESS` or + `KEY_NOT_REGISTERED`, never an empty database. +- `repairStore($dir, $keys, $options)`: restores a store record whose two copies are both lost, from a saved + `storeManifest` (complete) or, explicitly confirmed, from the keys given (`complete` false). One damaged copy is + repaired automatically. + +### Backup and restore (new, docs/BACKUP_RESTORE.md) + +- **`backup($db, $dir)`:** one database at one moment, taken while it is in use. Its files are verified, + `manifest.json` (SHA-256 of every file) is written last, and `store.json` is included. The directory is 0700, + its files 0600. +- **`restore($dir, $db, $options)`:** + - checks the backup completely, then publishes it as a new generation in one commit; + - into the database it was taken from, into an absent one, or over a damaged one (whose generations are kept); + - keys handed out after the backup are never handed out again; + - another database's backup only with `force`. + +### Monitoring (new, docs/OPERATIONS.md) + +- **`getMetrics($db = null)`:** counters of the process and metadata of a database. Nothing is scanned or written. + - the process: cache hits, misses and evictions, index reads, checkpoints, compactions, lock waits and + timeouts, recovery events, durability, the last integrity check, memory; + - a database: its identity, layout, counters and sizes by kind of file. + +### Security + +- **Permissions:** files `0640`, directories `0750` by default (`filePermissions`, `directoryPermissions`). +- **Paths:** `dbDir` is used by its canonical path, and a database directory that is a link is refused. +- **No secrets in logs:** recovery logs, metrics and error messages hold no key and no record value (tested). + +### Queries (docs/QUERY.md) + +- **A planner** reads only the candidates of each segment: + - field index buckets and ranges, and key lookups; + - it skips the shards the global indexes rule out; + - it stops as soon as `first()`, `exists()` or `limit()` has enough; + - top-k for `sort()` with a limit of up to 1000. + + `explain()` shows the plan without reading records. Every record read is still checked against every filter. +- **Sort order is total:** null < booleans < numbers < numeric strings < other strings < arrays; `min()`/`max()` use + it too. +- **Semantics fixed since 3.2:** + - `whereNot()` applies operators; + - a `key` filter means the same in every clause; + - NAN matches nothing; + - `limit(0)` returns nothing; + - aggregates ignore `select()`; + - groups and `distinct()` use exact typed identity; + - joins compare strictly and don't truncate floats; + - `like()` and `search()` fold Unicode case; + - a trailing `$` of a like pattern means "ends with". +- **Field indexes match by type**, as `find()` does: `1`, `1.0`, `"1"` and `true` are different values, and floats + don't depend on `precision` or the locale. +- **Spatial fixes:** + - polygons are closed (their east and north edges are included), and holes are respected; + - MultiPolygon components are handled one by one; + - malformed GeoJSON is invalid; + - distances to edges are exact; + - polygon validation is O(n log n), with exact orientation tests. + +### Performance (docs/PERFORMANCE.md) + +- Measured against the frozen 3.3 development state with the statistical performance gate (two independent runs, + ABBA child processes). That state was itself measured against 3.2.0: single writes to 10,000 records 12–17 times + faster, and no verified regression. The results and the known limits are in PERFORMANCE.md. +- **Bounded memory.** An operation holds the cache, its answer and one batch of the segment it works on, never the + whole database or a whole segment. A segment is read 10,000 records at a time, so memory doesn't grow with + `shardSize` or with a database that is not sharded (docs/CONFIGURATION.md). This holds for: + - reads, aggregations and writes by filter; + - field index builds and recovery; + - `checkIntegrity()` and the switch to shards. + + `compact()` holds its segment's live bytes, and a spatial index build its entries. With shards of 100,000 records, + a scan peaks at about 41 MiB and `checkIntegrity()` at 43 MiB (3.3-dev: 139 and 215 MiB). The migration tool + converts 10 million records at PHP's default `memory_limit` of 128M. +- **10 million records** (1000 shards): + - the first indexed query of a process takes 0.27 s (3.3-dev: 7.3 s); + - a warm query takes 26 ms and its process peaks at 1.3 GiB (3.3-dev: 31 ms, 2.3 GiB); + - queries work within a 512 MiB process (3.3-dev ran out of memory). +- **Data files in repeated use are held.** In a database of more than 16 segments, a process with a memory limit reads a + data file whole once its use pays for it, and its records no longer cost an open and a read each. The bytes stay under + half of the cache limit (docs/CONFIGURATION.md "The cache"), and they never change an answer. + +### Code and tests + +- **Modules:** `noneDB.php` is built from `src/` (one class per responsibility, `noneDB\Internal\…`), and + `noneDB-migrate.php` from `migrate/`. `php tools/build.php --check` proves both artifacts are the build. Neither + artifact contains test code: the test build adds fault points and counters. +- **Suites:** about 1950 tests (unit, integration, crash, concurrency, differential, migration), on PHP 8.1–8.5 + (8.3–8.5 and durable mode on Linux). + - Expected values come from the specification or from the test's own models of the formats. + - Mutation checks for every new mechanism. + - Migration fixtures are written by the old releases themselves. +- **Documentation examples** run in CI against the built library (`tests/Docs/doctest.php`). + +--- + ## v3.2.0 (2026-10-23) ### Reliability, security and correctness release -This release fixes every finding of the full code review (see `CODE_REVIEW.md`). Most changes are +This release fixes every finding of a full code review. The IDs in parentheses (K-01, Y-14, ...) name +those findings; the tests in `tests/Regression` are annotated with the same IDs. Most changes are internal; the few behaviour changes are listed under **Upgrade notes** below. --- diff --git a/README.md b/README.md index 6f270d7..0bbc225 100755 --- a/README.md +++ b/README.md @@ -1,281 +1,238 @@ # noneDB -[![Version](https://img.shields.io/badge/version-3.2.0-orange.svg)](CHANGES.md) -[![PHP Version](https://img.shields.io/badge/PHP-7.4%2B-blue.svg)](https://php.net) +[![Version](https://img.shields.io/badge/version-4.0.0-blue.svg)](CHANGES.md) +[![PHP Version](https://img.shields.io/badge/PHP-8.1%2B-blue.svg)](https://php.net) [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![Tests](https://img.shields.io/badge/tests-1084%20passed-brightgreen.svg)](tests/) -[![Concurrency](https://img.shields.io/badge/multi--process-safe-success.svg)](#concurrent-access) +[![Concurrency](https://img.shields.io/badge/multi--process-safe-success.svg)](#concurrency-and-crashes) -**noneDB** is a lightweight, file-based NoSQL database for PHP. No installation required - just include and go! +**noneDB** is an embedded document database for PHP: one file, no server, no extension. Include it and store +records in a directory. ## Features -- **Zero dependencies** - single PHP file -- **No database server required** - just include and use -- **O(1) key lookups** - JSONL storage with byte-offset indexing -- **Spatial indexing** - R-tree for geospatial queries (v3.1) -- **MongoDB-style operators** - `$gt`, `$gte`, `$lt`, `$lte`, `$ne`, `$in`, `$nin`, `$exists`, `$like`, `$regex`, `$contains` (v3.1) -- **Auto-sharding** for large datasets -- **Multi-process safe** - database-level locking, crash-safe writes, self-healing indexes -- **Method chaining** - fluent query builder interface +- **One file, no dependencies**: `noneDB.php`, PHP 8.1 or later +- **Records and queries**: insert, find, update, delete; a fluent query builder with MongoDB-style operators, + sorting, joins, grouping, aggregates and `explain()` +- **Indexes on disk**: key lookups through key-index checkpoints, paged field indexes, a global field index + per sharded database, R-tree spatial indexes for GeoJSON; a lookup reads only the pages it needs +- **Large databases**: automatic sharding; queries on 10 million records run in PHP's default `memory_limit` (128 MB) +- **Safe under processes and crashes**: database locks across processes, writes that throw are undone, a + killed process loses no acknowledged write, recovery is deterministic, damaged files are quarantined and + never silently read +- **Durable mode**: every write synced to stable storage before it returns (Linux) +- **Operations**: online backup and one-commit restore, metrics, integrity checks, key rotation +- **Migration** of databases written by noneDB 1.x to 3.x: a separate tool, `noneDB-migrate.php` ## Requirements -- PHP 7.4 or higher (tested on 7.4 up to 8.5) -- Write permission on database directory +- PHP 8.1 to 8.5 (`json` and `ctype`, on by default; `mbstring` recommended) +- A local file system (ext4, xfs, btrfs, APFS, NTFS). Network file systems and volumes shared by several + machines are not supported: locks use `flock()`. +- Durable mode: Linux. Performance mode (the default): Linux, macOS, Windows. ## Installation -### Manual -```php -include("noneDB.php"); -$db = new noneDB(); -``` - -### Composer ```bash composer require orhanayd/nonedb ``` ---- +or copy `noneDB.php` into your project: -## Quick Start + +```php +require 'noneDB.php'; +``` + +## Quick start ```php -insert("users", ["name" => "John", "age" => 25, "email" => "john@example.com"]); +$db->insert("users", [["name" => "Jane", "age" => 31], ["name" => "Ali", "age" => 17]]); -// Find -$users = $db->find("users", ["name" => "John"]); - -// Query Builder with Operators -$results = $db->query("users") - ->where([ - 'age' => ['$gte' => 18, '$lte' => 65], - 'status' => 'active' - ]) +$adults = $db->query("users") + ->where(['age' => ['$gte' => 18]]) ->sort('age', 'desc') ->limit(10) ->get(); -// Update -$db->update("users", [ - ["name" => "John"], - ["set" => ["email" => "john.doe@example.com"]] -]); - -// Delete -$db->delete("users", ["name" => "John"]); +$db->update("users", [["name" => "John"], ["set" => ["email" => "john.doe@example.com"]]]); +$db->delete("users", ["name" => "Ali"]); ``` ---- - ## Configuration -Create a `.nonedb` file in your project root (outside the web root): +Put a `.nonedb` file outside the web root, next to your entry script or in the project root: ```json { "secretKey": "YOUR_SECURE_RANDOM_STRING", - "dbDir": "./db/", - "autoCreateDB": true + "dbDir": "./db/" } ``` -- `secretKey` must be a non-empty string and not one of the example placeholders (32+ random characters - recommended: `php -r "echo bin2hex(random_bytes(32));"`). It makes database **file names** - unguessable; it does **not** encrypt the data — keep `dbDir` out of the web root. -- A relative `dbDir` (`./db/` or `db/`) is resolved against the `.nonedb` file's directory, so web pages - and CLI/cron scripts use the same databases. -- To change the key of existing databases use `noneDB::migrateKey($dbDir, $oldKey, $newKey)`. +- `secretKey`: a random secret of your own: `php -r "echo bin2hex(random_bytes(32));"` prints one. Published + placeholders, the one above too, are refused. It names the files of your databases, so they can't be guessed; it + does **not** encrypt them. Keep `dbDir` out of the web root. +- `dbDir`: where the databases live. A relative path is relative to the `.nonedb` file. +- Or pass the settings as an array: `new noneDB(['secretKey' => getenv('NONEDB_SECRET'), 'dbDir' => '/var/lib/app/db'])`. -**Or programmatically:** +Every setting (sharding, buffer, locks, durability, cache, permissions): [docs/CONFIGURATION.md](docs/CONFIGURATION.md). -```php -$db = new noneDB([ - 'secretKey' => getenv('NONEDB_SECRET'), // your own random key (example placeholders are rejected) - 'dbDir' => '/path/to/db/' -]); -``` - -**Development mode (no config required):** +## Queries ```php -noneDB::setDevMode(true); -$db = new noneDB(); -``` - -> See [docs/CONFIGURATION.md](docs/CONFIGURATION.md) for all options. - -### Database names - -Names may contain `A-Z a-z 0-9`, space, apostrophe (`'`) and hyphen (`-`). Anything else throws -`InvalidArgumentException` (names are never silently changed). `"0"` is a valid name. - ---- - -## Query Builder +$db->insert("products", [ + ["name" => "Lamp", "price" => 120, "category" => "home", "stock" => 4, "tags" => ["featured"]], + ["name" => "Phone", "price" => 480, "category" => "electronics", "stock" => 0, "tags" => []], +]); -```php -// Comparison operators $db->query("products") ->where([ 'price' => ['$gte' => 100, '$lte' => 500], - 'category' => ['$in' => ['electronics', 'gadgets']], - 'stock' => ['$gt' => 0] + 'category' => ['$in' => ['electronics', 'home']], + 'stock' => ['$gt' => 0], ]) - ->sort('rating', 'desc') - ->limit(20) - ->get(); - -// Pattern matching -$db->query("users") - ->where(['email' => ['$like' => 'gmail.com$']]) + ->sort('price', 'desc') ->get(); -// Existence check -$db->query("users") - ->where(['phone' => ['$exists' => true]]) - ->get(); +$db->query("products")->where(['tags' => ['$contains' => 'featured']])->count(); +$db->query("products")->where(['name' => ['$like' => '^La']])->first(); ``` -### Available Operators +| Operator | Matches | Example | +|---|---|---| +| `$eq`, `$ne` | equal / not equal (`===`) | `['status' => ['$ne' => 'guest']]` | +| `$gt`, `$gte`, `$lt`, `$lte` | numbers and numeric strings by value, strings lexically | `['age' => ['$gte' => 18]]` | +| `$in`, `$nin` | in / not in a list (`===`) | `['role' => ['$in' => ['a', 'b']]]` | +| `$exists` | the field is there (or not) | `['email' => ['$exists' => true]]` | +| `$like` | a pattern: `^` starts with, `$` ends with | `['name' => ['$like' => '^Jo']]` | +| `$regex` | a regular expression, case-insensitive | `['email' => ['$regex' => '@gmail\\.com$']]` | +| `$contains` | an array element or a substring | `['tags' => ['$contains' => 'featured']]` | -| Operator | Description | Example | -|----------|-------------|---------| -| `$gt` | Greater than | `['age' => ['$gt' => 18]]` | -| `$gte` | Greater than or equal | `['price' => ['$gte' => 100]]` | -| `$lt` | Less than | `['stock' => ['$lt' => 10]]` | -| `$lte` | Less than or equal | `['rating' => ['$lte' => 5]]` | -| `$ne` | Not equal | `['role' => ['$ne' => 'guest']]` | -| `$in` | In array | `['category' => ['$in' => ['a', 'b']]]` | -| `$nin` | Not in array | `['tag' => ['$nin' => ['spam']]]` | -| `$exists` | Field exists | `['email' => ['$exists' => true]]` | -| `$like` | Pattern match | `['name' => ['$like' => '^John']]` | -| `$regex` | Regex match | `['email' => ['$regex' => '@gmail']]` | -| `$contains` | Array/string contains | `['tags' => ['$contains' => 'featured']]` | +A field index makes equality, `$in` and range filters read only the matching records: -> See [docs/QUERY.md](docs/QUERY.md) for complete query reference. +```php +$db->createFieldIndex("products", "category"); +$db->query("products")->where(['category' => 'home'])->explain(); // the plan: which index, which shards +``` ---- +The query reference: [docs/QUERY.md](docs/QUERY.md). -## Spatial Queries +## Spatial queries ```php -// Create spatial index $db->createSpatialIndex("restaurants", "location"); - -// Insert GeoJSON data $db->insert("restaurants", [ 'name' => 'Ottoman Kitchen', - 'location' => ['type' => 'Point', 'coordinates' => [28.9784, 41.0082]] + 'open' => true, + 'location' => ['type' => 'Point', 'coordinates' => [28.9784, 41.0082]], ]); -// Find within radius $nearby = $db->query("restaurants") - ->withinDistance('location', 28.9784, 41.0082, 5000) // 5000 meters (5km) - ->where(['open_now' => true]) + ->withinDistance('location', 28.9784, 41.0082, 5000) // meters + ->where(['open' => true]) ->get(); - -// Find nearest K $closest = $db->nearest("restaurants", "location", 28.9784, 41.0082, 10); - -// Find in bounding box -$inArea = $db->withinBBox("restaurants", "location", 28.97, 41.00, 29.00, 41.03); ``` -> See [docs/SPATIAL.md](docs/SPATIAL.md) for complete spatial reference. +Geometries, rules and the R-tree: [docs/SPATIAL.md](docs/SPATIAL.md). ---- +## Concurrency and crashes -## Documentation +- Every operation locks its database: shared for reads, exclusive for writes, across processes. Concurrent + writes are serialized; keys are never reused. +- A write that throws leaves no record behind. A process that is killed loses no write that returned: complete + lines are kept, an unterminated line is never a record and is moved to a verified quarantine file before the + next writer cuts it, indexes replay the data after their checkpoints. +- Files that don't fit their checksums or their records are never silently used: a database that can't be + proven consistent fails with `FORMAT_DAMAGED`, and `restore()` from a backup is the way out. +- A multi-record call (`insert([...1000 records])`) is not a transaction under a kill: a prefix of it may stay. -| Document | Description | -|----------|-------------| -| [docs/QUERY.md](docs/QUERY.md) | Query builder, operators, filters | -| [docs/SPATIAL.md](docs/SPATIAL.md) | Geospatial indexing and queries | -| [docs/CONFIGURATION.md](docs/CONFIGURATION.md) | Configuration options | -| [docs/API.md](docs/API.md) | Complete API reference | +| Mode | A write that returned survives | +|---|---| +| `'durability' => 'performance'` (default) | the process dying (SIGKILL, fatal error), as long as the operating system runs on | +| `'durability' => 'durable'` (Linux) | a power loss as well: every sync is done before the write returns, or it fails | ---- +The contract: [docs/DURABILITY.md](docs/DURABILITY.md); the file format: [docs/STORAGE_FORMAT.md](docs/STORAGE_FORMAT.md). -## Performance +## Operations -| Operation | 10K Records | 100K Records | -|-----------|-------------|--------------| -| find(key) | **0.05 ms** | **0.06 ms** | -| find(key), first lookup of a process | 3 ms | 4 ms | -| find(filter) | 32 ms | 336 ms | -| insert (batch) | 121 ms | 1.3 s | -| update (filter, 20% of records) | 40 ms | 455 ms | -| count() | **< 1 ms** | **< 1 ms** | - -> PHP 8.2 on an Apple M2, including the per-operation database lock. Key lookups are O(1) with -> byte-offset indexing. Details, 3.1 comparison and SleekDB comparison: [docs/BENCHMARKS.md](docs/BENCHMARKS.md). +```php +$backup = __DIR__ . '/backups/users-' . date('Ymd-His'); // outside dbDir +$db->backup("users", $backup); // online: one consistent moment, verified, manifest last +$db->restore($backup, "users"); // one commit: the database is as it was, or restored ---- +$db->checkIntegrity("users"); // a full read-only check +$db->getMetrics("users"); // cache, locks, checkpoints, sizes, state: cheap counters +``` -## Testing +Backups, restores, monitoring, key rotation and what to do when the engine refuses to work: +[docs/BACKUP_RESTORE.md](docs/BACKUP_RESTORE.md), [docs/OPERATIONS.md](docs/OPERATIONS.md). -```bash -# Run all tests -composer test +## Upgrading from noneDB 1.x to 3.x -# Run with details -vendor/bin/phpunit --testdox +noneDB 4 reads only its own format and refuses a directory that holds older files (`UNSUPPORTED_FORMAT`): it +never mistakes them for an empty database. Convert them once with the migration tool, which verifies every +database and keeps a backup: -# Run specific suite -vendor/bin/phpunit --testsuite Feature +```bash +NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --backup-dir=/backups --confirm-stopped ``` ---- +The procedure and every source format: [docs/MIGRATION.md](docs/MIGRATION.md). What changed: [CHANGES.md](CHANGES.md). -## Concurrent Access - -Each operation locks its database (`flock()` on `{db}.nonedb.lock`): shared for reads, exclusive for -writes, across processes and instances. +## Performance -- **No lost updates** - concurrent inserts/updates/deletes are serialized; keys are never reused -- **Fresh caches** - a generation counter invalidates cached indexes when another process writes -- **Crash safety** - files are replaced atomically; a writer killed mid-operation leaves nothing - half-written: key, field and spatial indexes are repaired from the data on the next access -- **Lock timeout** - waiting longer than `lockTimeout` seconds throws `RuntimeException` +Measured against a fixed baseline with a statistical gate (two independent runs), on a database of 10 million +records, and against SQLite: [docs/PERFORMANCE.md](docs/PERFORMANCE.md). -The lock file also holds a small state record (change counter, pending buffer, index journal), so these -checks cost no extra file access. Don't delete `*.nonedb.lock` files while processes use the databases. +## Documentation -This is verified by multi-process stress tests and real `SIGKILL` crash tests in `tests/Regression`. +| Document | Contents | +|---|---| +| [docs/API.md](docs/API.md) | Every method, its results and errors | +| [docs/QUERY.md](docs/QUERY.md) | The query builder | +| [docs/SPATIAL.md](docs/SPATIAL.md) | Geospatial indexes and queries | +| [docs/CONFIGURATION.md](docs/CONFIGURATION.md) | Settings | +| [docs/DURABILITY.md](docs/DURABILITY.md) | What a successful, failed or uncertain write means | +| [docs/STORAGE_FORMAT.md](docs/STORAGE_FORMAT.md) | The files, normative | +| [docs/BACKUP_RESTORE.md](docs/BACKUP_RESTORE.md) | Backups and restores | +| [docs/OPERATIONS.md](docs/OPERATIONS.md) | Metrics, maintenance, failure situations | +| [docs/MIGRATION.md](docs/MIGRATION.md) | Converting databases of earlier versions | +| [docs/PERFORMANCE.md](docs/PERFORMANCE.md) | Measurements and how they are made | +| [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | The source tree and the build | +| [CHANGES.md](CHANGES.md) | Release notes | ---- +Runnable examples: [examples/](examples/) (`php examples/find.php`). -## Upgrading from 3.1 +## Development -Read the [v3.2.0 upgrade notes](CHANGES.md#upgrade-notes-read-before-upgrading): secretKey and -database names are validated, a relative `dbDir` follows the `.nonedb` file, buffering is off by default -and `compact()` no longer renumbers keys. Don't run 3.1 and 3.2 on the same database directory at the -same time. +`noneDB.php` and `noneDB-migrate.php` are built from `src/` and `migrate/`: edit those, never the artifacts. ---- +```bash +composer build # php tools/build.php +php tools/make-migration-fixtures.php # first: the databases the Migration suite reads (git history, PHP 7.4) +composer test # build check + every PHPUnit suite +vendor/bin/phpunit --testsuite Crash # Unit, Integration, Crash, Concurrency, Differential, Migration +php tests/Docs/doctest.php # the examples of the documentation, against noneDB.php +``` -## Contributing +## Disclaimer -1. Fork the repository -2. Create your feature branch (`git checkout -b feature/amazing-feature`) -3. Commit your changes (`git commit -m 'Add amazing feature'`) -4. Push to the branch (`git push origin feature/amazing-feature`) -5. Open a Pull Request +noneDB is provided "as is", without warranty of any kind, as the [MIT License](LICENSE) states. To the fullest +extent the law allows, its authors and contributors accept no liability for any loss or damage arising from its use, +including lost or corrupted data, downtime, security incidents and lost profits. ---- +You are responsible for deciding whether noneDB suits your use, testing it with your own data and workload, keeping +backups you have verified, and protecting your keys and files. noneDB does not encrypt data. The test results and +performance figures of this project describe the runs that produced them; they are no promise of how it will behave +for you. ## License -MIT License - see [LICENSE](LICENSE) file. - ---- +MIT License: see [LICENSE](LICENSE). ## Author diff --git a/build/baseline/noneDB-3.3dev.php b/build/baseline/noneDB-3.3dev.php new file mode 100644 index 0000000..cb2778e --- /dev/null +++ b/build/baseline/noneDB-3.3dev.php @@ -0,0 +1,13992 @@ + resolved entry script directory + + // Placeholder keys shipped in the docs/templates; never accepted as a real secret + private static $placeholderKeys = [ + 'CHANGE_THIS_TO_A_SECURE_RANDOM_STRING', + 'YOUR_SECURE_RANDOM_STRING', + 'YOUR_SECURE_RANDOM_STRING_HERE', + 'GENERATE_UNIQUE_64_CHAR_STRING', + 'your_secure_key', + 'your_key', + ]; + + // secretKey values live outside the object so print_r/var_export/serialize can't leak them + private static $secretKeys = []; + + private $dbDir = null; // Set via .nonedb config file or constructor + private $autoCreateDB = true; + private $devMode = false; + + // Sharding configuration + private $shardingEnabled=true; // Enable/disable auto-sharding + private $shardSize=10000; // Records per shard for NEW sharded databases (existing ones keep the size stored in their meta) + private $autoMigrate=true; // Auto-migrate legacy DBs to sharded format + + // File locking configuration + private $lockTimeout=5; // Max seconds to wait for a database lock (0 = wait forever) + private $lockRetryDelay=10000; // Microseconds between lock attempts (10ms) + + // Write buffer configuration (off by default since v3.2.0: direct writes are faster and always visible) + private $bufferEnabled=false; // Enable/disable write buffering + private $bufferSizeLimit=1048576; // 1MB buffer size limit per buffer + private $bufferCountLimit=10000; // Max records per buffer (safety limit) + private $bufferFlushOnRead=true; // Flush buffer before read operations + private $bufferFlushInterval=30; // Seconds between auto-flush (0 = disabled) + private $bufferAutoFlushOnShutdown=true; // Register shutdown handler for flush + + // Buffer state tracking (runtime) + private $bufferLastFlush=[]; // Track last flush time per DB/shard + private $shutdownHandlerRegistered=false; // Track if shutdown handler is registered + private static $bufferCounts=[]; // bufferPath => [size, count] so limits don't need a re-read per insert + private static $bufferEnds=[]; // bufferPath => size the buffer had after this process's last append + private static $flushingNow=[]; // database base path => true while its buffer is being flushed + + // Performance caches (runtime). All keys are full file paths (or key-fingerprint|dbname for hashes), + // so instances with different dbDir/secretKey never see each other's entries. + private $hashCache=[]; // keyFingerprint|dbname -> hash (PBKDF2 is expensive) + private $metaCache=[]; // meta path -> meta data + private $metaCacheTime=[]; // kept for API compatibility (unused since v3.2.0) + + // Index configuration + private $indexEnabled=true; // Kept for API compatibility (enableIndexing / isIndexingEnabled) + private $indexCache=[]; // jidx path -> JSONL index (see SegmentIndex) + private $verifiedIndexes=[]; // jidx path -> true: the cached index covers the data file as it is + private $shardedCache=[]; // meta path -> bool + + // JSONL Storage Engine + private $jsonlFormatCache=[]; // data path -> bool (JSONL vs legacy v2) + private $jsonlGarbageThreshold=0.3; // Trigger compaction when garbage > 30% of live records + + // Static caches for cross-instance sharing + private static $staticIndexCache=[]; + private static $staticVerifiedIndexes=[]; + private static $pendingRecoveryEvents=[]; // lock path -> event id -> event found by a reader, logged by the next writer + private static $uncheckpointed=[]; // lock path -> segments loaded without a checkpoint, written by the next writer + private static $indexStats=['loads' => 0, 'replayedLines' => 0, 'checkpoints' => 0]; // tests count the work + private static $staticShardedCache=[]; + private static $staticMetaCache=[]; + private static $staticMetaCacheTime=[]; + private static $staticHashCache=[]; + private static $staticFormatCache=[]; + private static $staticFileExistsCache=[]; + private static $staticSanitizeCache=[]; + private static $staticFieldIndexCache=[]; + private static $staticSpatialIndexCache=[]; + private static $staticGlobalSpatialIndexCache=[]; + private static $staticGenerations=[]; // lock path -> generation the caches were validated against + private static $staticHashFilesLoaded=[]; // hash cache file -> true + private static $staticCacheEnabled=true; // Enable/disable static caching + + // Process-wide lock registry (shared by every instance so two instances never self-deadlock) + private static $lockRegistry=[]; // lock path -> ['fp', 'mode' => LOCK_SH|LOCK_EX, 'depth', 'dbname', 'gen', 'flags', 'bumped', 'pending', 'scope', 'rolledBack', 'sid'] while locked + private static $lockHandles=[]; // lock path -> ['fp' => resource, 'pid' => int]: unlocked handles kept open for the next operation + private $generations=[]; // per-instance view of $staticGenerations + private static $sessionCounter=0; // id of the latest lock session ('sid' in the registry) + private static $shutdownCleanupRegistered=false; + private static $segmentPairTests=0; // edge pairs compared by ringSelfIntersects() (tests check the complexity) + private static $geometryValidations=0; // validateGeoJSON() calls (tests check that indexed candidates skip it) + private static $segmentReads=0; // readSegmentRecords() calls (tests check shard skipping) + private static $directoryListings=0; // listFilesWithPrefix() calls (tests check reads don't list the directory) + private static $pointEdgeTests=0; // point-edge tests of the polygon hole check (tests check the complexity) + private static $sweepBudget=null; // tests only: work the quick polygon checks may do before the sweep line (0 = sweep at once) + private static $mbstringFn=null; // tests only: callable() overriding "is mbstring available" + + // Durability (v3.3.0): optional fsync, what the last write did (getDurability()) + private $fsync=false; // sync data, meta and indexes to disk (power-loss durability where possible) + private static $syncFn=null; // tests only: callable($handle, 'file'|'dir'): bool, replaces fsync() + private static $flushFn=null; // tests only: callable($handle, $path): bool, replaces fflush() of appends + private static $renameFn=null; // tests only: callable($from, $to): bool, replaces rename() (and enables the retry) + private static $syncStats=['file' => 0, 'dir' => 0, 'failed' => 0]; + private static $lastSyncError=null; + private static $lastOperationDurability=null; + private static $dirSyncWorks=[]; // dbDir => can its directory be synced + private static $noWrites=0; // > 0 while checkIntegrity() runs: no recovery, no repair, no file written + private static $multiLock=0; // > 0 inside withSnapshot(): no exclusive lock may be waited for + + private static $faultHook=null; // tests only: callable($point, array $context), may throw or exit + private $hashFilesLoaded=[]; + + // Field indexing configuration + private $fieldIndexEnabled = true; // Enable field-based indexing + // Static caches have a soft size limit (v3.3): estimated bytes, checked when the last lock is released + private static $cacheMemoryLimit = null; // bytes, 0 = no limit, null = automatic (cacheLimitBytes); the last instance's setting counts + private static $cacheTouch = []; // cache key -> tick of its last use + private static $cacheTick = 0; + private static $cacheEvictions = 0; // entries dropped by evictCaches() (tests) + private static $cacheGrew = false; // something was loaded into a cache since evictCaches() last looked + private $fieldIndexCache = []; // field index path -> index + + // Spatial indexing configuration + private $spatialIndexEnabled = true; // Enable spatial R-tree indexing + private $spatialIndexCache = []; // spatial index path -> index + private $globalSpatialIndexCache = []; // global spatial index path -> index + private $rtreeNodeSize = 32; // Max entries per R-tree node + private $appendMismatch = false; // appendToFile(): the file didn't have the expected size + private $appendSyncFailed = false; // appendToFile(): fsync of the appended bytes failed (they were undone) + private static $exactFloats = false; // serialize_precision is -1 for the batch being encoded (withExactFloats) + private $dataSizes = []; // data path -> size, until another process writes (global index skips) + private $dataEpochs = []; // data path -> segment epoch read from its checkpoint (global index entries) + private $convertedSegments = []; // data path -> no v3.2 index file left for the segment + private $shardFilesUpTo = []; // database file prefix -> highest shard number with files (setOrphanShardsAside) + private $globalSkip = []; // lock path -> global field index path -> shards a lookup checks (globalCheckShards) + private $segWrites = []; // lock path -> ['seq' => n, 'paths' => [data path => seq]]: data files this process wrote + private static $staticDataSizes = []; + private static $staticGlobalSkip = []; + private static $staticSegWrites = []; + private static $staticDataEpochs = []; + private static $staticConvertedSegments = []; + private static $staticShardFilesUpTo = []; + private static $secondaryChecked = []; // data path -> "epoch:bytes" a secondary index was checked against + private static $tailCache = []; // data path -> [from, to, rows] of the last parsed data tail + private static $indexTokens = []; // field -> its index file name part (indexToken) + private $writableIndexes = []; // jidx path -> lock session id in which it was checked against its data file + private $indexedCandidates = []; // key => true: the last spatialCandidates() found it in an R-tree (valid geometry) + private static $staticWritableIndexes = []; + private $distanceCache = []; // unused since v3.3.0 (kept for code that inspects it) + + private $keyFingerprint = null; // HMAC of the secret, used to bind caches to the key + private $dbBases = []; // dbname -> file name prefix (depends on dbDir and key) + private $unsavedHashes = []; // dbname -> hash not yet in the persistent hash cache + private $dbBasesDir = null; // dbDir the prefixes were built for + + private static $devModeNoticeShown = false; + + /** + * Constructor - load config and initialize static caches + * @param array|null $config Optional config array; merged over the .nonedb file when one exists + * @throws \RuntimeException If no configuration can be found outside dev mode + * @throws \InvalidArgumentException If secretKey is missing, empty, not a string or a placeholder + */ + public function __construct($config = null){ + $this->loadConfig($config); + $this->validateSecretKey(); + if(((int)ini_get('mbstring.func_overload')) & 2){ + // Byte offsets are computed with strlen(); an overloaded strlen() would corrupt them + throw new \RuntimeException('noneDB: mbstring.func_overload must be disabled'); + } + + // Link instance caches to static caches for cross-instance sharing + if(self::$staticCacheEnabled){ + $this->indexCache = &self::$staticIndexCache; + $this->verifiedIndexes = &self::$staticVerifiedIndexes; + $this->shardedCache = &self::$staticShardedCache; + $this->metaCache = &self::$staticMetaCache; + $this->metaCacheTime = &self::$staticMetaCacheTime; + $this->hashCache = &self::$staticHashCache; + $this->jsonlFormatCache = &self::$staticFormatCache; + $this->fieldIndexCache = &self::$staticFieldIndexCache; + $this->spatialIndexCache = &self::$staticSpatialIndexCache; + $this->globalSpatialIndexCache = &self::$staticGlobalSpatialIndexCache; + $this->generations = &self::$staticGenerations; + $this->hashFilesLoaded = &self::$staticHashFilesLoaded; + $this->writableIndexes = &self::$staticWritableIndexes; + $this->dataSizes = &self::$staticDataSizes; + $this->globalSkip = &self::$staticGlobalSkip; + $this->segWrites = &self::$staticSegWrites; + $this->dataEpochs = &self::$staticDataEpochs; + $this->convertedSegments = &self::$staticConvertedSegments; + $this->shardFilesUpTo = &self::$staticShardFilesUpTo; + } + + if(!self::$shutdownCleanupRegistered){ + self::$shutdownCleanupRegistered = true; + register_shutdown_function(static function(){ + self::releaseInterruptedSessions(); + }); + } + } + + /** + * Shutdown: a fatal error (memory_limit, max_execution_time) or exit() inside an operation skips + * its finally blocks, so its locks would stay registered and later shutdown code would continue the + * half-done session. Release them and drop every cache instead: the next operation takes the lock + * again and recovers from the files like after a crash. + */ + private static function releaseInterruptedSessions(){ + if(empty(self::$lockRegistry)){ + return; + } + foreach(self::$lockRegistry as $entry){ + @flock($entry['fp'], LOCK_UN); + @fclose($entry['fp']); + } + self::$lockRegistry = []; + self::clearStaticCache(); + } + + /** + * Never expose the secret in debug output + * @return array + */ + public function __debugInfo(){ + return [ + 'dbDir' => $this->dbDir, + 'secretKey' => '***', + 'autoCreateDB' => $this->autoCreateDB, + 'shardingEnabled' => $this->shardingEnabled, + 'shardSize' => $this->shardSize, + 'bufferEnabled' => $this->bufferEnabled, + 'devMode' => $this->devMode, + ]; + } + + /** + * Instances hold a secret and open resources; they can't be serialized + * @throws \LogicException + */ + public function __serialize(): array { + throw new \LogicException('noneDB instances cannot be serialized'); + } + + public function __unserialize(array $data): void { + throw new \LogicException('noneDB instances cannot be unserialized'); + } + + public function __clone(){ + $source = self::$secretKeys[$this->cloneSourceId] ?? null; + $this->cloneSourceId = spl_object_id($this); + if($source !== null){ + self::$secretKeys[$this->cloneSourceId] = $source; + } + } + + private $cloneSourceId = null; + + public function __destruct(){ + // Anything still dirty belongs to a lock we no longer hold; it was flushed on release. + unset(self::$secretKeys[spl_object_id($this)]); + } + + private function getSecretKey(){ + return self::$secretKeys[spl_object_id($this)] ?? null; + } + + private function setSecretKey($key){ + self::$secretKeys[spl_object_id($this)] = $key; + $this->cloneSourceId = spl_object_id($this); + $this->keyFingerprint = null; + $this->dbBases = []; + } + + /** + * Load configuration. + * + * The .nonedb file is searched in: the entry script's directory, noneDB.php's directory, + * the Composer project root (when installed under vendor/) and the current working directory. + * A config array passed to the constructor is merged over the file (file is optional when the + * array already contains secretKey and dbDir). + * + * @param array|null $config Optional config array + * @throws \RuntimeException If config is missing and not in dev mode + */ + private function loadConfig($config = null){ + $arrayConfig = is_array($config) ? $config : []; + // A secretKey given in the array is used (and validated) as given, also null/false/'': + // never replaced by the file's key or the dev mode key + $keyGiven = array_key_exists('secretKey', $arrayConfig); + $arrayIsComplete = $keyGiven && isset($arrayConfig['dbDir']); + + // Load (and cache) the file config once per process + if(!self::$configLoaded){ + self::$configData = null; + self::$configFilePath = null; + self::$configFileDir = null; + + $configPath = self::findConfigFile(); + if($configPath !== null){ + $content = @file_get_contents($configPath); + if($content === false){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Cannot read config file: {$configPath}"); + } + } else { + $data = json_decode($content, true); + if(!is_array($data)){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Invalid JSON in config file: {$configPath}"); + } + } else { + self::$configData = $data; + self::$configFilePath = $configPath; + self::$configFileDir = dirname($configPath); + } + } + } + self::$configLoaded = true; + } + + if(self::$configData !== null){ + $fileConfig = self::$configData; + if(isset($arrayConfig['dbDir'])){ + unset($fileConfig['dbDir']); // overridden below; no need to resolve the file's one + } + $this->applyConfig($fileConfig, self::$configFileDir, true); + } + if(!empty($arrayConfig)){ + $this->applyConfig($arrayConfig, null, false); + } + if(self::$configData !== null || $arrayIsComplete || ($keyGiven && $this->dbDir !== null)){ + return; + } + + if(self::isDevModeEnabled()){ + $this->devMode = true; + $projectRoot = self::projectRoot(); + if($this->dbDir === null){ + $this->dbDir = $projectRoot . '/db/'; + } + if(!$keyGiven){ + // Deterministic for backward compatibility with databases created in dev mode before v3.2.0. + $this->setSecretKey('nonedb_dev_mode_key_' . md5(__DIR__)); + } + if(PHP_SAPI !== 'cli' && !self::$devModeNoticeShown){ + self::$devModeNoticeShown = true; + trigger_error('noneDB: development mode is active (predictable secretKey). Never use it in production.', E_USER_NOTICE); + } + return; + } + + throw new \RuntimeException( + "noneDB: Configuration file not found!\n" . + "Create a '.nonedb' config file in your project root.\n" . + "See '.nonedb.example' for reference.\n" . + "For development, set NONEDB_DEV_MODE=true environment variable or define('NONEDB_DEV_MODE', true);" + ); + } + + /** + * Candidate config file locations, in priority order + * @return array + */ + private static function configSearchPaths(): array { + $dirs = [self::scriptDir(), __DIR__]; + $root = self::projectRoot(); + if($root !== __DIR__){ + $dirs[] = $root; + } + $cwd = getcwd(); + if($cwd !== false){ + $dirs[] = $cwd; + } + $paths = []; + foreach(array_unique($dirs) as $dir){ + $paths[] = rtrim($dir, '/\\') . '/' . self::$configFile; + } + return $paths; + } + + private static function findConfigFile(): ?string { + foreach(self::configSearchPaths() as $path){ + if(is_file($path)){ + return $path; + } + } + return null; + } + + /** + * Absolute directory of the entry script (falls back to the working directory) + */ + private static function scriptDir(): string { + $script = $_SERVER['SCRIPT_FILENAME'] ?? ''; + $cwd = getcwd(); + // A relative SCRIPT_FILENAME depends on the working directory: both are part of the memo key + $memoKey = $script . "\0" . $cwd; + if(isset(self::$scriptDirs[$memoKey])){ + return self::$scriptDirs[$memoKey]; + } + $dir = null; + if($script !== ''){ + $real = realpath($script); + if($real !== false){ + $dir = dirname($real); + } else { + $realDir = realpath(dirname($script)); + if($realDir !== false){ + $dir = $realDir; + } + } + } + if($dir === null){ + $dir = $cwd !== false ? $cwd : __DIR__; + } + if(count(self::$scriptDirs) > 16){ + self::$scriptDirs = []; + } + return self::$scriptDirs[$memoKey] = $dir; + } + + /** + * Project root: the directory that contains vendor/ when installed with Composer, else noneDB.php's directory + */ + private static function projectRoot(): string { + $dir = str_replace('\\', '/', __DIR__); + $pos = strpos($dir, '/vendor/'); + if($pos !== false){ + return substr(__DIR__, 0, $pos); + } + return __DIR__; + } + + private static function isDevModeEnabled(): bool { + return getenv('NONEDB_DEV_MODE') === 'true' + || getenv('NONEDB_DEV_MODE') === '1' + || (defined('NONEDB_DEV_MODE') && NONEDB_DEV_MODE === true); + } + + private static function isAbsolutePath(string $path): bool { + return $path !== '' && ($path[0] === '/' || $path[0] === '\\' || preg_match('/^[A-Za-z]:[\\\\\/]/', $path) === 1); + } + + /** + * Apply configuration values to instance properties + * @param array $config Configuration array + * @param string|null $baseDir Directory that relative dbDir values resolve against (null: the entry script's) + * @param bool $fromFile True when the values come from the .nonedb file + */ + private function applyConfig(array $config, ?string $baseDir, bool $fromFile){ + if(array_key_exists('secretKey', $config)){ + $this->setSecretKey($config['secretKey']); + } + if(isset($config['dbDir'])){ + $this->dbDir = $this->resolveDbDir((string)$config['dbDir'], $baseDir, $fromFile); + } + if(isset($config['autoCreateDB'])){ + $this->autoCreateDB = self::configBool('autoCreateDB', $config['autoCreateDB']); + } + + // Sharding settings + if(isset($config['shardingEnabled'])){ + $this->shardingEnabled = self::configBool('shardingEnabled', $config['shardingEnabled']); + } + if(isset($config['shardSize'])){ + $shardSize = self::configNumber($config['shardSize'], true) ?? 0; + if($shardSize < 1){ + throw new \InvalidArgumentException("noneDB: 'shardSize' must be a positive integer"); + } + $this->shardSize = $shardSize; + } + if(isset($config['autoMigrate'])){ + $this->autoMigrate = self::configBool('autoMigrate', $config['autoMigrate']); + } + + // Compaction settings + if(isset($config['autoCompactThreshold'])){ + $threshold = self::configNumber($config['autoCompactThreshold'], false) ?? 0.0; + if($threshold <= 0){ + throw new \InvalidArgumentException("noneDB: 'autoCompactThreshold' must be greater than 0"); + } + $this->jsonlGarbageThreshold = $threshold; + } + + // Lock settings + if(isset($config['lockTimeout'])){ + // Seconds, fractions allowed (0.5 used to become 0 = wait forever) + $timeout = $config['lockTimeout']; + if(!is_int($timeout) && !is_float($timeout) && !(is_string($timeout) && is_numeric($timeout))){ + $timeout = null; + } + $timeout = $timeout === null ? NAN : (float)$timeout; + if(!is_finite($timeout) || $timeout < 0){ + throw new \InvalidArgumentException("noneDB: 'lockTimeout' must be 0 (wait forever) or a positive number of seconds"); + } + $this->lockTimeout = $timeout; + } + if(isset($config['lockRetryDelay'])){ + $delay = self::configNumber($config['lockRetryDelay'], true) ?? 0; + if($delay < 1){ + throw new \InvalidArgumentException("noneDB: 'lockRetryDelay' must be a positive number of microseconds"); + } + $this->lockRetryDelay = $delay; + } + + // Write buffer + if(isset($config['bufferEnabled'])){ + $this->bufferEnabled = self::configBool('bufferEnabled', $config['bufferEnabled']); + } + + // Use indexes for queries (indexes are always kept up to date) + if(isset($config['fieldIndexEnabled'])){ + $this->fieldIndexEnabled = self::configBool('fieldIndexEnabled', $config['fieldIndexEnabled']); + } + if(isset($config['spatialIndexEnabled'])){ + $this->spatialIndexEnabled = self::configBool('spatialIndexEnabled', $config['spatialIndexEnabled']); + } + if(isset($config['fsync'])){ + $this->fsync = self::configBool('fsync', $config['fsync']); + } + if(isset($config['cacheMemoryLimit'])){ + $limit = $config['cacheMemoryLimit']; + if($limit === 'auto'){ + self::$cacheMemoryLimit = null; + } elseif(!(is_int($limit) && $limit >= 0) && !(is_string($limit) && ctype_digit($limit))){ + throw new \InvalidArgumentException("noneDB: 'cacheMemoryLimit' must be a non-negative integer (bytes) or 'auto'"); + } else { + self::$cacheMemoryLimit = (int)$limit; + } + } + } + + /** + * A numeric setting: an int, a finite float or a numeric string (an integer when $integer); anything + * else (booleans, arrays, "10abc", NAN) is null, and the caller rejects it + * @return int|float|null + */ + private static function configNumber($value, bool $integer){ + if(is_string($value) && is_numeric(trim($value))){ + $value = trim($value) + 0; + } + if(is_int($value)){ + return $integer ? $value : (float)$value; + } + if(is_float($value) && is_finite($value) && (!$integer || floor($value) === $value)){ + return $integer ? (int)$value : $value; + } + return null; + } + + /** + * A boolean setting: true/false, 1/0, "1"/"0", "true"/"false" (any case). Anything else is an error + * (before v3.3 any non-empty string, "false" included, meant true). + * @throws \InvalidArgumentException + */ + private static function configBool(string $name, $value): bool { + if(is_bool($value)){ + return $value; + } + if($value === 1 || $value === 0){ + return $value === 1; + } + if(is_string($value)){ + $lower = strtolower(trim($value)); + if($lower === '1' || $lower === 'true'){ + return true; + } + if($lower === '0' || $lower === 'false'){ + return false; + } + } + throw new \InvalidArgumentException("noneDB: '{$name}' must be true or false"); + } + + /** + * Resolve a configured dbDir to an absolute path ending with a slash. + * Relative paths (with or without a leading ./) are relative to the config file's directory + * (file config) or to the entry script's directory (array config). + */ + private function resolveDbDir(string $dbDir, ?string $baseDir, bool $fromFile): string { + if(!self::isAbsolutePath($dbDir)){ + $dotted = substr($dbDir, 0, 2) === './' || substr($dbDir, 0, 3) === '../' + || $dbDir === '.' || $dbDir === '..'; + $relative = preg_replace('#^(\./)+#', '', $dbDir); + $baseDir = $baseDir ?? self::scriptDir(); + $resolved = rtrim($baseDir, '/\\') . '/' . $relative; + + // Before v3.2.0 "./x" in the file was resolved against the entry script and a plain "x" + // against the working directory. Keep using such a directory (with a deprecation notice) + // until the data is moved. + $legacyBase = is_dir($resolved) ? null : ($dotted ? ($fromFile ? self::scriptDir() : null) : getcwd()); + if(is_string($legacyBase)){ + $legacy = rtrim($legacyBase, '/\\') . '/' . $relative; + if(is_dir($legacy) && realpath($legacy) !== realpath($resolved)){ + trigger_error( + "noneDB: dbDir '{$dbDir}' is now resolved relative to " . ($fromFile ? 'the .nonedb file' : 'the entry script') + . " ({$baseDir}); using the old location {$legacy} for now. Move the directory or use an absolute path.", + E_USER_DEPRECATED + ); + $resolved = $legacy; + } + } + $dbDir = $resolved; + } + + if(substr($dbDir, -1) !== '/' && substr($dbDir, -1) !== '\\'){ + $dbDir .= '/'; + } + return $dbDir; + } + + /** + * Reject missing, empty, non-string and placeholder secret keys + * @throws \InvalidArgumentException + */ + private function validateSecretKey(){ + $key = $this->getSecretKey(); + if(!is_string($key) || trim($key) === ''){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' must be a non-empty string. Set it in your .nonedb file or constructor config " + . "(generate one with: php -r \"echo bin2hex(random_bytes(32));\")." + ); + } + if(in_array($key, self::$placeholderKeys, true)){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' is still the example placeholder. Replace it with a random secret " + . "(php -r \"echo bin2hex(random_bytes(32));\"). If you already have data, see noneDB::migrateKey()." + ); + } + if($this->dbDir === null){ + throw new \RuntimeException("noneDB: 'dbDir' is not configured"); + } + } + + /** + * Check if config file exists + * @return bool + */ + public static function configExists(): bool { + return self::findConfigFile() !== null; + } + + /** + * Get the config file template path + * @return string|null + */ + public static function getConfigTemplate(): ?string { + $templatePath = __DIR__ . '/.nonedb.example'; + return file_exists($templatePath) ? $templatePath : null; + } + + /** + * Clear config cache (useful for testing) + * @return void + */ + public static function clearConfigCache(): void { + self::$configLoaded = false; + self::$configData = null; + self::$configFilePath = null; + self::$configFileDir = null; + } + + /** + * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) + * @return bool + */ + public static function isDevMode(): bool { + return self::isDevModeEnabled(); + } + + /** + * Set development mode programmatically + * Useful for testing or when env vars are not available + * @param bool $enabled + * @return void + */ + public static function setDevMode(bool $enabled): void { + if($enabled && !defined('NONEDB_DEV_MODE')){ + define('NONEDB_DEV_MODE', true); + } + } + + /** + * Re-key existing databases: rename every file created with $oldKey to the name $newKey produces. + * + * File names are {hash}-{dbname}... where hash depends on the secretKey, so changing the key + * without this makes existing data invisible. Run it once, with no other process using the + * databases, then update secretKey in your config. Each database is locked while it is renamed. + * The old per-key hash cache and the pre-3.2 `.nonedb_hash_cache` file are removed. + * + * @param string $dbDir Database directory + * @param string $oldKey Key the databases were created with (the old placeholder is accepted here) + * @param string $newKey New secret key + * @return array ['success' => bool, 'databases' => string[], 'files' => int, 'error' => ?string] + */ + public static function migrateKey(string $dbDir, string $oldKey, string $newKey): array { + $result = ['success' => false, 'databases' => [], 'files' => 0, 'error' => null]; + if(trim($oldKey) === '' || trim($newKey) === ''){ + $result['error'] = 'Both keys must be non-empty strings'; + return $result; + } + if(in_array($newKey, self::$placeholderKeys, true)){ + $result['error'] = 'The new key is a placeholder; generate a random one'; + return $result; + } + if($oldKey === $newKey){ + $result['error'] = 'The new key is the same as the old key'; + return $result; + } + if(!empty(self::$lockRegistry)){ + // Inside a database operation of this process: its own lock would block the renames forever + $result['error'] = 'migrateKey() can\'t run while this process holds a database lock (inside a database operation)'; + return $result; + } + $dbDir = rtrim($dbDir, '/\\') . '/'; + if(!is_dir($dbDir)){ + $result['error'] = "Directory not found: {$dbDir}"; + return $result; + } + + // Databases created with the old key: any file of theirs counts, so a run that was interrupted + // (some files already renamed) finds them again and finishes + $databases = []; + $files = scandir($dbDir) ?: []; + foreach($files as $file){ + if(preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+?)(?:_s\\d+)?\\.nonedb/", $file, $m) + && !isset($databases[$m[2]]) && hash_pbkdf2('sha256', $m[2], $oldKey, 1000, 20) === $m[1]){ + $databases[$m[2]] = $m[1]; + } + } + + // Plan every rename first, refuse on any conflict + $plan = []; + foreach($databases as $dbname => $oldHash){ + $newHash = hash_pbkdf2('sha256', $dbname, $newKey, 1000, 20); + foreach($files as $file){ + // {hash}-{name}.*, {hash}-{name}_sN.*, and spatial files {hash}.nonedb.* / {hash}_sN.nonedb.* + $named = strpos($file, $oldHash . '-' . $dbname . '.') === 0 || strpos($file, $oldHash . '-' . $dbname . '_s') === 0; + $spatial = strpos($file, $oldHash . '.nonedb.') === 0 || preg_match('/^' . $oldHash . '_s\d+\.nonedb\./', $file) === 1; + if(!$named && !$spatial){ + continue; + } + $target = $newHash . substr($file, strlen($oldHash)); + if(file_exists($dbDir . $target)){ + $result['error'] = "Target file already exists: {$target}"; + return $result; + } + $plan[$dbname][] = [$file, $target]; + } + } + + foreach($plan as $dbname => $renames){ + $oldHash = $databases[$dbname]; + $lockPath = $dbDir . $oldHash . '-' . $dbname . '.nonedb.lock'; + // Databases from before v3.2 have no lock file: the one opened here is removed afterwards + clearstatcache(true, $lockPath); + $hadLock = file_exists($lockPath); + $fp = @fopen($lockPath, 'c+b'); + if($fp === false || !flock($fp, LOCK_EX)){ + if($fp !== false){ + fclose($fp); + } + if(!$hadLock){ + @unlink($lockPath); + } + $result['error'] = "Could not lock database '{$dbname}'"; + return $result; + } + try { + // The files that make the database exist (data, meta, info) go last but one, so an + // interrupted run still finds the database under the old key and can be repeated; the lock + // file last, so writers keep blocking on it until we are done + $rank = function($file) use ($oldHash, $dbname){ + if(substr($file, -12) === '.nonedb.lock'){ + return 2; + } + $base = $oldHash . '-' . $dbname; + return in_array($file, [$base . '.nonedb', $base . '.nonedb.meta', $base . '.nonedbinfo'], true) ? 1 : 0; + }; + usort($renames, function($a, $b) use ($rank){ + return $rank($a[0]) <=> $rank($b[0]) ?: strcmp($a[0], $b[0]); + }); + foreach($renames as $pair){ + if(!@rename($dbDir . $pair[0], $dbDir . $pair[1])){ + $result['error'] = "Failed to rename {$pair[0]}"; + return $result; + } + $result['files']++; + self::faultPoint('migrateKey:renamed', ['file' => $pair[0]]); + } + } finally { + flock($fp, LOCK_UN); + fclose($fp); + if(!$hadLock){ + @unlink($lockPath); + } + } + $result['databases'][] = $dbname; + } + + // Old caches can't be used with the new key (and the legacy one leaks names) + $oldFingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', $oldKey), 0, 16); + @unlink($dbDir . '.nonedb_hash_cache_' . $oldFingerprint); + @unlink($dbDir . '.nonedb_hash_cache'); + self::clearStaticCache(); + + $result['success'] = true; + return $result; + } + + /** + * HMAC fingerprint of the secret; binds caches to the key without revealing it + */ + private function getKeyFingerprint(){ + if($this->keyFingerprint === null){ + $this->keyFingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', (string)$this->getSecretKey()), 0, 16); + } + return $this->keyFingerprint; + } + + /** + * Path of the persistent hash cache for this dbDir + secretKey. + * The file name is derived from the key, so different keys never share (or poison) a cache, + * and the file can't be guessed without the key. + */ + private function getHashCacheFile(){ + return $this->dbDir . '.nonedb_hash_cache_' . $this->getKeyFingerprint(); + } + + /** + * Load the persistent hash cache for this dbDir + key (once per process) + * @return void + */ + private function loadPersistentHashCache(){ + $file = $this->getHashCacheFile(); + if(isset($this->hashFilesLoaded[$file])){ + return; + } + $this->hashFilesLoaded[$file] = true; + + $fp = is_file($file) ? @fopen($file, 'rb') : false; // not a directory in its place + if($fp === false){ + return; + } + $content = ''; + if(flock($fp, LOCK_SH)){ + $content = @stream_get_contents($fp); + flock($fp, LOCK_UN); + } + fclose($fp); + + $loaded = json_decode((string)$content, true); + if(!is_array($loaded)){ + return; + } + $prefix = $this->getKeyFingerprint() . '|'; + foreach($loaded as $dbname => $hash){ + if(is_string($dbname) && is_string($hash) && preg_match('/^[0-9a-f]{20}$/', $hash)){ + $this->hashCache[$prefix . $dbname] = $hash; + } + } + } + + /** + * Add one name -> hash entry to the persistent cache (locked read-merge-write). + * Entries whose database no longer exists are dropped. + * @return void + */ + private function savePersistentHashCacheEntry($dbname, $hash){ + try { + $this->ensureDbDir(); + } catch(\RuntimeException $e){ + return; // the cache is optional + } + $file = $this->getHashCacheFile(); + $isNew = !file_exists($file); + $fp = @fopen($file, 'c+b'); + if($fp === false){ + return; + } + if($isNew){ + @chmod($file, 0600); + } + if(!flock($fp, LOCK_EX)){ + fclose($fp); + return; + } + try { + $entries = json_decode((string)stream_get_contents($fp), true); + if(!is_array($entries)){ + $entries = []; + } + $entries[$dbname] = $hash; + foreach($entries as $name => $h){ + if(!is_string($h)){ + unset($entries[$name]); + continue; + } + $base = $this->dbDir . $h . '-' . $name; + if($name !== $dbname && !file_exists($base . '.nonedb') && !file_exists($base . '.nonedb.meta') && !file_exists($base . '.nonedbinfo')){ + unset($entries[$name]); + } + } + $json = json_encode($entries, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if($json !== false){ + ftruncate($fp, 0); + rewind($fp); + fwrite($fp, $json); + fflush($fp); + } + } finally { + flock($fp, LOCK_UN); + fclose($fp); + } + } + + /** + * Clear all static caches (useful for testing or memory management) + * @return void + */ + public static function clearStaticCache(){ + self::$staticIndexCache = []; + self::$staticVerifiedIndexes = []; + self::$uncheckpointed = []; + self::$staticShardedCache = []; + self::$staticMetaCache = []; + self::$staticMetaCacheTime = []; + self::$staticHashCache = []; + self::$staticFormatCache = []; + self::$staticFileExistsCache = []; + self::$staticSanitizeCache = []; + self::$staticFieldIndexCache = []; + self::$staticSpatialIndexCache = []; + self::$staticGlobalSpatialIndexCache = []; + self::$staticGenerations = []; + self::$staticHashFilesLoaded = []; + self::$bufferCounts = []; + self::$bufferEnds = []; + self::$staticWritableIndexes = []; + self::$staticDataSizes = []; + self::$staticGlobalSkip = []; + self::$staticSegWrites = []; + self::$staticDataEpochs = []; + self::$staticConvertedSegments = []; + self::$staticShardFilesUpTo = []; + self::$secondaryChecked = []; + self::$cacheTouch = []; + self::$tailCache = []; + foreach(self::$lockHandles as $handle){ + @fclose($handle['fp']); + } + self::$lockHandles = []; + } + + /** + * Disable static caching (each instance uses its own cache) + * @return void + */ + public static function disableStaticCache(){ + self::$staticCacheEnabled = false; + } + + /** + * Enable static caching (default) + * @return void + */ + public static function enableStaticCache(){ + self::$staticCacheEnabled = true; + } + + /** + * Cached file_exists check. Entries are dropped whenever another process writes the database + * (see withLock), so the cache can't go stale across processes. + * + * @param string $path File path to check + * @return bool True if file exists + */ + private function cachedFileExists($path){ + if(!self::$staticCacheEnabled){ + clearstatcache(true, $path); + return file_exists($path); + } + if(isset(self::$staticFileExistsCache[$path])){ + return self::$staticFileExistsCache[$path]; + } + clearstatcache(true, $path); + $exists = file_exists($path); + self::$staticFileExistsCache[$path] = $exists; + return $exists; + } + + /** + * Mark file as existing in cache (call after creating file) + * @param string $path File path + */ + private function markFileExists($path){ + if(self::$staticCacheEnabled){ + self::$staticFileExistsCache[$path] = true; + } + } + + /** + * Mark file as not existing in cache (call after deleting file) + * @param string $path File path + */ + private function markFileNotExists($path){ + if(self::$staticCacheEnabled){ + self::$staticFileExistsCache[$path] = false; + } + } + + /** + * Validate a database name. + * Allowed characters: A-Z a-z 0-9, space, apostrophe and hyphen. Anything else is rejected + * (before v3.2.0 such characters were silently stripped, which let different names collide). + * + * @param mixed $dbname Database name + * @return string The (unchanged) database name + * @throws \InvalidArgumentException + */ + private function normalizeDbName($dbname){ + if(is_int($dbname)){ + $dbname = (string)$dbname; + } + if(!is_string($dbname)){ + throw new \InvalidArgumentException('noneDB: database name must be a string'); + } + if(isset(self::$staticSanitizeCache[$dbname])){ + return $dbname; + } + if($dbname === '' || preg_match("/^[A-Za-z0-9' -]+$/", $dbname) !== 1){ + throw new \InvalidArgumentException( + "noneDB: invalid database name '{$dbname}'. Only letters A-Z/a-z, digits, space, apostrophe (') and hyphen (-) are allowed." + ); + } + if(self::$staticCacheEnabled){ + self::$staticSanitizeCache[$dbname] = true; + } + return $dbname; + } + + /** + * Hash a database name for its file names (PBKDF2-SHA256 keyed with secretKey). + * Cached per key fingerprint in memory and in a per-key file, so the secret is always honoured. + */ + private function hashDBName($dbname){ + $cacheKey = $this->getKeyFingerprint() . '|' . $dbname; + if(isset($this->hashCache[$cacheKey])){ + return $this->hashCache[$cacheKey]; + } + + $this->loadPersistentHashCache(); + if(isset($this->hashCache[$cacheKey])){ + return $this->hashCache[$cacheKey]; + } + + $hash = hash_pbkdf2("sha256", (string)$dbname, (string)$this->getSecretKey(), 1000, 20); + $this->hashCache[$cacheKey] = $hash; + // Stored by the next write (noteWrite): reads never create files or directories (A-11) + $this->unsavedHashes[(string)$dbname] = $hash; + + return $hash; + } + + /** + * Common file-name prefix of every file belonging to a database: {dbDir}{hash}-{dbname} + */ + private function dbBase($dbname){ + if($this->dbBasesDir !== $this->dbDir){ + $this->dbBases = []; + $this->dbBasesDir = $this->dbDir; + } + return $this->dbBases[$dbname] ?? ($this->dbBases[$dbname] = $this->dbDir . $this->hashDBName($dbname) . '-' . $dbname); + } + + /** + * Path of the main (non-sharded) data file + */ + private function getDataPath($dbname){ + return $this->dbBase($dbname) . '.nonedb'; + } + + /** + * Create the database directory (and protect it from direct web access) if needed + */ + private function ensureDbDir(){ + if(is_dir($this->dbDir)){ + return; + } + if(!@mkdir($this->dbDir, 0755, true) && !is_dir($this->dbDir)){ + throw new \RuntimeException("noneDB: cannot create database directory {$this->dbDir}"); + } + // Deny direct HTTP access on Apache; harmless elsewhere + @file_put_contents($this->dbDir . '.htaccess', "Require all denied\nDeny from all\n"); + @file_put_contents($this->dbDir . 'index.html', ''); + } + + + /** + * The cache limit in bytes, 0 = none: the cacheMemoryLimit setting, or by default half of PHP's + * memory_limit (no limit without one, as before v3.3): the cache doesn't make a process run out of + * memory, and doesn't slow down one that may use all the memory it needs + */ + private static function cacheLimitBytes(){ + if(self::$cacheMemoryLimit !== null){ + return self::$cacheMemoryLimit; + } + $ini = strtolower(trim((string)ini_get('memory_limit'))); + $bytes = (int)$ini; + if($bytes <= 0){ + return 0; + } + $unit = substr($ini, -1); + $bytes *= $unit === 'g' ? 1073741824 : ($unit === 'm' ? 1048576 : ($unit === 'k' ? 1024 : 1)); + return intdiv($bytes, 2); + } + + /** + * Keep the static caches under cacheMemoryLimit (a soft limit: estimated bytes, not memory_get_usage() + * of the caches; v3.3). Runs when the last lock of the process is released; while the process uses less + * memory than the limit it does nothing. Least recently used entries go first, in batches of a tenth. + * Every evicted entry is loaded again when needed. + */ + private function evictCaches(){ + $limit = self::cacheLimitBytes(); + if(memory_get_usage() <= $limit && $limit >= 1048576){ + return; + } + $entries = []; + $total = 0; + $count = function($a){ return is_array($a) ? count($a) : 0; }; + foreach(self::$staticIndexCache as $key => $index){ + $bytes = 96 + 80 * $count($index['p'] ?? null); + $entries[] = [self::$cacheTouch[$key] ?? 0, 'index', $key, $bytes]; + $total += $bytes; + } + foreach(self::$staticFieldIndexCache as $key => $index){ + if(isset($index['b'])){ + $bytes = 128 + 48 * count($index['b']) + 24 * array_sum(array_map(function($k){ return is_array($k) ? count($k) : 1; }, $index['b'])); + } elseif(isset($index['map'])){ + $bytes = 128 + 64 * count($index['map']) + 64 * count($index['sh'] ?? []); + } elseif(isset($index['values'])){ + $bytes = 128 + 72 * count($index['values']); + } else { + $bytes = 256; + } + $entries[] = [self::$cacheTouch[$key] ?? 0, 'field', $key, $bytes]; + $total += $bytes; + } + foreach(self::$staticSpatialIndexCache as $key => $tree){ + $bytes = 128 + 200 * (int)($tree['cnt'] ?? $count($tree['leafOf'] ?? null)); + $entries[] = [self::$cacheTouch[$key] ?? 0, 'spatial', $key, $bytes]; + $total += $bytes; + } + foreach(self::$staticGlobalSpatialIndexCache as $key => $global){ + $bytes = 128 + 160 * $count($global['sh'] ?? null); + $entries[] = [self::$cacheTouch[$key] ?? 0, 'gspatial', $key, $bytes]; + $total += $bytes; + } + if($total <= $limit){ + return; + } + usort($entries, function($a, $b){ return $a[0] <=> $b[0]; }); + $batch = max(1, intdiv(count($entries), 10)); + $dropped = 0; + foreach($entries as [$tick, $cache, $key, $bytes]){ + if($total <= $limit && $dropped % $batch === 0){ + break; + } + if($cache === 'index'){ + unset(self::$staticIndexCache[$key], self::$staticVerifiedIndexes[$key], self::$staticWritableIndexes[$key]); + } elseif($cache === 'field'){ + unset(self::$staticFieldIndexCache[$key]); + } elseif($cache === 'spatial'){ + unset(self::$staticSpatialIndexCache[$key]); + } else { + unset(self::$staticGlobalSpatialIndexCache[$key]); + } + unset(self::$cacheTouch[$key]); + $total -= $bytes; + $dropped++; + self::$cacheEvictions++; + } + } + + // ========================================== + // DATABASE LOCKING AND STATE (v3.2.0) + // ========================================== + // + // Every database has a lock file {base}.nonedb.lock. Besides being the flock target it holds a small + // state record. It is read on every lock acquisition anyway, so the checks below cost no extra I/O: + // + // "{generation}:{epoch}[ v][ b][ j=main,0,3 | J]" space padded to LOCK_STATE_SIZE bytes + // + // generation bumped by the first file write of a write session (v3.3: sessions that write nothing + // keep it). A value other than the one this process last saw means another process + // changed the database: its caches are checked again (key indexes: only the new data). + // epoch 8 hex digits (v3.3), new when a data file is replaced or removed (compaction, migration, + // drop) and for every new lock file: another epoch drops every cache of the database, so a + // recreated lock file can't validate an old cache by reaching the same generation again. + // v3.2 reads "{generation}:{epoch}" as the generation and writes no epoch (then all + // caches are dropped, and the next exclusive lock holder writes a new epoch). + // v verified: the one-time checks for pre-3.2 leftovers (old buffer files, no index list) + // are done, so the flags below are authoritative and nothing is re-checked per operation. + // b the write buffer may hold records; they are flushed before the next read or write. + // j=... / J journal: shard meta counters of these segments (J: all) may lag behind the data because a + // writer died (or a migration was interrupted); the next lock holder recounts them. + // Key, field and spatial indexes need none: they replay the data after their checkpoint. + // + // The record is overwritten in place with a fixed size (one small write: no truncate, no rename). + // Lock files are never deleted (dropDB keeps them), and their handles stay open between operations. + + private const LOCK_STATE_SIZE = 128; + private const LOCK_HANDLE_LIMIT = 64; + + /** Exception code: a write failed and undoing it failed too (outcome unknown, the next lock holder recovers) */ + public const ROLLBACK_FAILED = 3301; + /** Exception code: the database lock could not be taken (lockTimeout) or was lost */ + public const LOCK_FAILED = 3302; + /** Exception code (internal): an exclusive lock was needed while a snapshot holds several shared ones */ + private const SNAPSHOT_RETRY = 3303; + + private const EARTH_HALF_CIRCUMFERENCE = 20040000.0; // meters: no two points are farther apart + + // Rollback scope (v3.3.0). The outermost exclusive withLock() call of a database opens one scope; nested + // exclusive calls on the same database join it. There are no nested scopes: cutting a file back to an + // earlier size can't drop an earlier outer append and keep a later inner one. Every append to a data or + // buffer file records the size the file had before the scope's first append to it (a "mark"). When an + // exception leaves the outermost call, every marked file is cut back to its mark. + // Work that must not depend on the caller's success (buffer flush, shard migration) runs in place only + // while the scope has no appends yet, committing its own appends step by step (runIndependent); + // requested later, it is deferred until the scope committed, like compaction, which replaces whole files. + // A deferred job that fails doesn't fail the operation that committed before it: it is logged and runs + // again on a later operation. + + private function getLockPath($dbname){ + return $this->dbBase($dbname) . '.nonedb.lock'; + } + + /** + * Run $fn while holding the database lock: shared for reads, exclusive for writes. + * + * The lock is process-wide and re-entrant. A nested exclusive request upgrades a shared lock. + * + * @param string $dbname Database name (already normalized) + * @param bool $exclusive + * @param callable $fn + * @return mixed Whatever $fn returns + */ + private function withLock($dbname, $exclusive, callable $fn){ + $path = $this->getLockPath($dbname); + if($exclusive && self::$multiLock > 0){ + // A snapshot holds shared locks of several databases: waiting for an exclusive one here could + // deadlock with another snapshot. withSnapshot() does that work alone first and tries again. + throw new \RuntimeException("noneDB: database '{$dbname}' needs maintenance before the snapshot", self::SNAPSHOT_RETRY); + } + + if(isset(self::$lockRegistry[$path])){ + if(!empty(self::$lockRegistry[$path]['lost'])){ + throw new \RuntimeException("noneDB: the lock on database '{$dbname}' was lost (timed out while upgrading)", self::LOCK_FAILED); + } + if($exclusive && self::$lockRegistry[$path]['mode'] !== LOCK_EX){ + // flock can't upgrade atomically. On BSD/macOS a failed non-blocking upgrade keeps the shared + // lock, so two processes upgrading at once would wait for each other until lockTimeout: release + // first, then wait for the exclusive lock. Another writer may run in between, so the state is + // read again and whatever it left half-done is recovered (A-4). + $this->flushPendingGlobals($dbname, $path); // while what they describe still holds + flock(self::$lockRegistry[$path]['fp'], LOCK_UN); + $this->fault('upgrade:released', ['db' => $dbname]); + try { + $this->acquireFlock(self::$lockRegistry[$path]['fp'], LOCK_EX, $dbname); + } catch(\Throwable $e){ + self::$lockRegistry[$path]['lost'] = true; // never continue reading without a lock + throw $e; + } + self::$lockRegistry[$path]['mode'] = LOCK_EX; + $this->loadLockState($dbname, $path); + $this->prepareDb($dbname, $path); + } + self::$lockRegistry[$path]['depth']++; + try { + if($exclusive && self::$lockRegistry[$path]['scope'] === null){ + return $this->runWriteScope($dbname, $path, $fn); + } + return $fn(); + } finally { + self::$lockRegistry[$path]['depth']--; + } + } + + $mode = $exclusive ? LOCK_EX : LOCK_SH; + $fp = $this->lockFile($path, $mode, $dbname); + self::$lockRegistry[$path] = [ + 'fp' => $fp, 'mode' => $mode, 'depth' => 1, 'dbname' => $dbname, + 'gen' => 0, 'epoch' => null, 'flags' => [], 'bumped' => false, 'pending' => [], + 'scope' => null, 'rolledBack' => false, 'sid' => ++self::$sessionCounter, 'globals' => [], + ]; + + $ok = false; + try { + $this->loadLockState($dbname, $path); + $this->prepareDb($dbname, $path); + $result = $exclusive ? $this->runWriteScope($dbname, $path, $fn) : $fn(); + $ok = true; + return $result; + } finally { + try { + if(self::$lockRegistry[$path]['mode'] === LOCK_EX){ + $this->finishWriteSession($dbname, $path, $ok); + } + $this->flushPendingGlobals($dbname, $path, $ok); + } finally { + $fp = self::$lockRegistry[$path]['fp']; + unset(self::$lockRegistry[$path]); + flock($fp, LOCK_UN); + $this->keepLockHandle($path, $fp); + // The cache size is looked at only after something new was cached + if(self::$cacheGrew && empty(self::$lockRegistry)){ + self::$cacheGrew = false; + if(self::$staticCacheEnabled && self::cacheLimitBytes() > 0){ + $this->evictCaches(); + } + } + } + } + } + + /** + * End of an exclusive session: write pending spatial indexes, close the journal entries of this + * session, and make sure the generation moved so other processes drop their caches + */ + private function finishWriteSession($dbname, $path, $ok){ + if($ok){ + if(empty(self::$lockRegistry[$path]['rolledBack'])){ + $this->checkpointTouchedSegments($dbname, $path); + } + // After a rollback (of a deferred job) the journal stays: the next lock holder rebuilds + if(!empty(self::$lockRegistry[$path]['flags']['j']) && empty(self::$lockRegistry[$path]['rolledBack'])){ + // Data, indexes and meta of every journaled segment are written + unset(self::$lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + } else { + // Partial failure: forget in-memory state, the files are the source of truth. + // A journal entry stays, so the next lock holder rebuilds what may be half-written. + $this->forgetDirtySpatial($dbname); + $this->invalidateDbCaches($dbname); + } + if(!self::$lockRegistry[$path]['bumped']){ + return; // nothing was written: the generation stays + } + if(!$this->fsync){ + self::$lastOperationDurability = ['op' => "write to '{$dbname}'", 'required' => [], 'completed' => [], + 'durable' => false, 'reason' => 'fsync is off']; + } elseif(isset(self::$lockRegistry[$path]['durability'])){ + $this->finishDurability($path, $dbname); + } + } + + /** + * Run $fn with shared locks on several databases at once (joins, v3.3.0): every read sees the same + * state of all of them. Locks are taken in one canonical order (by lock file path), so two snapshots + * never wait for each other in a circle, and no exclusive lock is ever waited for while several are + * held: work that needs one (recovery, an index repair) is done for each database alone first. + */ + private function withSnapshot(array $dbnames, callable $fn){ + $names = []; + foreach($dbnames as $name){ + $name = $this->normalizeDbName($name); + $lock = $this->getLockPath($name); + $dir = realpath(dirname($lock)); + $names[($dir === false ? dirname($lock) : $dir) . '/' . basename($lock)] = $name; + } + ksort($names, SORT_STRING); + $names = array_values($names); + if(count($names) < 2){ + return $fn(); + } + $started = microtime(true); + while(true){ + foreach($names as $name){ + // Alone: recovery, buffer flush and key index repairs happen here, not under the snapshot + $this->withReadLock($name, function() use ($name){ + foreach($this->listSegments($name) as $segment){ + $this->readJsonlIndex($name, $segment); + } + }); + } + try { + return $this->withSharedLocks($names, $fn); + } catch(\RuntimeException $e){ + if($e->getCode() !== self::SNAPSHOT_RETRY){ + throw $e; + } + if($this->lockTimeout > 0 && microtime(true) - $started >= $this->lockTimeout){ + throw new \RuntimeException("noneDB: timed out waiting for a consistent snapshot of " . implode(', ', $names), self::LOCK_FAILED, $e); + } + usleep($this->lockRetryDelay); + } + } + } + + private function withSharedLocks(array $names, callable $fn, $i = 0){ + if($i >= count($names)){ + self::$multiLock++; + try { + if(self::$faultHook !== null){ $this->fault('snapshot:held', ['dbs' => $names]); } + return $fn(); + } finally { + self::$multiLock--; + } + } + return $this->withReadLock($names[$i], function() use ($names, $fn, $i){ + return $this->withSharedLocks($names, $fn, $i + 1); + }); + } + + /** + * Run the outermost exclusive call of a database inside a rollback scope (see above), then the jobs + * deferred until its commit, each in a scope of its own + */ + private function runWriteScope($dbname, $path, callable $fn){ + if(!empty(self::$pendingRecoveryEvents[$path]) && self::$noWrites === 0){ + // What readers of this process found (they don't write): logged once, deduplicated by id + $events = self::$pendingRecoveryEvents[$path]; + unset(self::$pendingRecoveryEvents[$path]); + foreach($events as $id => $event){ + $this->logRecoveryEvent($dbname, $event, $id); + } + } + self::$lockRegistry[$path]['scope'] = ['marks' => [], 'deferred' => [], 'independent' => false]; + try { + $result = $fn(); + } catch(\Throwable $e){ + $this->rollbackScope($dbname, $path, $e); // rethrows + } + $deferred = self::$lockRegistry[$path]['scope']['deferred']; + self::$lockRegistry[$path]['scope'] = null; + if(self::$faultHook !== null){ $this->fault('scope:commit', ['db' => $dbname]); } + foreach($deferred as $job => $work){ + try { + $this->runWriteScope($dbname, $path, function() use ($dbname, $job, $work){ + $this->fault('job:' . $job, ['db' => $dbname]); + return $this->runIndependent($dbname, $job, $work); + }); + } catch(\Throwable $e){ + // The operation itself is committed; the job runs again on a later operation + $this->logRecoveryEvent($dbname, ['action' => 'deferred-failed', 'job' => $job, 'error' => $e->getMessage()]); + break; + } + } + return $result; + } + + /** + * Undo the appends of a failed scope and rethrow (ROLLBACK_FAILED when a file can't be cut back) + */ + private function rollbackScope($dbname, $path, \Throwable $e){ + $marks = self::$lockRegistry[$path]['scope']['marks']; + self::$lockRegistry[$path]['scope'] = null; + self::$lockRegistry[$path]['rolledBack'] = true; + $failed = []; + foreach($marks as $file => $mark){ + try { + $this->fault('rollback:truncate', ['db' => $dbname, 'file' => $file]); + if(!$this->truncateBack($file, $mark)){ + $failed[] = $file; + } + } catch(\Throwable $t){ + $failed[] = $file; + } + } + $this->forgetDirtySpatial($dbname); + $this->invalidateDbCaches($dbname); + if(!empty($failed)){ + self::$lockRegistry[$path]['flags']['j']['*'] = true; + try { + $this->saveLockState($path); + } catch(\Throwable $ignored){ + } + throw new \RuntimeException("noneDB: a write to database '{$dbname}' failed and could not be undone: outcome unknown, the database is recovered on its next access", self::ROLLBACK_FAILED, $e); + } + if(!empty($marks) && $this->isSharded($dbname)){ + try { + $this->recountMeta($dbname); // keys reserved by the failed write stay unused + } catch(\Throwable $ignored){ + // the journal entry stays, recovery recounts + } + } + throw $e; + } + + /** + * Cut a file back to the size recorded in a mark, if it is still the same file + */ + private function truncateBack($file, array $mark){ + list($size, $dev, $ino) = $mark; + $this->noteWrite($file); + $fp = @fopen($file, 'r+b'); + if($fp === false){ + return false; + } + try { + $stat = fstat($fp); + if($stat === false || ($ino && ($stat['ino'] !== $ino || $stat['dev'] !== $dev))){ + return false; + } + return $stat['size'] <= $size || ftruncate($fp, $size); + } finally { + fclose($fp); + clearstatcache(true, $file); + } + } + + /** + * Record the size a file had before the scope's first append to it + */ + private function markScopeFile($dbname, $file, $size, $fp){ + $lock = $this->getLockPath($dbname); + if(isset(self::$lockRegistry[$lock]['scope']) && !isset(self::$lockRegistry[$lock]['scope']['marks'][$file])){ + $stat = fstat($fp); + self::$lockRegistry[$lock]['scope']['marks'][$file] = [$size, $stat ? $stat['dev'] : 0, $stat ? $stat['ino'] : 0]; + } + } + + /** + * Run work that must not be undone by its caller's failure (buffer flush, migration): in place while the + * scope has no appends, else deferred until the scope committed (once per $job). Returns null if deferred. + */ + private function runIndependent($dbname, $job, callable $work){ + $path = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$path]['scope'])){ + return $work(); + } + if(!empty(self::$lockRegistry[$path]['scope']['marks'])){ + if(!isset(self::$lockRegistry[$path]['scope']['deferred'][$job])){ + self::$lockRegistry[$path]['scope']['deferred'][$job] = $work; + $this->fault('job:deferred', ['db' => $dbname, 'job' => $job]); + } + return null; + } + if(self::$faultHook !== null){ $this->fault('independent:run', ['db' => $dbname, 'job' => $job]); } + $was = self::$lockRegistry[$path]['scope']['independent']; + self::$lockRegistry[$path]['scope']['independent'] = true; + try { + $result = $work(); + $this->commitIndependentStep($dbname); + return $result; + } finally { + if(isset(self::$lockRegistry[$path]['scope'])){ + self::$lockRegistry[$path]['scope']['independent'] = $was; + } + } + } + + /** + * Work that replaces or removes data files (dropDB, compact, migrate) can't be undone by a rollback: + * inside a write of the same database that appended and hasn't committed (a nested call from the + * caller's own code) it is refused before it touches any file + * @throws \LogicException + */ + private function refuseInUncommittedWrite($dbname, $operation){ + $scope = self::$lockRegistry[$this->getLockPath($dbname)]['scope'] ?? null; + if($scope !== null && !empty($scope['marks']) && empty($scope['independent'])){ + throw new \LogicException("noneDB: {$operation}() can't run inside a write to database '{$dbname}' that hasn't committed yet (it replaces or deletes data files, which the write's rollback couldn't undo); call it after that write"); + } + } + + /** + * Before a file is replaced, renamed or removed: never one that holds appends a rollback may have to cut + * back (whatever path asks for it) + * @throws \LogicException + */ + private function refuseUncommittedFile($file, $operation){ + foreach(self::$lockRegistry as $entry){ + if(isset($entry['scope']['marks'][$file]) && empty($entry['scope']['independent'])){ + throw new \LogicException("noneDB: {$operation} of " . basename($file) . " refused: it holds appends of database '{$entry['dbname']}' that aren't committed yet"); + } + } + } + + /** + * Inside independent work: its appends so far are final (the scope had none before it started) + */ + private function commitIndependentStep($dbname){ + $path = $this->getLockPath($dbname); + if(!empty(self::$lockRegistry[$path]['scope']['independent'])){ + self::$lockRegistry[$path]['scope']['marks'] = []; + } + } + + private function getRecoveryLogPath($dbname){ + return $this->dbBase($dbname) . '.nonedb.recovery'; + } + + /** + * Append an entry to the database's recovery log (caller holds the exclusive lock). Best effort: a + * failure to log never fails the operation. Past 1 MiB the log moves to {log}.1 (replacing it). + */ + private function logRecoveryEvent($dbname, array $event, $dedupId = null){ + try { + $path = $this->getRecoveryLogPath($dbname); + if($dedupId !== null){ + $id = '"id":' . json_encode($dedupId); + foreach([$path . '.1', $path] as $file){ + $content = @file_get_contents($file); + // the id is followed by the next member or ends the entry + if($content !== false && (strpos($content, $id . ',') !== false || strpos($content, $id . '}') !== false)){ + return; // already logged + } + } + } + clearstatcache(true, $path); + if(@filesize($path) > 1048576){ + $this->fsRename($path, $path . '.1'); + } + $line = $this->encodeJson(['t' => time()] + $event); + if($line !== false){ + $this->appendToFile($path, $line . "\n"); + } + } catch(\Throwable $ignored){ + } + } + + /** + * True when mbstring can be used for Unicode case folding (like(), search()) + * @internal used by noneDBQuery; not part of the public API + */ + public static function hasMbstring(): bool { + return self::$mbstringFn !== null ? (bool)(self::$mbstringFn)() : function_exists('mb_convert_case'); + } + + /** + * valueKey() for the query builder + * @return \Closure + */ + private function valueKeyFunction(){ + return \Closure::fromCallable([self::class, 'valueKey']); + } + + /** + * The filter parameter of find(), count(), sum(), ...: 0, null (all records) and arrays are filters, + * anything else is invalid (null) + * @return array|null + */ + private function normalizeFilter($filter){ + if($filter === null || is_int($filter)){ + return []; + } + return is_array($filter) ? $filter : null; + } + + /** + * Records matching a filter parameter (see normalizeFilter); false for an invalid filter or a missing + * database without autoCreateDB. Caller holds the lock. + * @return array|false + */ + private function filteredUnlocked($dbname, $filter){ + $filters = $this->normalizeFilter($filter); + return $filters === null ? false : $this->findUnlocked($dbname, $filters); + } + + /** + * Test hook (no effect unless a test installed one) + */ + private function fault($point, array $context = []){ + if(self::$faultHook !== null){ + (self::$faultHook)($point, $context); + } + } + + private static function faultPoint($point, array $context = []){ + if(self::$faultHook !== null){ + (self::$faultHook)($point, $context); + } + } + + /** + * Open (or reuse) a lock file and lock it. + * A kept handle is only used in the process that opened it (a forked child would share the parent's + * locks), and any handle is only trusted while the path still names the same file: if the file was + * deleted or replaced meanwhile, the file the path names now is locked instead. + * @return resource + */ + private function lockFile($path, $mode, $dbname){ + $pid = getmypid(); + for($attempt = 0; ; $attempt++){ + $kept = self::$lockHandles[$path] ?? null; + unset(self::$lockHandles[$path]); + if($kept !== null && $kept['pid'] === $pid){ + $fp = $kept['fp']; + } else { + if($kept !== null){ + @fclose($kept['fp']); + } + $this->ensureDbDir(); + // "e": close-on-exec. A kept handle must not leak into programs this process starts: + // an inherited copy would keep a lock alive after this process died holding it. + $fp = @fopen($path, 'c+be'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot open lock file for database '{$dbname}'", self::LOCK_FAILED); + } + } + try { + $this->acquireFlock($fp, $mode, $dbname); + } catch(\Throwable $e){ + fclose($fp); + throw $e; + } + $own = fstat($fp); + clearstatcache(true, $path); + $named = @stat($path); + if($named !== false && ($own === false || !$own['ino'] || !$named['ino'] + || ($own['ino'] === $named['ino'] && $own['dev'] === $named['dev']))){ + return $fp; + } + flock($fp, LOCK_UN); + fclose($fp); + if($attempt >= 10){ + throw new \RuntimeException("noneDB: the lock file of database '{$dbname}' keeps being replaced", self::LOCK_FAILED); + } + } + } + + /** + * Keep an unlocked lock file handle open for the next operation (the oldest is closed past the limit) + */ + private function keepLockHandle($path, $fp){ + self::$lockHandles[$path] = ['fp' => $fp, 'pid' => getmypid()]; + if(count(self::$lockHandles) > self::LOCK_HANDLE_LIMIT){ + reset(self::$lockHandles); + $oldest = key(self::$lockHandles); + @fclose(self::$lockHandles[$oldest]['fp']); + unset(self::$lockHandles[$oldest]); + } + } + + /** + * Read the state record (caller holds the lock). If another process wrote the database since this + * process last looked, all cached state of the database is dropped. + */ + private function loadLockState($dbname, $path){ + $fp = self::$lockRegistry[$path]['fp']; + fseek($fp, 0); + $tokens = explode(' ', trim((string)fread($fp, self::LOCK_STATE_SIZE))); + $state = (string)array_shift($tokens); // "{generation}:{epoch}" + $flags = []; + foreach($tokens as $token){ + if($token === 'v' || $token === 'b'){ + $flags[$token] = true; + } elseif($token === 'J'){ + $flags['j']['*'] = true; + } elseif(strncmp($token, 'j=', 2) === 0){ + foreach(explode(',', substr($token, 2)) as $segment){ + if($segment !== ''){ + $flags['j'][$segment] = true; + } + } + } + } + // The caches were checked against this exact state: nothing to do (the usual case, one comparison; + // the epoch is parsed when needed: false = the valid one of the state) + $seen = $this->generations[$path] ?? null; + $epoch = false; + if($seen !== $state){ + $colon = strpos($state, ':'); + $epoch = $colon === false ? null : substr($state, $colon + 1); + if($epoch !== null && (strlen($epoch) !== 8 || !ctype_xdigit($epoch))){ + $epoch = null; + } + $seenEpoch = $seen === null || strpos($seen, ':') === false ? null : substr($seen, strpos($seen, ':') + 1); + if($seen === null || $epoch === null || $seenEpoch !== $epoch){ + $this->invalidateDbCaches($dbname); + } else { + $this->refreshDbCaches($dbname); // same epoch, another generation: the files only grew + } + $this->generations[$path] = $epoch === null ? null : $state; + } + self::$lockRegistry[$path]['gen'] = (int)$state; + self::$lockRegistry[$path]['state'] = $state; + self::$lockRegistry[$path]['epoch'] = $epoch; + self::$lockRegistry[$path]['flags'] = $flags; + } + + /** + * Write the state record of a database held exclusively. $bump: a file is about to be written, so the + * generation moves (once per session; noteWrite()): even a writer that dies half-way makes other + * processes check their caches. A record without epoch gets one. + */ + private function saveLockState($path, $bump = false){ + if($bump && !self::$lockRegistry[$path]['bumped']){ + self::$lockRegistry[$path]['gen']++; + self::$lockRegistry[$path]['bumped'] = true; + } + $epoch = $this->registryEpoch($path); + if($epoch === null){ + $epoch = self::$lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + } + $state = self::$lockRegistry[$path]['gen'] . ':' . $epoch; + $this->generations[$path] = $state; + $flags = self::$lockRegistry[$path]['flags']; + if(!empty($flags['v'])){ + $state .= ' v'; + } + if(!empty($flags['b'])){ + $state .= ' b'; + } + if(!empty($flags['j'])){ + $segments = ' j=' . implode(',', array_keys($flags['j'])); + $state .= (isset($flags['j']['*']) || strlen($state . $segments) > self::LOCK_STATE_SIZE) ? ' J' : $segments; + } + $fp = self::$lockRegistry[$path]['fp']; + fseek($fp, 0); + if(fwrite($fp, str_pad($state, self::LOCK_STATE_SIZE)) !== self::LOCK_STATE_SIZE || !fflush($fp)){ + throw new \RuntimeException("noneDB: cannot write the lock file of database '" . self::$lockRegistry[$path]['dbname'] . "'"); + } + // With fsync on, setting the journal or the buffer flag must reach the disk before the data it + // protects (clearing them doesn't matter: a power loss drops every cache anyway) + if($this->fsync && (!empty($flags['j']) || !empty($flags['b']))){ + $synced = $this->syncHandle($fp, $path); + if(!empty($flags['b'])){ + $this->noteSync('file', $synced); + } + } + } + + /** + * Change the state flag 'b' (write buffer may hold records); caller holds the exclusive lock + */ + private function setBufferFlag($dbname, $on){ + $path = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$path]) || !empty(self::$lockRegistry[$path]['flags']['b']) === $on){ + return; + } + if($on){ + self::$lockRegistry[$path]['flags']['b'] = true; + } else { + unset(self::$lockRegistry[$path]['flags']['b']); + } + $this->saveLockState($path); + } + + /** + * True when the state record of a locked database is authoritative (see 'v' above) + */ + private function isVerified($dbname){ + $path = $this->getLockPath($dbname); + return !empty(self::$lockRegistry[$path]['flags']['v']); + } + + /** + * After taking the lock: rebuild what a crashed writer left half-updated, and once per database + * look for pre-3.2 leftovers. Both need the exclusive lock and are rare. + */ + private function prepareDb($dbname, $path){ + if(self::$noWrites > 0){ + return; // checkIntegrity(): reports what recovery would do instead of doing it + } + $flags = self::$lockRegistry[$path]['flags']; + if(!empty($flags['v']) && empty($flags['j']) && self::$lockRegistry[$path]['epoch'] !== null){ + return; + } + $this->withLock($dbname, true, function() use ($dbname, $path){ + // Re-read under the exclusive lock: another process may have done it meanwhile + $noEpoch = self::$lockRegistry[$path]['epoch'] === null; + if(empty(self::$lockRegistry[$path]['flags']['v'])){ + $this->verifyDb($dbname, $path); + } elseif($noEpoch){ + $this->saveLockState($path); // written by v3.2 or a new lock file: give it an epoch + } + if($noEpoch){ + // v3.2 may have used the database: its own index files may be back, and it can leave the + // shard counters of a database this version wrote behind the data + $this->forgetIndexConversion($dbname); + if($this->isSharded($dbname)){ + $this->recountMeta($dbname); + } + } + if(!empty(self::$lockRegistry[$path]['flags']['j'])){ + $this->recoverPendingIndexes($dbname, $path); + } + }); + } + + /** + * One-time check of a database whose state record isn't verified yet (created before v3.2, or its + * lock file was deleted): pick up pre-3.2 write buffers and store the index list, so neither has to + * be looked for again + */ + private function verifyDb($dbname, $path){ + if($this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)) || !empty($this->legacyShardBufferFiles($dbname))){ + self::$lockRegistry[$path]['flags']['b'] = true; + } + $manifestPath = $this->getIndexManifestPath($dbname); + clearstatcache(true, $manifestPath); + if(!is_file($manifestPath)){ + unset($this->fieldIndexCache[$manifestPath]); + $manifest = $this->deriveIndexManifest($dbname); + if(!empty($manifest['fields']) || !empty($manifest['spatial'])){ + $this->saveIndexManifest($dbname, $manifest); + } + } + self::$lockRegistry[$path]['flags']['v'] = true; + $this->saveLockState($path); + } + + /** + * Called before a segment's data changes. Index files of v3.2 the segment still has are converted + * first (v3.2 would not see the change in them). The journal is needed only for shard meta counters + * (sharded) and migrations ($always): field and spatial indexes follow the data by themselves. + */ + private function markIndexesPending($dbname, $shardId, $always = false){ + $path = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$path])){ + return; + } + $this->convertSegmentIndexes($dbname, $shardId); + if(!$always && $shardId === null){ + return; + } + $segment = $shardId === null ? 'main' : (string)$shardId; + if(isset(self::$lockRegistry[$path]['pending'][$segment])){ + return; + } + self::$lockRegistry[$path]['pending'][$segment] = true; + self::$lockRegistry[$path]['flags']['j'][$segment] = true; + $this->saveLockState($path, true); // the data write follows + } + + /** + * Rebuild field and spatial indexes of the segments a crashed writer left half-updated + * (caller holds the exclusive lock) + */ + private function recoverPendingIndexes($dbname, $path){ + $pending = self::$lockRegistry[$path]['flags']['j'] ?? []; + $known = $this->listSegments($dbname); + $segments = isset($pending['*']) ? $known : array_map(function($segment){ + return $segment === 'main' ? null : (int)$segment; + }, array_keys($pending)); + $manifest = $this->getIndexManifest($dbname); + $segments = array_values(array_filter($segments, function($shardId) use ($known){ + return in_array($shardId, $known, true); + })); + foreach($segments as $shardId){ + // Key, field and spatial indexes follow the data by themselves (checkpoint + replay) + $this->readJsonlIndex($dbname, $shardId); + } + // A writer that died left its temp files (atomic writes, quarantine) + $this->removeStaleTemp($this->dbBase($dbname)); + $this->removeStaleTemp($this->dbDir . $this->hashDBName($dbname)); + // A migration that died: its shards (still the old file) or its marker (already sharded) + $this->discardInterruptedMigration($dbname); + if($this->isSharded($dbname)){ + // A write may have died between its data and its meta update + $this->recountMeta($dbname); + // A migration may have died after switching to shards but before removing the old file + $legacy = $this->getDataPath($dbname); + clearstatcache(true, $legacy); + if(file_exists($legacy)){ + $this->bumpEpoch($dbname); + $this->fsUnlink($legacy . '.jidx'); + $this->fsUnlink($legacy); + $this->markFileNotExists($legacy); + unset($this->indexCache[$legacy . '.jidx'], $this->jsonlFormatCache[$legacy]); + foreach($manifest['fields'] as $field){ + $this->deleteFieldIndexFile($dbname, $field, null); + } + foreach($manifest['spatial'] as $field){ + $this->fsUnlink($this->getSpatialIndexPath($dbname, $field, null)); + $this->fsUnlink($this->getLegacySpatialIndexPath($dbname, $field, null)); + } + } + } + unset(self::$lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + + /** + * flock with timeout (lockTimeout seconds; 0 waits forever) + * @throws \RuntimeException on timeout + */ + private function acquireFlock($fp, $mode, $dbname){ + if($this->lockTimeout <= 0){ + if(!flock($fp, $mode)){ + throw new \RuntimeException("noneDB: could not lock database '{$dbname}'", self::LOCK_FAILED); + } + return; + } + $start = microtime(true); + while(!flock($fp, $mode | LOCK_NB)){ + if((microtime(true) - $start) >= $this->lockTimeout){ + throw new \RuntimeException("noneDB: timed out after {$this->lockTimeout}s waiting for the lock on database '{$dbname}'", self::LOCK_FAILED); + } + usleep($this->lockRetryDelay); + } + } + + /** + * A file is about to be written: the first write of an exclusive session bumps the generation of the + * database the file belongs to (see saveLockState). A database file written without its database's + * exclusive lock is a bug: LogicException. Files of no database (dbDir housekeeping) pass. + * @throws \LogicException + */ + private function noteWrite($file){ + $file = (string)$file; + unset($this->dataSizes[$file]); // a size cached for global index checks + $owner = null; + foreach(self::$lockRegistry as $path => $entry){ + // Files of a database start with {dbDir}{hash}: the lock path up to the hash + $prefix = substr($path, 0, strrpos($path, '/') + 21); + if(strncmp($file, $prefix, strlen($prefix)) === 0 && strpos('-._', substr($file, strlen($prefix), 1)) !== false){ + $owner = $path; + break; + } + } + if($owner !== null){ + if(substr($file, -7) === '.nonedb'){ + // A data file: global index lookups check its shard's newer lines (globalCheckShards) + $writes = &$this->segWrites[$owner]; + $writes['seq'] = ($writes['seq'] ?? 0) + 1; + $writes['paths'][$file] = $writes['seq']; + unset($writes); + } + if(self::$lockRegistry[$owner]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: write to " . basename($file) . " under a shared lock of database '" . self::$lockRegistry[$owner]['dbname'] . "'"); + } + if(!self::$lockRegistry[$owner]['bumped']){ + $this->saveLockState($owner, true); + } + } elseif(preg_match('/^[0-9a-f]{20}[-._]/', basename($file)) === 1){ + throw new \LogicException("noneDB: write to " . basename($file) . " without the lock of its database"); + } + if(!empty($this->unsavedHashes)){ + $unsaved = $this->unsavedHashes; + $this->unsavedHashes = []; + foreach($unsaved as $dbname => $hash){ + $this->savePersistentHashCacheEntry((string)$dbname, $hash); + } + } + } + + /** + * Drop every cached entry that belongs to a database (index, meta, field/spatial indexes, file-exists) + */ + private function invalidateDbCaches($dbname){ + $prefix = $this->dbDir . $this->hashDBName($dbname); + $len = strlen($prefix); + $caches = [ + &$this->indexCache, &$this->verifiedIndexes, &$this->shardedCache, &$this->metaCache, &$this->metaCacheTime, + &$this->jsonlFormatCache, &$this->fieldIndexCache, &$this->spatialIndexCache, + &$this->globalSpatialIndexCache, &$this->dataSizes, &$this->dataEpochs, &$this->convertedSegments, &$this->shardFilesUpTo, + &self::$secondaryChecked, &self::$tailCache, + ]; + foreach($caches as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($cache[$key]); + } + } + } + unset($cache); + foreach(array_keys(self::$staticFileExistsCache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset(self::$staticFileExistsCache[$key]); + } + } + $lockPath = $this->getLockPath($dbname); + unset($this->globalSkip[$lockPath], $this->segWrites[$lockPath]); + foreach(array_keys(self::$bufferCounts) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset(self::$bufferCounts[$key], self::$bufferEnds[$key]); + } + } + foreach(array_keys($this->writableIndexes) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->writableIndexes[$key]); + } + } + } + + /** + * Caches of a database after another process wrote it without replacing a data file (same epoch): + * everything is dropped but the key indexes, which are checked again (the data only grew: the new lines + * are replayed on the next read) + */ + private function refreshDbCaches($dbname){ + $prefix = $this->dbDir . $this->hashDBName($dbname); + $len = strlen($prefix); + // Kept (they describe data bytes that didn't change, the new lines are replayed when used): key + // indexes, field indexes and R-trees, what was checked against the data, parsed tails, epochs + $keep = []; + foreach(['indexCache' => &$this->indexCache, 'fieldIndexCache' => &$this->fieldIndexCache, + 'spatialIndexCache' => &$this->spatialIndexCache, 'dataEpochs' => &$this->dataEpochs, + 'secondaryChecked' => &self::$secondaryChecked, 'tailCache' => &self::$tailCache] as $name => &$cache){ + foreach($cache as $key => $value){ + if(strncmp((string)$key, $prefix, $len) === 0 + && ($name !== 'fieldIndexCache' && $name !== 'spatialIndexCache' || (is_array($value) && isset($value['kind']) && empty($value['legacy'])))){ + $keep[$name][$key] = $value; + } + } + } + unset($cache); + $this->invalidateDbCaches($dbname); + foreach($keep['indexCache'] ?? [] as $key => $value){ + $this->indexCache[$key] = $value; + } + foreach($keep['fieldIndexCache'] ?? [] as $key => $value){ + $this->fieldIndexCache[$key] = $value; + } + foreach($keep['spatialIndexCache'] ?? [] as $key => $value){ + $this->spatialIndexCache[$key] = $value; + } + foreach($keep['dataEpochs'] ?? [] as $key => $value){ + $this->dataEpochs[$key] = $value; + } + foreach($keep['secondaryChecked'] ?? [] as $key => $value){ + self::$secondaryChecked[$key] = $value; + } + foreach($keep['tailCache'] ?? [] as $key => $value){ + self::$tailCache[$key] = $value; + } + } + + /** + * Epoch of a locked database's state record (null: none or invalid) + */ + private function registryEpoch($path){ + $epoch = self::$lockRegistry[$path]['epoch'] ?? null; + if($epoch === false){ + $state = self::$lockRegistry[$path]['state']; + $epoch = substr($state, strpos($state, ':') + 1); // a state with a valid epoch (see loadLockState) + self::$lockRegistry[$path]['epoch'] = $epoch; + } + return $epoch; + } + + /** + * A data file of the database is about to be replaced or removed (compaction, migration, drop): new + * epoch, so every other process drops all its caches of the database (caller holds the exclusive lock) + */ + /** + * Global field and spatial index files changed in memory during this lock session (a checkpoint per + * shard updates them): written once here, at the end of the session, at an epoch change or before the + * lock is upgraded, instead of once per shard (O(shards²) for an operation that checkpoints every + * shard). Not written when the session failed or the epoch changed since: a missing entry only means + * the shard is never skipped. + */ + private function flushPendingGlobals($dbname, $path, $ok = true){ + $pending = self::$lockRegistry[$path]['globals'] ?? []; + if(empty($pending)){ + return; + } + self::$lockRegistry[$path]['globals'] = []; + $epoch = $this->registryEpoch($path); + foreach($pending as $file => [$kind, $field, $deferredAt]){ + $global = $kind === 'gfx' ? ($this->fieldIndexCache[$file] ?? null) : ($this->globalSpatialIndexCache[$file] ?? null); + if(!$ok || !empty(self::$lockRegistry[$path]['rolledBack']) || $deferredAt !== $epoch || $global === null){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + continue; + } + try { + if($kind === 'gfx'){ + $this->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + } catch(\Throwable $e){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: a shard without a fitting entry is never skipped + } + } + } + + private function bumpEpoch($dbname){ + $path = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$path]) || self::$lockRegistry[$path]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: new epoch of database '{$dbname}' without its exclusive lock"); + } + $this->flushPendingGlobals($dbname, $path); // under the epoch their entries belong to + self::$lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + $this->saveLockState($path, true); + unset($this->globalSkip[$path], $this->segWrites[$path]); + // What this process checked against the data files no longer holds for the one being replaced + $prefix = $this->dbDir . $this->hashDBName($dbname); + foreach([&$this->dataSizes, &$this->dataEpochs, &self::$secondaryChecked, &self::$tailCache] as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, strlen($prefix)) === 0){ + unset($cache[$key]); + } + } + } + unset($cache); + } + + // ========================================== + // ATOMIC FILE OPERATIONS + // ========================================== + + /** + * Encode JSON, returning false instead of producing invalid output + * @return string|false + */ + private function encodeJson($data, $prettyPrint = false, $extraFlags = 0){ + $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | $extraFlags; + if($prettyPrint){ + $flags |= JSON_PRETTY_PRINT; + } + // json_encode() writes floats with serialize_precision digits: anything but -1 (the default, + // shortest exact form) would store rounded values (0.30000000000000004 as 0.3 at 10). + // Loops encoding many records run inside withExactFloats(): no ini_get()/finally per record. + if(self::$exactFloats || ini_get('serialize_precision') === '-1'){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; + } + } + return $this->withExactFloats(function() use ($data, $flags){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; + } + }); + } + + /** + * Run $fn with serialize_precision set to -1 (checked once instead of once per encoded value) + */ + private function withExactFloats(callable $fn){ + if(self::$exactFloats){ + return $fn(); + } + $precision = ini_get('serialize_precision'); + if($precision !== '-1'){ + ini_set('serialize_precision', '-1'); + } + self::$exactFloats = true; + try { + return $fn(); + } finally { + self::$exactFloats = false; + if($precision !== '-1'){ + ini_set('serialize_precision', (string)$precision); + } + } + } + + /** + * Encode one JSONL record line (without the trailing newline) + * @throws \InvalidArgumentException when the record can't be represented as JSON (invalid UTF-8, NAN, INF, ...) + */ + private function encodeRecord(array $record){ + // Keep floats floats: 2.0 / -0.0 are stored as 2.0 / -0.0, not as the ints 2 / 0 + $json = $this->encodeJson($record, false, JSON_PRESERVE_ZERO_FRACTION); + if($json === false){ + throw new \InvalidArgumentException('noneDB: record cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value): ' . json_last_error_msg()); + } + return $json; + } + + /** + * Write a whole file atomically: temp file in the same directory + rename. + * Readers see either the old or the new content, never a partial file. + * @param string|bool $mode How it must survive a power loss: + * 'derived' (false): indexes, rebuilt from the data if lost; synced when the fsync setting is on + * 'durable': meta, index list, info; synced when the fsync setting is on, needed by the operation + * 'replace' (true): a data file replaced (compaction, migration); synced whenever PHP can + * @return bool + */ + private function writeFileAtomic($path, $content, $mode = 'derived', $note = true){ + $mode = $mode === true ? 'replace' : ($mode === false ? 'derived' : $mode); + if($mode === 'replace'){ + $this->refuseUncommittedFile($path, 'replacement'); + } + if($note){ + $this->noteWrite($path); + } + $sync = function_exists('fsync') && ($this->fsync || $mode === 'replace'); + $dir = dirname($path); + if(!is_dir($dir) && !@mkdir($dir, 0755, true) && !is_dir($dir)){ + return false; + } + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + if($sync){ + $fp = @fopen($tmp, 'xb'); + $ok = $fp !== false; + for($done = 0, $length = strlen($content); $ok && $done < $length; $done += $n){ + $n = fwrite($fp, $done === 0 ? $content : substr($content, $done)); + $ok = $n !== false && $n > 0; + } + if($fp !== false){ + $synced = $ok && fflush($fp) ? $this->syncHandle($fp, $path) : false; + fclose($fp); + if($mode !== 'derived'){ + $this->noteSync('file', $synced); + } + $ok = $ok && $synced !== false; + } + if(!$ok){ + @unlink($tmp); + return false; + } + } else { + $written = @file_put_contents($tmp, $content); + if($written === false || $written !== strlen($content)){ + @unlink($tmp); + return false; + } + if($mode !== 'derived'){ + $this->noteSync('file', null); + } + } + if(!$this->renameWithRetry($tmp, $path)){ + @unlink($tmp); + return false; + } + clearstatcache(true, $path); + if($sync){ + $dirSynced = $this->syncDirectory($dir); + if($mode !== 'derived'){ + $this->noteSync('dir', $dirSynced); // a failed directory sync is reported, not fatal + } + } elseif($mode !== 'derived'){ + $this->noteSync('dir', null); + } + return true; + } + + /** + * Write a file derived from the data (a checkpoint) atomically without moving the generation: its + * content is the same whoever writes it, so a reader may write it under the shared lock + */ + private function writeDerivedFile($path, $content){ + return $this->writeFileAtomic($path, $content, 'derived', false); + } + + // File system writes (v3.3.0, S3): every write to a database file goes through writeFileAtomic(), + // appendToFile() or the helpers below. Each calls noteWrite() first: the generation moves before the + // file changes, so other processes drop their caches even if this one dies half-way, and a write + // without the database's exclusive lock is a LogicException. SourceWriteGuardTest fails on a raw + // write call anywhere else. + + private function fsUnlink($path){ + clearstatcache(true, $path); + if(!file_exists($path) && !is_link($path)){ + return false; // nothing to remove: no write + } + $this->refuseUncommittedFile($path, 'removal'); + $this->noteWrite($path); + $ok = @unlink($path); + clearstatcache(true, $path); + return $ok; + } + + private function fsRename($from, $to){ + $this->refuseUncommittedFile($from, 'rename'); + $this->refuseUncommittedFile($to, 'replacement'); + $this->noteWrite($from); + $this->noteWrite($to); + $ok = $this->renameWithRetry($from, $to); + clearstatcache(true, $from); + clearstatcache(true, $to); + return $ok; + } + + private function fsTouch($path){ + $this->noteWrite($path); + $ok = @touch($path); + clearstatcache(true, $path); + return $ok; + } + + /** Cut a file to $size bytes */ + private function fsTruncate($path, $size){ + $this->noteWrite($path); + $fp = @fopen($path, 'r+b'); + if($fp === false){ + return false; + } + $ok = ftruncate($fp, $size); + fclose($fp); + clearstatcache(true, $path); + return $ok; + } + + /** Write a whole small file in place (not atomic: for content that never matters to the data) */ + private function fsPutContents($path, $content){ + $this->noteWrite($path); + return @file_put_contents($path, $content) === strlen($content); + } + + /** + * Append bytes to a file. Returns the offset the bytes start at, or false. + * A partial write is rolled back so the file never ends in a torn line. + * @param string|null $dbname Database whose rollback scope records the append (data and buffer files) + * @param int|null $expected Size the caller's index says the file has: anything else writes nothing + * and sets $this->appendMismatch + * @return int|false + */ + private function appendToFile($path, $bytes, $dbname = null, $expected = null, $segment = null){ + $this->noteWrite($path); + $this->appendMismatch = false; + $this->appendSyncFailed = false; + $fp = @fopen($path, 'c+b'); + if($fp === false){ + return false; + } + try { + if(fseek($fp, 0, SEEK_END) !== 0){ + return false; + } + $offset = ftell($fp); + if($offset === false){ + return false; + } + if($expected !== null && $offset !== (int)$expected){ + // The file changed behind the cached index. An unterminated tail only (an append that + // never completed): quarantined and cut, then this append goes where the index expects it. + // Anything else: nothing is written, the caller drops its index (rebuilt on next access). + if($dbname === null || $offset < (int)$expected || $this->lastNewlineIn($path, (int)$expected, $offset) >= 0){ + $this->appendMismatch = true; + return false; + } + $this->quarantineAndCut($dbname, $segment, $path, (int)$expected, $offset); + fseek($fp, 0, SEEK_END); + $offset = ftell($fp); + if($offset !== (int)$expected){ + $this->appendMismatch = true; + return false; + } + } + if($dbname !== null){ + $this->markScopeFile($dbname, $path, $offset, $fp); + } + $length = strlen($bytes); + $done = 0; + while($done < $length){ + $n = fwrite($fp, $done === 0 ? $bytes : substr($bytes, $done)); + if($n === false || $n === 0){ + $this->undoAppend($fp, $offset, $path); + return false; + } + $done += $n; + } + // Written bytes the stream couldn't hand to the OS (disk full, I/O error): the append failed + if(!(self::$flushFn !== null ? (self::$flushFn)($fp, $path) : fflush($fp))){ + $this->undoAppend($fp, $offset, $path); + return false; + } + if($dbname !== null){ + // Data and buffer: synced with fsync on (a failed sync undoes this append), the directory + // too when the append created the file + if(!$this->syncForWrite($fp, $path)){ + $this->appendSyncFailed = true; + $this->undoAppend($fp, $offset, $path); + return false; + } + if($offset === 0){ + $this->syncForWrite(null, dirname($path), 'dir'); + } + if(self::$faultHook !== null){ $this->fault('append:after', ['db' => $dbname, 'path' => $path]); } + } + return $offset; + } finally { + fclose($fp); + clearstatcache(true, $path); + } + } + + /** + * Cut a failed append back. If that fails too, the bytes may still be there: an exception, so the + * write's rollback scope tries again (ROLLBACK_FAILED when it can't either), never a plain failure + * that would leave them to be read as a record + * @throws \RuntimeException + */ + private function undoAppend($fp, $offset, $path){ + try { + if(self::$faultHook !== null){ $this->fault('append:undo', ['path' => $path]); } + $ok = ftruncate($fp, $offset); + } catch(\Throwable $e){ + $ok = false; + } + if(!$ok){ + throw new \RuntimeException("noneDB: an append to " . basename($path) . " failed and could not be cut back"); + } + } + + /** + * Read and decode a JSON file + * + * @param string $path File path + * @param mixed $default Value returned when the file doesn't exist or is empty + * @return mixed Decoded data, $default, or null when the file is corrupt + */ + private function atomicRead($path, $default = null){ + clearstatcache(true, $path); + if(!is_file($path)){ + return $default; + } + $content = @file_get_contents($path); + if($content === false || $content === ''){ + return $default; + } + $data = json_decode($content, true); + if($data === null && json_last_error() !== JSON_ERROR_NONE){ + return null; // corrupted + } + return $data !== null ? $data : $default; + } + + /** + * Encode and write a JSON file atomically. Nothing is touched when encoding fails. + * + * @param string $path File path + * @param mixed $data Data to write (will be JSON encoded) + * @param bool $prettyPrint Use JSON_PRETTY_PRINT + * @return bool Success + */ + private function atomicWrite($path, $data, $prettyPrint = false, $mode = 'derived'){ + $json = $this->encodeJson($data, $prettyPrint); + if($json === false){ + return false; + } + return $this->writeFileAtomic($path, $json, $mode); + } + + // ========================================== + // STORAGE CORE (v3.2.0) + // ========================================== + // + // A database is one "segment" when it is not sharded ({base}.nonedb + {base}.nonedb.jidx) or one + // segment per shard ({base}_sN.nonedb + .jidx) listed in {base}.nonedb.meta. Segment ids are null + // (non-sharded) or the shard id. Every write runs under the database's exclusive lock (withLock), + // reads under a shared lock, so index read-modify-write cycles are never interleaved. + // + // Data files are append-only JSONL. Every record line carries its "key"; deletions append a + // tombstone line {"__nonedb_deleted":key}. The .jidx index maps key => [offset, length] and stores + // "s", the data size it covers. The index is used only while the data file ends exactly at "s", or + // continues after it with an unterminated tail (a crashed append, never committed: readers ignore it, + // the next writer moves it to a quarantine file and cuts it off). Anything else (complete lines the + // index doesn't know, a shorter file, a missing or corrupt index) rebuilds the index from the data. + + /** + * Files of dbDir whose path starts with $prefix and whose remaining name matches $regex (if given), + * sorted. Not glob(): a dbDir containing [ ] * ? is a pattern to glob() and matches nothing. + * @return string[] + */ + private function listFilesWithPrefix($prefix, $regex = null){ + self::$directoryListings++; + $dir = $this->dbDir; + if(strncmp($prefix, $dir, strlen($dir)) !== 0){ + return []; + } + $start = substr($prefix, strlen($dir)); + $length = strlen($start); + $files = []; + foreach(@scandir($dir) ?: [] as $file){ + if(strncmp($file, $start, $length) === 0 && ($regex === null || preg_match($regex, substr($file, $length)) === 1)){ + $files[] = $dir . $file; + } + } + return $files; + } + + /** + * Every file of a database except its lock file: {hash}-{name}.*, {hash}-{name}_sN.*, and the files + * named after the hash only ({hash}.nonedb.* / {hash}_sN.nonedb.*: spatial indexes, quarantine) + * @return string[] + */ + private function listDbFiles($dbname){ + $base = $this->dbBase($dbname); + $hashBase = $this->dbDir . $this->hashDBName($dbname); + $files = array_merge( + $this->listFilesWithPrefix($base . '.nonedb'), + $this->listFilesWithPrefix($base . '_s', '/^\d+\.nonedb/'), + $this->listFilesWithPrefix($hashBase . '.nonedb.'), + $this->listFilesWithPrefix($hashBase . '_s', '/^\d+\.nonedb\./') + ); + $lock = $base . '.nonedb.lock'; + return array_values(array_filter(array_unique($files), function($file) use ($lock){ + return $file !== $lock; + })); + } + + /** + * Get the file path for a specific shard + * @param string $dbname + * @param int $shardId + * @return string + */ + private function getShardPath($dbname, $shardId){ + return $this->dbBase($dbname) . "_s" . $shardId . ".nonedb"; + } + + /** + * Get the meta file path for a database + * @param string $dbname + * @return string + */ + private function getMetaPath($dbname){ + return $this->dbBase($dbname) . ".nonedb.meta"; + } + + /** + * Data file of a segment (null = non-sharded database) + */ + private function segDataPath($dbname, $shardId){ + return $shardId === null ? $this->getDataPath($dbname) : $this->getShardPath($dbname, $shardId); + } + + /** + * Check if a database is sharded + * @param string $dbname + * @return bool + */ + private function isSharded($dbname){ + $metaPath = $this->getMetaPath($dbname); + if(isset($this->shardedCache[$metaPath])){ + return $this->shardedCache[$metaPath]; + } + + clearstatcache(true, $metaPath); + $result = file_exists($metaPath); + $this->shardedCache[$metaPath] = $result; + return $result; + } + + /** + * True when the database has data (non-sharded file or sharded meta) + */ + private function dbExists($dbname){ + return $this->cachedFileExists($this->getDataPath($dbname)) || $this->isSharded($dbname); + } + + /** + * Read shard metadata + * @param string $dbname + * @return array|null + */ + private function readMeta($dbname){ + $meta = $this->atomicRead($this->getMetaPath($dbname), null); + return is_array($meta) && isset($meta['shards']) && is_array($meta['shards']) ? $meta : null; + } + + /** + * Get cached meta data. The cache is dropped whenever another process writes the + * database (generation check in withLock), so no TTL is needed. + * @param string $dbname + * @param bool $forceRefresh Force refresh from disk + * @return array|null + */ + private function getCachedMeta($dbname, $forceRefresh = false){ + $metaPath = $this->getMetaPath($dbname); + + self::$cacheTouch[$metaPath] = ++self::$cacheTick; + if(!$forceRefresh && isset($this->metaCache[$metaPath])){ + return $this->metaCache[$metaPath]; + } + + $meta = $this->readMeta($dbname); + if($meta === null){ + clearstatcache(true, $metaPath); + if(is_file($metaPath)){ + // There but unreadable (e.g. zero-length after a power loss): rebuilt from the shards. + // checkIntegrity() writes nothing: it works with a meta derived in memory, not kept + if(self::$noWrites > 0){ + try { + return $this->deriveShardMeta($dbname); + } catch(\RuntimeException $e){ + return null; + } + } + $meta = $this->recoverMeta($dbname); + } + } + if($meta !== null){ + $this->metaCache[$metaPath] = $meta; + self::$cacheGrew = true; + return $meta; + } + unset($this->metaCache[$metaPath]); + return null; + } + + /** + * Rebuild an unreadable meta file from the shard files (exclusive lock): the shard size is derived from + * the keys each shard holds (the configured size when it fits), nextKey from the highest key ever used. + * A missing meta is never "recovered": shard files without one are leftovers (see setOrphanShardsAside). + * @return array|null + * @throws \RuntimeException when no shard size fits the keys found + */ + private function recoverMeta($dbname){ + return $this->withLock($dbname, true, function() use ($dbname){ + $metaPath = $this->getMetaPath($dbname); + unset($this->metaCache[$metaPath]); + $meta = $this->readMeta($dbname); + clearstatcache(true, $metaPath); + if($meta !== null || !is_file($metaPath)){ + return $meta; + } + $meta = $this->deriveShardMeta($dbname); + $this->writeMeta($dbname, $meta, true); + $this->recountMeta($dbname); + $this->logRecoveryEvent($dbname, ['action' => 'meta-rebuilt', 'shards' => count($meta['shards']), 'shardSize' => $meta['shardSize']]); + return $this->getCachedMeta($dbname); + }); + } + + /** + * A shard meta derived from the shard files alone (reads only; counters 0, recountMeta() fills them) + * @return array + * @throws \RuntimeException when no shard size fits the keys found + */ + private function deriveShardMeta($dbname){ + $base = $this->dbBase($dbname); + $ranges = []; + $recorded = []; // shard sizes the shard checkpoints recorded ("z", v3.3) + foreach($this->listFilesWithPrefix($base . '_s', '/^\d+\.nonedb$/') as $file){ + $shardId = (int)substr($file, strlen($base) + 2); + $checkpoint = $this->atomicRead($file . '.jidx', null); + if(is_array($checkpoint) && isset($checkpoint['z']) && is_int($checkpoint['z']) && $checkpoint['z'] > 0){ + $recorded[$checkpoint['z']] = true; + } + $min = null; + $max = null; + $fp = @fopen($file, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->classifyLine(substr($line, 0, -1), 0, null)) !== null){ + $min = $min === null ? $entry[1] : min($min, $entry[1]); + $max = $max === null ? $entry[1] : max($max, $entry[1]); + } + } + if($fp !== false){ + fclose($fp); + } + $ranges[$shardId] = [$min, $max]; + } + ksort($ranges); + // i * size <= min key of shard i, max key of shard i < (i + 1) * size + $low = 1; + $high = PHP_INT_MAX; + $nextKey = 0; + foreach($ranges as $shardId => $range){ + if($range[0] === null){ + continue; + } + if($shardId > 0){ + $high = min($high, intdiv($range[0], $shardId)); + } + $low = max($low, intdiv($range[1], $shardId + 1) + 1); + $nextKey = max($nextKey, $range[1] + 1); + } + if($low > $high){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable and its shard size can't be derived; restore it, or call rebuildIndex() after writing a meta file with the right shardSize"); + } + // The size a shard checkpoint recorded, else the configured one, if it fits every shard's keys + $size = count($recorded) === 1 && key($recorded) >= $low && key($recorded) <= $high ? key($recorded) + : ($this->shardSize >= $low && $this->shardSize <= $high ? $this->shardSize : $low); + $meta = ['version' => 1, 'shardSize' => $size, 'totalRecords' => 0, 'deletedCount' => 0, 'nextKey' => $nextKey, 'shards' => []]; + foreach(array_keys($ranges) as $shardId){ + $meta['shards'][] = ['id' => $shardId, 'file' => '_s' . $shardId, 'count' => 0, 'deleted' => 0]; + } + return $meta; + } + + /** + * Write shard metadata (caller holds the exclusive lock) + * @param string $dbname + * @param array $meta + * @return bool + * @throws \RuntimeException when the meta file can't be written + */ + private function writeMeta($dbname, $meta, $sync = false){ + $path = $this->getMetaPath($dbname); + if(!$this->atomicWrite($path, $meta, true, $sync ? 'replace' : 'durable')){ + throw new \RuntimeException("noneDB: failed to write metadata of database '{$dbname}'"); + } + $this->metaCache[$path] = $meta; + $this->shardedCache[$path] = true; + $this->markFileExists($path); + return true; + } + + /** + * Records per shard of a database (existing sharded databases keep the size stored in their meta) + */ + private function shardSizeOf($dbname){ + $meta = $this->isSharded($dbname) ? $this->getCachedMeta($dbname) : null; + if($meta !== null && isset($meta['shardSize']) && (int)$meta['shardSize'] > 0){ + return (int)$meta['shardSize']; + } + return $this->shardSize; + } + + /** + * Segment ids of a database: [null] when not sharded, the shard ids otherwise, [] if it doesn't exist + */ + private function listSegments($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->getCachedMeta($dbname); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + return array_map('intval', array_column($meta['shards'], 'id')); + } + return $this->cachedFileExists($this->getDataPath($dbname)) ? [null] : []; + } + + /** + * Segment that holds a key, or false if that shard doesn't exist + * @return int|null|false + */ + private function segmentForKey($dbname, $key){ + if(!$this->isSharded($dbname)){ + return null; + } + $shardId = intdiv($key, $this->shardSizeOf($dbname)); + return in_array($shardId, $this->listSegments($dbname), true) ? $shardId : false; + } + + // ========================================== + // JSONL SEGMENT INDEX + // ========================================== + + /** + * Get JSONL index path + * @param string $dbname + * @param int|null $shardId Null for non-sharded + * @return string + */ + private function getJsonlIndexPath($dbname, $shardId = null){ + return $this->segDataPath($dbname, $shardId) . ".jidx"; + } + + // Segment index (v3.3.0): checkpoint + replay. The data file is a log; the .jidx file is a checkpoint + // of the index at byte offset "s" of it: + // {"v":4,"format":"jsonl","created":T,"updated":T,"n":next key,"d":garbage lines,"t":tombstone lines, + // "s":S,"h":"crc32b of the up to 256 bytes before S","e":"segment epoch","z":shard size,"sid":shard id, + // "p":{key:offset},"l":{key:length}} + // Loading reads the checkpoint and replays the complete lines after S (an unterminated tail is never + // part of it, readers never cut it). Writes append to the data and update the cached index only; a + // write session writes a new checkpoint of a segment it changed once its tail reached 64 lines plus + // 1/16 of the live records, or 1 MiB (checkpointTouchedSegments). A checkpoint counts only for its data + // file: S within it, at a line end, h matching; a data file is replaced only after its checkpoint was + // deleted. No "o" key on purpose: v3.2 then knows no index and rebuilds it from the data (downgrade). + // A v3 index (v3.2) is read like a checkpoint and deleted before the first append (v3.2 would trust its + // stale "n"). "e" is the segment's data epoch for the secondary indexes: new when the data file is + // replaced or the index rebuilt without a checkpoint, assigned only under the exclusive lock. + // Runtime keys of a cached index (never written): "ck" = S of the checkpoint on disk (-1: none), + // "tl" = lines after it, "v3" = the file on disk is a v3 index. + + private const CHECKPOINT_LINES = 64; + private const CHECKPOINT_BYTES = 1048576; + + private function newJsonlIndex(){ + return ['v' => 4, 'format' => 'jsonl', 'created' => time(), 'n' => 0, 'd' => 0, 't' => 0, 's' => 0, + 'e' => null, 'p' => [], 'l' => [], 'ck' => -1, 'tl' => 0]; + } + + /** + * A segment's key index (offset and length of every live record), from the cache, or loaded from its + * checkpoint and the data after it + * @param string $dbname + * @param int|null $shardId + * @return array|null Null when the segment doesn't exist + */ + private function readJsonlIndex($dbname, $shardId = null){ + $dataPath = $this->segDataPath($dbname, $shardId); + if(!$this->cachedFileExists($dataPath)){ + return null; + } + $path = $dataPath . '.jidx'; + if(isset($this->indexCache[$path])){ + self::$cacheTouch[$path] = ++self::$cacheTick; + if(isset($this->verifiedIndexes[$path])){ + return $this->indexCache[$path]; + } + // Another process wrote the database since this one cached the index, without replacing a data + // file (same epoch): the data only grew, the new lines are replayed + $index = $this->indexCache[$path]; + unset($this->indexCache[$path]); + clearstatcache(true, $dataPath); + $size = @filesize($dataPath); + if($size !== false && $size >= (int)$index['s']){ + if($size > (int)$index['s']){ + $this->replaySegment($dbname, $shardId, $index, $size); + } + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->maybeReaderCheckpoint($dbname, $shardId); + return $this->indexCache[$path]; + } + } + return $this->loadSegmentIndex($dbname, $shardId); + } + + /** + * Load a segment index: its checkpoint (or a v3.2 index) when it fits the data file, then the lines + * after it. A checkpoint that doesn't fit (the data changed behind it) rebuilds the index and the + * segment's secondary indexes under the exclusive lock; a missing one only replays the whole file. + * @return array|null + */ + private function loadSegmentIndex($dbname, $shardId){ + self::$indexStats['loads']++; + $dataPath = $this->segDataPath($dbname, $shardId); + $path = $dataPath . '.jidx'; + clearstatcache(true, $dataPath); + if(!is_file($dataPath)){ + if(file_exists($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + $this->markFileNotExists($dataPath); + return null; + } + $size = (int)filesize($dataPath); + $checkpoint = $this->atomicRead($path, null); + $index = null; + $oldMap = null; + if(is_array($checkpoint) && isset($checkpoint['p'], $checkpoint['l'], $checkpoint['s']) && is_array($checkpoint['p']) && is_array($checkpoint['l'])){ + if(!$this->checkpointFits($dataPath, (int)$checkpoint['s'], $size, $checkpoint['h'] ?? null)){ + return $this->repairSegmentIndex($dbname, $shardId, true); + } + $index = $checkpoint; + $index['ck'] = (int)$checkpoint['s']; + $index['tl'] = 0; + $index['e'] = isset($checkpoint['e']) && is_string($checkpoint['e']) ? $checkpoint['e'] : null; + } elseif(is_array($checkpoint) && isset($checkpoint['o']) && is_array($checkpoint['o'])){ + if(isset($checkpoint['s'])){ + // v3.2 index: used when it ends at a line end of this file with no complete line after it. + // A complete line after it: a v3.2 writer died between its append and its index write. + $covered = (int)$checkpoint['s']; + if($covered > $size || $this->segmentCrc($dataPath, $covered) === null + || ($covered < $size && $this->lastNewlineIn($dataPath, $covered, $size) >= 0)){ + return $this->repairSegmentIndex($dbname, $shardId, true); + } + $index = $this->indexFromV3($checkpoint); + } else { + $oldMap = $checkpoint; // before v3.2: no "s", rebuilt from the data below + } + } + if($index === null){ + if(!$this->isJsonlFormat($dataPath)){ + return $this->repairSegmentIndex($dbname, $shardId); // a pre-3.0 file: converted + } + $index = $this->newJsonlIndex(); + if(is_array($checkpoint)){ + $index['created'] = $checkpoint['created'] ?? $index['created']; + if($shardId === null && isset($checkpoint['n'])){ + $index['n'] = (int)$checkpoint['n']; // keys of deleted records are never handed out again + } + } + } + $this->replaySegment($dbname, $shardId, $index, $size, $oldMap); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + self::$cacheTouch[$path] = ++self::$cacheTick; + self::$cacheGrew = true; + if($index['ck'] < 0){ + // No checkpoint to continue (missing, empty): the next write session of this process writes one + self::$uncheckpointed[$this->getLockPath($dbname)][$shardId === null ? 'main' : $shardId] = $shardId; + } + $this->maybeReaderCheckpoint($dbname, $shardId); + return $this->indexCache[$path]; + } + + /** + * A v3 index (key => [offset, length]) as a loaded checkpoint + */ + private function indexFromV3(array $old){ + $index = $this->newJsonlIndex(); + foreach($old['o'] as $key => $location){ + if(is_array($location) && count($location) === 2){ + $index['p'][(int)$key] = (int)$location[0]; + $index['l'][(int)$key] = (int)$location[1]; + } + } + $index['created'] = $old['created'] ?? $index['created']; + $index['n'] = (int)($old['n'] ?? 0); + $index['d'] = (int)($old['d'] ?? 0); + $index['t'] = (int)($old['t'] ?? 0); + $index['s'] = (int)$old['s']; + $index['ck'] = (int)$old['s']; + $index['v3'] = true; + return $index; + } + + /** + * True when a checkpoint covering $covered bytes belongs to this data file: within it, at a line end, + * and the bytes before $covered have the checksum it recorded + */ + private function checkpointFits($dataPath, $covered, $size, $crc){ + if($covered < 0 || $covered > $size || !is_string($crc)){ + return false; + } + return $this->segmentCrc($dataPath, $covered) === $crc; + } + + /** + * crc32b of the up to 256 bytes before $offset of a data file; null when $offset is not a line end + */ + private function segmentCrc($dataPath, $offset){ + if($offset === 0){ + return hash('crc32b', ''); + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return null; + } + $start = max(0, $offset - 256); + fseek($fp, $start); + $bytes = (string)fread($fp, $offset - $start); + fclose($fp); + return strlen($bytes) === $offset - $start && substr($bytes, -1) === "\n" ? hash('crc32b', $bytes) : null; + } + + /** + * Apply the complete lines of bytes [s, $size) of a segment's data file to its index: the last line of + * a key wins, a tombstone deletes. An unterminated tail stays outside ("s" ends before it; the next + * writer quarantines and cuts it). A line that doesn't decode (glued to a torn tail by v3.2) is + * salvaged when a record or tombstone can be found at its end, and reported either way. + * @param array|null $oldMap an older index of the segment (any version), used to salvage glued lines + * @param bool $report Report salvaged and skipped lines (not for checkIntegrity()) + */ + private function replaySegment($dbname, $shardId, array &$index, $size, $oldMap = null, $report = true){ + $from = (int)$index['s']; + if($shardId !== null){ + $index['z'] = $this->shardSizeOf($dbname); + $index['sid'] = $shardId; + } + if($size <= $from){ + return; + } + $fp = @fopen($this->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return; + } + $base = $shardId === null ? 0 : $shardId * $this->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->shardSizeOf($dbname); + $p = $index['p']; + $l = $index['l']; + unset($index['p'], $index['l']); // sole owner: changed in place, not copied + $n = (int)$index['n']; + $d = (int)$index['d']; + $t = (int)$index['t']; + $lines = 0; + $oldEnds = null; + $events = []; + fseek($fp, $from); + $read = $from; + $offset = $from; // file offset of $buffer[0] + $buffer = ''; + while($read < $size){ + $chunk = fread($fp, min(1048576, $size - $read)); + if($chunk === false || $chunk === ''){ + break; + } + $read += strlen($chunk); + $buffer .= $chunk; + $pos = 0; + while(($newline = strpos($buffer, "\n", $pos)) !== false){ + $json = substr($buffer, $pos, $newline - $pos); + $lineOffset = $offset + $pos; + $entry = $this->classifyLine($json, $base, $limit); // a key of another shard is no line of this one + if($entry === null){ + if($oldEnds === null){ + $oldEnds = []; + foreach(($oldMap === null ? [] : $this->indexEntries($oldMap)) as $k => $location){ + if(is_array($location) && count($location) === 2){ + $oldEnds[(int)$location[0] + (int)$location[1]] = [(int)$k, (int)$location[0]]; + } + } + } + $entry = $this->salvageLine($json, $lineOffset, $oldEnds, $base, $limit); + $events[] = [ + 'action' => $entry === null ? 'skip-line' : 'salvaged', + 'seg' => $shardId === null ? 'm' : (string)$shardId, + 'off' => $lineOffset, 'len' => strlen($json) + 1, + 'sha256' => hash('sha256', $json . "\n"), + ] + ($entry === null ? [] : ['key' => $entry[1]]); + } + if($entry === null){ + $d++; + } else { + $key = $entry[1]; + if($entry[0] === 't'){ + if(isset($p[$key])){ + unset($p[$key], $l[$key]); + $d++; + } + $t++; + } else { + if(isset($p[$key])){ + $d++; + } + $p[$key] = $lineOffset + $entry[2]; + $l[$key] = $entry[3]; + } + if($key >= $base && $key - $base + 1 > $n){ + $n = $key - $base + 1; + } + } + $lines++; + $pos = $newline + 1; + } + $offset += $pos; + $buffer = (string)substr($buffer, $pos); + } + fclose($fp); + $index['p'] = $p; + $index['l'] = $l; + $index['n'] = $n; + $index['d'] = $d; + $index['t'] = $t; + $index['s'] = $offset; + $index['tl'] = (int)$index['tl'] + $lines; + self::$indexStats['replayedLines'] += $lines; + foreach($report ? $events : [] as $event){ + $event['id'] = $event['seg'] . '.0.' . $event['off'] . '.' . $event['len'] . '.' . $event['sha256']; + unset($event['sha256']); + $this->reportRecoveryEvent($dbname, $event); + } + } + + /** + * A recovery decision: logged at once under the exclusive lock, else kept in memory and logged by the + * next exclusive session of this process (readers never write); getRecoveryLog() shows both + */ + private function reportRecoveryEvent($dbname, array $event){ + $lock = $this->getLockPath($dbname); + if(self::$noWrites === 0 && isset(self::$lockRegistry[$lock]) && self::$lockRegistry[$lock]['mode'] === LOCK_EX){ + $this->logRecoveryEvent($dbname, $event, $event['id']); + } else { + self::$pendingRecoveryEvents[$lock][$event['id']] = ['t' => time()] + $event; + } + } + + /** + * A segment index changed by a write (exclusive lock): cached, and checkpointed when the session ends + */ + private function storeSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + self::$lockRegistry[$this->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + if(self::$faultHook !== null){ $this->fault('index:written', ['db' => $dbname, 'shard' => $shardId]); } + } + + /** + * True when the lines after a segment's checkpoint are many enough to write a new one + */ + private function needsCheckpoint(array $index){ + return (int)$index['ck'] < 0 || !empty($index['v3']) + || (int)$index['tl'] >= self::CHECKPOINT_LINES + intdiv(count($index['p']), 16) + || (int)$index['s'] - (int)$index['ck'] >= self::CHECKPOINT_BYTES; + } + + /** + * Write a segment's checkpoint (best effort: a failure only means a longer replay next time). An + * index without epoch gets one, which only the exclusive lock holder may assign. + * @return bool + */ + private function writeCheckpoint($dbname, $shardId, array &$index){ + if(self::$noWrites > 0){ + return false; + } + $lock = $this->getLockPath($dbname); + $exclusive = isset(self::$lockRegistry[$lock]) && self::$lockRegistry[$lock]['mode'] === LOCK_EX; + if($index['e'] === null){ + if(!$exclusive){ + return false; + } + $index['e'] = bin2hex(random_bytes(4)); + } + $dataPath = $this->segDataPath($dbname, $shardId); + $crc = $this->segmentCrc($dataPath, (int)$index['s']); + if($crc === null){ + return false; + } + $file = $index; + unset($file['ck'], $file['tl'], $file['v3']); + $file['v'] = 4; + $file['format'] = 'jsonl'; + $file['updated'] = time(); + $file['h'] = $crc; + if($shardId !== null){ + $file['z'] = $this->shardSizeOf($dbname); + $file['sid'] = $shardId; + } + $json = $this->encodeJson($file); + if($json === false || !$this->writeDerivedFile($dataPath . '.jidx', $json)){ + return false; + } + $index['ck'] = (int)$index['s']; + $index['tl'] = 0; + unset($index['v3']); + self::$indexStats['checkpoints']++; + $this->indexCache[$dataPath . '.jidx'] = $index; + $this->dataEpochs[$dataPath] = $index['e']; + if(self::$faultHook !== null){ $this->fault('checkpoint:written', ['db' => $dbname, 'shard' => $shardId]); } + return true; + } + + /** + * End of a successful write session: checkpoint the segments it changed that need one + */ + private function checkpointTouchedSegments($dbname, $lock){ + $touched = self::$lockRegistry[$lock]['touched'] ?? []; + $segments = $touched + (self::$uncheckpointed[$lock] ?? []); + unset(self::$uncheckpointed[$lock]); + foreach($segments as $shardId){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + if(!isset($this->indexCache[$path], $this->verifiedIndexes[$path]) || !$this->needsCheckpoint($this->indexCache[$path])){ + continue; + } + $index = $this->indexCache[$path]; + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + // a checkpoint is never needed for correctness + } + } + self::$lockRegistry[$lock]['touched'] = []; + foreach($touched as $shardId){ + try { + $this->checkpointSegmentSecondaries($dbname, $shardId); + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + } + } + } + + /** + * A reader that replayed a long tail writes the checkpoint itself (shared lock: the data can't change, + * the content is the same as a writer's; no generation bump). Only on top of an existing checkpoint with + * an epoch, and not inside a write session of this process (that one checkpoints at its end). + */ + private function maybeReaderCheckpoint($dbname, $shardId){ + $lock = $this->getLockPath($dbname); + if(self::$noWrites > 0 || !isset(self::$lockRegistry[$lock]) || self::$lockRegistry[$lock]['mode'] !== LOCK_SH){ + return; + } + $path = $this->getJsonlIndexPath($dbname, $shardId); + $index = $this->indexCache[$path]; + if($index['e'] === null || (int)$index['ck'] < 0 || !empty($index['v3']) || !$this->needsCheckpoint($index)){ + return; + } + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + } + } + + /** + * Remove bytes [$from, $to) at the end of a data or buffer file, keeping them in a quarantine file + * first (exclusive lock). Fails closed: if the bytes can't be stored and verified, nothing is cut + * and the write that needed the cut is refused. + * @throws \RuntimeException + */ + private function quarantineAndCut($dbname, $shardId, $path, $from, $to, $action = 'cut-tail'){ + $quarantine = $this->quarantineBytes($dbname, $shardId, $path, $from, $to); + if(!$this->fsTruncate($path, $from)){ + throw new \RuntimeException("noneDB: could not cut the torn end of a file of database '{$dbname}'"); + } + $this->logRecoveryEvent($dbname, ['action' => $action, 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']], $quarantine['id']); + return $quarantine; + } + + /** + * Copy bytes [$from, $to) of a file into {hash}.nonedb.q.{segment}.0.{offset}.{length}.{sha256}.bin + * (segment: m, a shard id, or b for a buffer file) + * and verify the copy. An existing file with the same name is reused only if its bytes match. + * @return array ['id' => string, 'file' => string, 'durable' => true|false|'unknown'] + * @throws \RuntimeException + */ + private function quarantineBytes($dbname, $shardId, $path, $from, $to){ + $length = $to - $from; + $seg = $shardId === null ? 'm' : (string)$shardId; + $src = @fopen($path, 'rb'); + if($src === false){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + $tmp = $this->dbDir . $this->hashDBName($dbname) . '.nonedb.q.tmp' . bin2hex(random_bytes(4)); + $this->noteWrite($tmp); + $out = null; + try { + $this->fault('quarantine:write', ['db' => $dbname, 'path' => $path]); + $out = @fopen($tmp, 'xb'); + if($out === false){ + throw new \RuntimeException("noneDB: could not create a quarantine file for database '{$dbname}'"); + } + $hash = hash_init('sha256'); + fseek($src, $from); + $left = $length; + while($left > 0){ + $chunk = (string)fread($src, min(65536, $left)); + if($chunk === ''){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + hash_update($hash, $chunk); + if(fwrite($out, $chunk) !== strlen($chunk)){ + throw new \RuntimeException("noneDB: could not write a quarantine file for database '{$dbname}'"); + } + $left -= strlen($chunk); + } + $sha = hash_final($hash); + fflush($out); + $fileSynced = $this->syncHandle($out); + fclose($out); + $out = null; + + $id = "{$seg}.0.{$from}.{$length}.{$sha}"; + $name = $this->hashDBName($dbname) . ".nonedb.q.{$id}.bin"; + $target = $this->dbDir . $name; + clearstatcache(true, $target); + if(file_exists($target)){ + @unlink($tmp); // the same event again (e.g. the cut didn't happen last time): keep the first copy + $fileSynced = null; + } elseif(!@rename($tmp, $target)){ + throw new \RuntimeException("noneDB: could not publish a quarantine file for database '{$dbname}'"); + } + $dirSynced = $this->syncDirectory($this->dbDir); + if(@filesize($target) !== $length || hash_file('sha256', $target) !== $sha){ + throw new \RuntimeException("noneDB: quarantine file {$name} doesn't match the bytes it should hold"); + } + if($fileSynced === false || $dirSynced === false){ + $durable = false; + } elseif($fileSynced === null && !function_exists('fsync')){ + $durable = false; + } elseif($dirSynced === null){ + $durable = 'unknown'; + } else { + $durable = true; + } + return ['id' => $id, 'file' => $name, 'durable' => $durable]; + } finally { + fclose($src); + if($out !== null && $out !== false){ + fclose($out); + } + if(is_file($tmp)){ + @unlink($tmp); + } + } + } + + /** + * fsync() a file handle where PHP and the stream support it (PHP >= 8.1, plain files). Never emits a + * warning: the result goes to the durability report. + * @return bool|null Null when it can't be done here + */ + private function syncHandle($fp, $path = null){ + if(!function_exists('fsync')){ + return null; + } + $meta = @stream_get_meta_data($fp); + if(!is_array($meta) || ($meta['wrapper_type'] ?? '') !== 'plainfile'){ + return null; + } + $ok = self::$syncFn !== null ? (bool)(self::$syncFn)($fp, 'file') : @fsync($fp); + $this->countSync('file', $ok, $path); + return $ok; + } + + /** + * fsync() a directory so a new, renamed or deleted file in it survives a power loss (not possible on + * Windows: NTFS journals its metadata) + * @return bool|null Null when it can't be done here + */ + private function syncDirectory($dir){ + if(!function_exists('fsync') || PHP_OS_FAMILY === 'Windows'){ + return null; + } + $fp = @fopen(rtrim($dir, '/\\') . '/', 'r'); + if($fp === false){ + $this->countSync('dir', false, $dir); + return false; + } + try { + $ok = self::$syncFn !== null ? (bool)(self::$syncFn)($fp, 'dir') : @fsync($fp); + $this->countSync('dir', $ok, $dir); + return $ok; + } finally { + fclose($fp); + } + } + + private function countSync($kind, $ok, $path){ + self::$syncStats[$kind]++; + if(!$ok){ + self::$syncStats['failed']++; + self::$lastSyncError = ['path' => $path === null ? null : basename((string)$path), 'op' => $kind, 'time' => time()]; + } + } + + /** + * Record a sync the current write needs for power-loss durability: true (done), false (failed), + * null (not done: fsync off or not available) + */ + private function noteSync($kind, $result){ + if(!$this->fsync){ + return; // nothing is synced: the report says so without tracking each write + } + foreach(self::$lockRegistry as $path => $entry){ + if($entry['mode'] === LOCK_EX){ + $current = self::$lockRegistry[$path]['durability'][$kind] ?? true; + self::$lockRegistry[$path]['durability'][$kind] = $current === false || $result === false ? false + : ($current === null || $result === null ? null : true); + } + } + } + + /** + * Sync a data/meta write when the fsync setting is on; records what the operation needed + * @return bool False when the sync failed + */ + private function syncForWrite($fp, $path, $kind = 'file'){ + if(!$this->fsync){ + $this->noteSync($kind, null); + return true; + } + $ok = $kind === 'dir' ? $this->syncDirectory($path) : $this->syncHandle($fp, $path); + $this->noteSync($kind, $ok); + return $ok !== false; + } + + /** + * rename() that retries where it can fail for a moment: on Windows while another process has the target + * open (5, 10, 20, 40, 80, 160 ms, within lockTimeout). The target is never deleted first. + */ + private function renameWithRetry($from, $to){ + $fn = self::$renameFn; + $retry = PHP_OS_FAMILY === 'Windows' || $fn !== null; + $started = microtime(true); + foreach([0, 5, 10, 20, 40, 80, 160] as $i => $delay){ + if($delay > 0){ + if(!$retry || ($this->lockTimeout > 0 && microtime(true) - $started + $delay / 1000 > $this->lockTimeout)){ + return false; + } + usleep($delay * 1000); + clearstatcache(true, $from); + if(!file_exists($from)){ + return false; + } + } + if($fn !== null ? $fn($from, $to) : @rename($from, $to)){ + return true; + } + } + return false; + } + + /** + * Durability of this setup and of the last write (v3.3.0): + * - capability: what PHP and the platform can do (fsync() needs PHP 8.1; directories can't be synced + * on Windows) + * - effective: what is attempted with the fsync setting ('none', 'file', 'file+dir') + * - lastOperationDurability: whether every sync the last write needed for power-loss durability + * succeeded (an earlier success never counts for a later write) + * - lastSyncError, syncs (counters) and notes + * Process crashes never need fsync: what was written survives them (see docs/CONFIGURATION.md). + * @return array + */ + public function getDurability(): array { + $file = function_exists('fsync'); + $dir = $file && $this->directorySyncWorks(); + $notes = []; + if(!$file){ + $notes[] = 'fsync() needs PHP 8.1: writes are not synced to disk, a power loss can lose recent writes'; + } + if(PHP_OS_FAMILY === 'Darwin'){ + $notes[] = 'macOS: fsync() does not flush the drive cache (F_FULLFSYNC is not available in PHP)'; + } + if(PHP_OS_FAMILY === 'Windows'){ + $notes[] = 'Windows: directories can not be synced; new and renamed files depend on NTFS'; + } + return [ + 'capability' => ['file' => $file, 'dir' => $dir], + 'effective' => !$this->fsync || !$file ? 'none' : ($dir ? 'file+dir' : 'file'), + 'lastOperationDurability' => self::$lastOperationDurability + ?? ['op' => null, 'required' => [], 'completed' => [], 'durable' => false, 'reason' => 'no write yet'], + 'lastSyncError' => self::$lastSyncError, + 'syncs' => self::$syncStats, + 'notes' => $notes, + ]; + } + + private function directorySyncWorks(){ + if(PHP_OS_FAMILY === 'Windows'){ + return false; + } + if(!array_key_exists($this->dbDir, self::$dirSyncWorks)){ + if(!is_dir($this->dbDir)){ + return false; // nothing to sync yet: checked again once the directory exists + } + $fp = @fopen($this->dbDir, 'r'); + self::$dirSyncWorks[$this->dbDir] = $fp !== false; + if($fp !== false){ + fclose($fp); + } + } + return self::$dirSyncWorks[$this->dbDir]; + } + + /** + * Summary of the syncs a finished write session needed (getDurability()) + */ + private function finishDurability($path, $dbname){ + $done = self::$lockRegistry[$path]['durability'] ?? []; + $required = array_values(array_intersect(['file', 'dir'], array_keys($done))); + $completed = array_values(array_filter($required, function($kind) use ($done){ return $done[$kind] === true; })); + if(!$this->fsync){ + $durable = false; + $reason = 'fsync is off'; + } elseif(!function_exists('fsync')){ + $durable = false; + $reason = 'fsync() needs PHP 8.1'; + } elseif(in_array(false, $done, true)){ + $durable = false; + $reason = 'a ' . implode('/', array_keys(array_filter($done, function($r){ return $r === false; }))) . ' sync failed'; + } elseif(array_key_exists('dir', $done) && PHP_OS_FAMILY === 'Windows'){ + $durable = 'unknown'; + $reason = 'directories can not be synced on Windows'; + } elseif(in_array(null, $done, true)){ + $durable = false; + $reason = 'a file could not be synced (not a plain file)'; + } else { + $durable = true; + $reason = null; + } + self::$lastOperationDurability = ['op' => "write to '{$dbname}'", 'required' => $required, 'completed' => $completed, + 'durable' => $durable, 'reason' => $reason]; + } + + /** + * Delete temp files ("*.tmp" + 8 hex digits) starting with $prefix: what a writer that died left + * behind (T2-12). Exclusive lock only: no other writer can be in the middle of an atomic write. + */ + private function removeStaleTemp($prefix){ + if(self::$noWrites > 0){ + return; + } + foreach($this->listFilesWithPrefix($prefix, '/\.tmp[0-9a-f]{8}$/') as $file){ + @unlink($file); + } + } + + /** + * Position of the last "\n" in bytes [$from, $to) of a file, -1 if there is none + */ + private function lastNewlineIn($path, $from, $to){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return -1; + } + try { + $end = $to; + while($end > $from){ + $start = max($from, $end - 65536); + fseek($fp, $start); + $chunk = (string)fread($fp, $end - $start); + $pos = strrpos($chunk, "\n"); + if($pos !== false){ + return $start + $pos; + } + $end = $start; + } + return -1; + } finally { + fclose($fp); + } + } + + /** + * Rebuild a segment index from its data file (exclusive lock, upgrading a shared one): a checkpoint + * that doesn't fit the data, a key the index points at that isn't there. When the rebuilt index + * differs from the old one (or there was none), the segment's field and spatial indexes are rebuilt as + * well: they may not know the records found in the data. Without $force (pre-3.0 file): converted, + * then loaded normally. + * @param bool $secondaries Also rebuild field/spatial indexes when needed (recovery does it itself) + * @return array|null + */ + private function repairSegmentIndex($dbname, $shardId, $force = false, $secondaries = true){ + return $this->withLock($dbname, true, function() use ($dbname, $shardId, $force, $secondaries){ + $dataPath = $this->segDataPath($dbname, $shardId); + $path = $dataPath . '.jidx'; + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + clearstatcache(true, $dataPath); + if(!file_exists($dataPath)){ + return null; + } + if(!is_file($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + // A pre-3.0 {"data": [...]} file (it has no index) is converted, never scanned as JSONL + $this->ensureJsonlFormat($dbname, $shardId); + if(!$force){ + $this->jsonlFormatCache[$dataPath] = true; // converted (or not convertible: read as JSONL) + return $this->loadSegmentIndex($dbname, $shardId); + } + $old = $this->atomicRead($path, null); + $index = $this->rebuildSegmentIndexFromData($dbname, $shardId, is_array($old) ? $old : null); + if($shardId !== null){ + $this->recountMeta($dbname); + } + if($secondaries && (!is_array($old) || $this->indexEntries($old) != $this->indexEntries($index))){ + $this->rebuildSecondaryIndexes($dbname, [$shardId]); + } + return $index; + }); + } + + /** + * key => [offset, length] of an index of any version (empty if it has no map) + */ + private function indexEntries(array $index){ + if(isset($index['p'], $index['l']) && is_array($index['p']) && is_array($index['l'])){ + $entries = []; + foreach($index['p'] as $key => $offset){ + $entries[(int)$key] = [(int)$offset, (int)($index['l'][$key] ?? -1)]; + } + } else { + $entries = isset($index['o']) && is_array($index['o']) ? $index['o'] : []; + } + ksort($entries, SORT_NUMERIC); + return $entries; + } + + /** + * Rebuild field and spatial indexes of some segments (and the global ones of a sharded database) + * from the data. Caller holds the exclusive lock. + */ + private function rebuildSecondaryIndexes($dbname, array $segments){ + $manifest = $this->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; + } + $known = $this->listSegments($dbname); + // One index that can't be built must never lock the whole database: it is dropped (queries scan, + // the next write or query builds it again) and logged + $guard = function(callable $build, $what, array $files) use ($dbname){ + try { + $build(); + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + foreach($files as $file){ + $this->fsUnlink($file); + unset($this->fieldIndexCache[$file], $this->spatialIndexCache[$file]); + } + $this->logRecoveryEvent($dbname, ['action' => 'index-dropped', 'index' => $what, 'error' => $e->getMessage()]); + } + }; + foreach($segments as $shardId){ + if(!in_array($shardId, $known, true)){ + continue; + } + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field, $shardId){ + $legacy = $this->getLegacyFieldIndexPath($dbname, $field, $shardId); + $this->fsUnlink($legacy); + unset($this->fieldIndexCache[$legacy]); + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + }, 'field ' . $field, [$this->getFieldIndexPath($dbname, $field, $shardId)]); + } + foreach($manifest['spatial'] as $field){ + $guard(function() use ($dbname, $field, $shardId, $manifest){ + $legacy = $this->getLegacySpatialIndexPath($dbname, $field, $shardId); + $this->fsUnlink($legacy); + unset($this->spatialIndexCache[$legacy]); + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, + isset($manifest['spatialOptions'][$field]['nodeSize']) ? ['nodeSize' => $manifest['spatialOptions'][$field]['nodeSize']] : []); + $this->writeSpatialIndex($dbname, $field, $tree, $shardId); + if($shardId !== null){ + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); + } + }, 'spatial ' . $field, [$this->getSpatialIndexPath($dbname, $field, $shardId)]); + } + unset($this->convertedSegments[$this->segDataPath($dbname, $shardId)]); + } + if($this->isSharded($dbname)){ + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field){ + $this->deleteLegacyGlobalFieldIndex($dbname, $field); + $this->rebuildGlobalFieldIndex($dbname, $field); + }, 'global field ' . $field, [$this->getGlobalFieldIndexPath($dbname, $field)]); + } + } + } + + /** + * Index of a segment a writer is about to change (exclusive lock), taken out of the cache: the + * writer changes it in place and stores it back (storeSegmentIndex). Checked against the data file + * when it was not yet known to match it: an unterminated tail (crashed append) is quarantined and cut + * off, so the next line can't be glued to it. A v3.2 index file is deleted before the first append + * (v3.2 would trust its stale key counter); a checkpoint replaces it when the session ends. A checked + * index stays checked until its cache is dropped; appendToFile() compares the size it finds with the + * index anyway (no extra system call), which catches anything else. + * @return array|null + */ + private function writableSegmentIndex($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null){ + return null; + } + $dataPath = $this->segDataPath($dbname, $shardId); + $path = $dataPath . '.jidx'; + if(!isset($this->writableIndexes[$path])){ + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + if($size < (int)$index['s']){ + $index = $this->repairSegmentIndex($dbname, $shardId, true); + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + } + if($size > (int)$index['s']){ + $this->quarantineAndCut($dbname, $shardId, $dataPath, (int)$index['s'], $size); + } + $this->writableIndexes[$path] = true; + } + if(!empty($index['v3'])){ + $this->fsUnlink($path); + unset($index['v3']); + $index['ck'] = -1; + $this->indexCache[$path] = $index; + } + unset($this->indexCache[$path]); // the caller is its only holder until it stores it back + return $index; + } + + /** + * A writer's index goes back to the cache unchanged (nothing to write after all) + */ + private function releaseSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + } + + /** + * Rebuild a segment index from the whole data file and write its checkpoint (exclusive lock): new + * segment epoch, as there is no checkpoint it continues + * @param array|null $old The index file found (created time, key counter, salvage of glued lines) + * @return array + */ + private function rebuildSegmentIndexFromData($dbname, $shardId, $old = null){ + $dataPath = $this->segDataPath($dbname, $shardId); + if($old === null){ + $old = $this->atomicRead($dataPath . '.jidx', null); + $old = is_array($old) ? $old : null; + } + $index = $this->newJsonlIndex(); + if($old !== null){ + $index['created'] = $old['created'] ?? $index['created']; + if($shardId === null && isset($old['n'])){ + $index['n'] = (int)$old['n']; + } + } + clearstatcache(true, $dataPath); + $this->replaySegment($dbname, $shardId, $index, (int)@filesize($dataPath), $old); // its entries only if a line is damaged + $index['e'] = bin2hex(random_bytes(4)); + $path = $dataPath . '.jidx'; + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->writeCheckpoint($dbname, $shardId, $index); + return $this->indexCache[$path]; + } + + /** + * Classify one complete data line: ['r', key, 0, length] (record), ['t', key] (tombstone), or null + * when it isn't a valid line of this segment + */ + private function classifyLine($json, $base, $limit){ + $record = json_decode($json, true); + if(!is_array($record)){ + return null; + } + if(count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $key = $record['__nonedb_deleted']; + return $key >= $base && ($limit === null || $key < $limit) ? ['t', $key] : null; + } + if(isset($record['key']) && is_int($record['key'])){ + $key = $record['key']; + return $key >= $base && ($limit === null || $key < $limit) ? ['r', $key, 0, strlen($json)] : null; + } + return null; + } + + /** + * Find a record or tombstone at the end of a line that doesn't decode (written by an older version + * right after a torn tail). First where the old index says a record ended; then candidates from the + * rightmost "{" leftwards: inside a record no "{" starts a JSON text that also ends the line, so the + * first one that decodes is where the record starts. + * @return array|null Like classifyLine(), with the record's offset inside the line + */ + private function salvageLine($json, $lineOffset, array $oldEnds, $base, $limit){ + $end = $lineOffset + strlen($json); + if(isset($oldEnds[$end]) && $oldEnds[$end][1] > $lineOffset){ + $start = $oldEnds[$end][1] - $lineOffset; + $entry = $this->classifySalvaged(substr($json, $start), $base, $limit); + if($entry !== null && $entry[0] === 'r' && $entry[1] === $oldEnds[$end][0]){ + return ['r', $entry[1], $start, strlen($json) - $start]; + } + } + $pos = strlen($json); + for($tries = 0; $tries < 4096; $tries++){ + $pos = $pos > 0 ? strrpos($json, '{', $pos - strlen($json) - 1) : false; + if($pos === false || $pos === 0){ + return null; + } + $entry = $this->classifySalvaged(substr($json, $pos), $base, $limit); + if($entry !== null){ + return $entry[0] === 't' ? $entry : ['r', $entry[1], $pos, strlen($json) - $pos]; + } + } + return null; + } + + /** + * classifyLine() for a salvage candidate: also requires an object whose last member is "key" + */ + private function classifySalvaged($candidate, $base, $limit){ + $entry = $this->classifyLine($candidate, $base, $limit); + if($entry === null || $entry[0] === 't'){ + return $entry; + } + $record = json_decode($candidate, true); + $keys = array_keys($record); + return end($keys) === 'key' && $keys !== range(0, count($keys) - 1) ? $entry : null; + } + + /** + * Recompute per-shard and total counters in the meta file from the shard indexes + */ + private function recountMeta($dbname){ + $meta = $this->getCachedMeta($dbname, true); + if($meta === null){ + return; + } + $size = (int)$meta['shardSize']; + $total = 0; + $nextKey = 0; + foreach($meta['shards'] as $i => $shard){ + $index = $this->readJsonlIndex($dbname, (int)$shard['id']); + $count = $index === null ? 0 : count($index['p']); + $meta['shards'][$i]['count'] = $count; + $total += $count; + if($index !== null){ + // n of a shard counts every key it ever had (deleted ones and the kept top tombstone too) + $nextKey = max($nextKey, empty($index['p']) ? 0 : max(array_keys($index['p'])) + 1); + if((int)$index['n'] > 0){ + $nextKey = max($nextKey, (int)$shard['id'] * $size + (int)$index['n']); + } + } + } + $meta['totalRecords'] = $total; + $meta['nextKey'] = max((int)($meta['nextKey'] ?? 0), $nextKey); + $this->writeMeta($dbname, $meta); + } + + /** + * Detect if a data file is in JSONL format (vs the pre-3.0 single JSON document {"data": [...]}) + * @param string $path + * @return bool True if JSONL format + */ + private function isJsonlFormat($path){ + if(isset($this->jsonlFormatCache[$path])){ + return $this->jsonlFormatCache[$path]; + } + if(!$this->cachedFileExists($path)){ + return true; + } + + // A pre-3.0 file is one JSON document {"data": [...]} without a line end; a JSONL line ends with + // "\n", and a record always has a "key" field, so a record whose first field is "data" is never + // mistaken. Only the first line is read: a missing .jidx is normal since v3.3 (checkpoints). + $isJsonl = true; + $fp = @fopen($path, 'rb'); + if($fp !== false){ + $first = fgets($fp); + fclose($fp); + if($first !== false && substr($first, -1) !== "\n"){ + $decoded = json_decode($first, true); + if(is_array($decoded) && array_keys($decoded) === ['data'] && is_array($decoded['data'])){ + $isJsonl = false; + } + } + } + + $this->jsonlFormatCache[$path] = $isJsonl; + return $isJsonl; + } + + /** + * Ensure a segment is in JSONL format (auto-migrates the pre-3.0 format) + * @param string $dbname + * @param int|null $shardId + * @return bool + */ + private function ensureJsonlFormat($dbname, $shardId = null){ + $path = $this->segDataPath($dbname, $shardId); + if($this->isJsonlFormat($path)){ + return true; + } + return $this->migrateToJsonl($path, $dbname, $shardId); + } + + /** + * Migrate a pre-3.0 {"data": [...]} file to JSONL (exclusive lock) + * @param string $path Source file path + * @param string $dbname Database name + * @param int|null $shardId Shard ID or null for non-sharded + * @return bool Success + */ + private function migrateToJsonl($path, $dbname, $shardId = null){ + return $this->withLock($dbname, true, function() use ($path, $dbname, $shardId){ + unset($this->jsonlFormatCache[$path]); + if($this->isJsonlFormat($path)){ + return true; // migrated by someone else meanwhile + } + + $data = json_decode((string)@file_get_contents($path), true); + if(!is_array($data) || !isset($data['data']) || !is_array($data['data'])){ + return false; + } + + $index = $this->newJsonlIndex(); + $base = ($shardId !== null) ? ($shardId * $this->shardSizeOf($dbname)) : 0; + $buffer = ''; + $offset = 0; + $slots = 0; + foreach($data['data'] as $localKey => $record){ + $slots = max($slots, (int)$localKey + 1); + if(!is_array($record)){ + continue; // deleted (null) or invalid slot: not stored, not garbage + } + $key = $base + (int)$localKey; + unset($record['key']); + $record['key'] = $key; + $json = $this->encodeJson($record); + if($json === false){ + continue; + } + $index['p'][$key] = $offset; + $index['l'][$key] = strlen($json); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + } + $index['n'] = $slots; + // Keep the highest slot ever used (see compactJsonl) + if($slots > 0 && empty($data['data'][$slots - 1]) && !is_array($data['data'][$slots - 1] ?? null)){ + $buffer .= '{"__nonedb_deleted":' . ($base + $slots - 1) . "}\n"; + $index['t'] = 1; + } + + $this->bumpEpoch($dbname); // the data file is replaced + $this->fsUnlink($path . '.jidx'); + if(!$this->writeFileAtomic($path, $buffer, true)){ + return false; + } + $this->jsonlFormatCache[$path] = true; + $index['s'] = strlen($buffer); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[$path . '.jidx'] = $index; + $this->verifiedIndexes[$path . '.jidx'] = true; + $this->writeCheckpoint($dbname, $shardId, $index); + return true; + }); + } + + // ========================================== + // RECORD I/O + // ========================================== + + /** + * Batch read multiple JSONL records (one open, 64KB buffered sequential reads) + * @param string $path File path + * @param array $positions [key => offset, ...] + * @param array $lengths [key => length, ...] + * @param bool $mismatch Set to true when a line doesn't hold the expected record + * @return array [key => record, ...] + */ + private function readJsonlRecordsBatch($path, $positions, $lengths, &$mismatch = false){ + $mismatch = false; + if(empty($positions)){ + return []; + } + + $handle = @fopen($path, 'rb'); + if($handle === false){ + $mismatch = true; + return []; + } + + $records = []; + $bufferSize = 65536; + $buffer = ''; + $bufferStart = -1; + $bufferEnd = -1; + + // Read in file order. Appends keep the index in that order; updated records moved to the end. + $inOrder = true; + $previous = -1; + foreach($positions as $position){ + if($position < $previous){ + $inOrder = false; + break; + } + $previous = $position; + } + if(!$inOrder){ + asort($positions, SORT_NUMERIC); + } + + foreach($positions as $key => $position){ + $offset = (int)$position; + $length = (int)($lengths[$key] ?? 0); + if($length <= 0){ + $mismatch = true; + continue; + } + + if($offset >= $bufferStart && ($offset + $length) <= $bufferEnd){ + $json = substr($buffer, $offset - $bufferStart, $length); + } else { + fseek($handle, $offset, SEEK_SET); + // One record: read just its bytes (T2-11); several: 64 KB at a time + $buffer = (string)fread($handle, count($positions) === 1 ? $length : max($length, $bufferSize)); + $bufferStart = $offset; + $bufferEnd = $offset + strlen($buffer); + $json = substr($buffer, 0, $length); + } + + $record = json_decode((string)$json, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ + $mismatch = true; + continue; + } + $records[$key] = $record; + } + + fclose($handle); + return $records; + } + + /** + * Read records of one segment, verified against their keys. A stale or corrupt index is + * rebuilt from the data file once. + * @param array|null $keys Keys to read (null = all) + * @return array [key => record] sorted by key + */ + private function readSegmentRecords($dbname, $shardId, $keys = null){ + self::$segmentReads++; + for($attempt = 0; $attempt < 2; $attempt++){ + $index = $attempt === 0 + ? $this->readJsonlIndex($dbname, $shardId) + : $this->repairSegmentIndex($dbname, $shardId, true); + if($index === null){ + return []; + } + if($keys === null){ + $positions = $index['p']; + } else { + $positions = []; + foreach($keys as $key){ + if(isset($index['p'][$key])){ + $positions[$key] = $index['p'][$key]; + } + } + } + $mismatch = false; + $records = $this->readJsonlRecordsBatch($this->segDataPath($dbname, $shardId), $positions, $index['l'], $mismatch); + if(!$mismatch){ + ksort($records, SORT_NUMERIC); + return $records; + } + } + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + + /** + * Append records ([key => record]) to a segment and point the index at them. + * Everything is encoded before anything is written, so an unencodable record writes nothing. + * @throws \InvalidArgumentException|\RuntimeException + */ + private function appendSegmentRecords($dbname, $shardId, array &$index, array $records){ + if(empty($records)){ + return; + } + $lines = $this->withExactFloats(function() use ($records){ + $lines = []; + foreach($records as $key => $record){ + unset($record['key']); + $record['key'] = (int)$key; + $lines[$key] = $this->encodeRecord($record); + } + return $lines; + }); + // After encoding: a record that can't be encoded leaves no journal entry behind + $this->markIndexesPending($dbname, $shardId); + $buffer = implode("\n", $lines) . "\n"; + $dataPath = $this->segDataPath($dbname, $shardId); + $offset = $this->appendToFile($dataPath, $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + $this->markFileExists($dataPath); + foreach($lines as $key => $json){ + if(isset($index['p'][$key])){ + $index['d']++; // the line it had before is garbage now + } + $index['p'][$key] = $offset; + $index['l'][$key] = strlen($json); + $offset += strlen($json) + 1; + } + $index['tl'] = (int)$index['tl'] + count($lines); + } + + /** + * Append tombstones for deleted keys, so an index rebuilt from the data won't resurrect them + */ + private function appendTombstones($dbname, $shardId, array &$index, array $keys){ + if(empty($keys)){ + return; + } + $this->markIndexesPending($dbname, $shardId); + $buffer = ''; + foreach($keys as $key){ + $buffer .= '{"__nonedb_deleted":' . (int)$key . "}\n"; + } + $offset = $this->appendToFile($this->segDataPath($dbname, $shardId), $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + foreach($keys as $key){ + if(isset($index['p'][$key])){ + unset($index['p'][$key], $index['l'][$key]); + $index['d']++; // the old record line + } + } + $index['t'] = (int)$index['t'] + count($keys); // tombstone lines + $index['tl'] = (int)$index['tl'] + count($keys); + } + + /** + * An append to a segment wrote nothing: throw. When the data file didn't have the size the index + * covers (it changed behind this process's back), the cached index is dropped as well. + * @throws \RuntimeException + */ + private function failedAppend($dbname, $shardId){ + if($this->appendSyncFailed){ + throw new \RuntimeException("noneDB: fsync of the data of database '{$dbname}' failed; nothing was written"); + } + if($this->appendMismatch){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + throw new \RuntimeException("noneDB: data file of database '{$dbname}' changed unexpectedly; nothing was written, its index is rebuilt on the next access"); + } + throw new \RuntimeException("noneDB: failed to write data of database '{$dbname}'"); + } + + /** + * Create an empty segment (data file + index) + * @param string $dbname + * @param int|null $shardId + * @return bool + */ + private function createJsonlDatabase($dbname, $shardId = null){ + $path = $this->segDataPath($dbname, $shardId); + clearstatcache(true, $path); + if(!file_exists($path) && !$this->fsTouch($path)){ + throw new \RuntimeException("noneDB: cannot create data file of database '{$dbname}'"); + } + $this->markFileExists($path); + $this->jsonlFormatCache[$path] = true; + $index = $this->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[$path . '.jidx'] = $index; + $this->verifiedIndexes[$path . '.jidx'] = true; + // No checkpoint yet (ck -1): the session writes one when it ends, with the first records (until then + // the new segment replays from 0, which is the same index) + self::$lockRegistry[$this->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + return true; + } + + // ========================================== + // WRITE PATHS (callers hold the exclusive lock) + // ========================================== + + /** + * Insert validated records. Assigns keys, writes data + indexes, updates field/spatial indexes + * and migrates to sharded format when the size threshold is reached. + * @param array $items Records without "key" + * @return array ['n' => int] or ['n' => 0, 'error' => string] + */ + private function insertRecords($dbname, array $items, $startKey = null){ + if(empty($items)){ + return ["n" => 0]; + } + if(!$this->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->createDBUnlocked($dbname); + } + + // With a start key the records get keys startKey, startKey+1, ... and keys that are already + // stored are skipped, so replaying the same insert after a crash never duplicates anything. + $keyed = null; + $endKey = null; + if($startKey !== null){ + $keyed = []; + foreach(array_values($items) as $i => $item){ + $keyed[(int)$startKey + $i] = $item; + } + $endKey = (int)$startKey + count($items); + foreach(array_keys($this->selectRecords($dbname, ['key' => array_keys($keyed)])) as $existing){ + unset($keyed[$existing]); + } + } + + if($this->isSharded($dbname)){ + $this->insertShardedRecords($dbname, $items, $keyed, $endKey); + return ["n" => count($items)]; + } + + $index = $this->writableSegmentIndex($dbname, null); + if($index === null){ + $this->createJsonlDatabase($dbname, null); + $index = $this->writableSegmentIndex($dbname, null); + } + + if($keyed !== null){ + $records = $keyed; + $index['n'] = max((int)$index['n'], $endKey); + } else { + $records = []; + foreach($items as $item){ + $records[(int)$index['n']] = $item; + $index['n']++; + } + } + $this->appendSegmentRecords($dbname, null, $index, $records); + $this->storeSegmentIndex($dbname, null, $index); + $this->afterRecordsChanged($dbname, null, [], $this->withKeys($records)); + + if($this->shardingEnabled && $this->autoMigrate && $index['n'] >= $this->shardSize){ + // Replaces the data file: after this write committed + $this->runIndependent($dbname, 'migrate', function() use ($dbname){ + return $this->migrateToSharded($dbname); + }); + } + return ["n" => count($items)]; + } + + /** + * Insert into a sharded database. Keys come from meta.nextKey; key K lives in shard intdiv(K, shardSize). + */ + private function insertShardedRecords($dbname, array $items, $keyed = null, $endKey = null){ + $meta = $this->getCachedMeta($dbname, true); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + $size = (int)$meta['shardSize']; + $nextKey = (int)$meta['nextKey']; + // The meta may lag behind the data (a meta update lost in a power loss): never hand out a key the + // last shard has already used + if(!empty($meta['shards'])){ + $last = max(array_map('intval', array_column($meta['shards'], 'id'))); + $lastIndex = $this->readJsonlIndex($dbname, $last); + if($lastIndex !== null){ + $nextKey = max($nextKey, $last * $size + (int)$lastIndex['n']); + } + } + + $byShard = []; + if($keyed !== null){ + foreach($keyed as $key => $item){ + $byShard[intdiv($key, $size)][$key] = $item; + } + $nextKey = max($nextKey, (int)$endKey); + $items = $keyed; + } else { + foreach($items as $item){ + $byShard[intdiv($nextKey, $size)][$nextKey] = $item; + $nextKey++; + } + } + + // Reserve the keys first: a crash later leaves a gap, never a reused key + $shardPos = []; + foreach($meta['shards'] as $i => $shard){ + $shardPos[(int)$shard['id']] = $i; + } + $this->setOrphanShardsAside($dbname, array_keys(array_diff_key($byShard, $shardPos))); + foreach($byShard as $shardId => $records){ + if(!isset($shardPos[$shardId])){ + $meta['shards'][] = ["id" => $shardId, "file" => "_s" . $shardId, "count" => 0, "deleted" => 0]; + $shardPos[$shardId] = count($meta['shards']) - 1; + } + $meta['shards'][$shardPos[$shardId]]['count'] += count($records); + } + // Journal first: if we die after reserving, recovery recounts the meta from the shards + foreach(array_keys($byShard) as $shardId){ + $this->markIndexesPending($dbname, $shardId); + } + $meta['nextKey'] = $nextKey; + $meta['totalRecords'] = (int)$meta['totalRecords'] + count($items); + $this->writeMeta($dbname, $meta); + + // A failure below is undone by the rollback scope (appends of every shard cut back, counters + // recounted); the reserved keys stay unused and the new shards stay listed (empty) + foreach($byShard as $shardId => $records){ + $index = $this->writableSegmentIndex($dbname, $shardId); + if($index === null){ + $this->createJsonlDatabase($dbname, $shardId); + $index = $this->writableSegmentIndex($dbname, $shardId); + } + $this->appendSegmentRecords($dbname, $shardId, $index, $records); + $index['n'] = max((int)$index['n'], max(array_keys($records)) - $shardId * $size + 1); + $this->storeSegmentIndex($dbname, $shardId, $index); + $this->afterRecordsChanged($dbname, $shardId, [], $this->withKeys($records)); + } + } + + /** + * Files of a shard the meta doesn't list (left by an interrupted dropDB of v3.2, or a meta update lost + * in a power loss) must not be taken over by a new shard with the same number: their records were + * deleted, or belong to keys the meta doesn't know any more. They are renamed to *.orphan.{time} + * (reported by getDB()), never deleted. + */ + private function setOrphanShardsAside($dbname, array $shardIds){ + if(empty($shardIds)){ + return; + } + // The directory is listed only for shards up to the highest one that had files at the last listing: + // above it only this process created shard files since (another process's write drops the number, + // invalidateDbCaches). A listing per new shard made inserts into many shards O(shards²) + $base = $this->dbBase($dbname); + $upTo = max(array_map('intval', $shardIds)); + if(isset($this->shardFilesUpTo[$base]) && min(array_map('intval', $shardIds)) > $this->shardFilesUpTo[$base]){ + $this->shardFilesUpTo[$base] = $upTo; + return; + } + $wanted = array_flip(array_map('intval', $shardIds)); + $files = []; + foreach([$base . '_s' => '/^\d+\.nonedb/', + $this->dbDir . $this->hashDBName($dbname) . '_s' => '/^\d+\.nonedb\.sidx\./'] as $prefix => $regex){ + foreach($this->listFilesWithPrefix($prefix, $regex) as $file){ + $shardId = (int)substr($file, strlen($prefix)); + $upTo = max($upTo, $shardId); + if(isset($wanted[$shardId])){ + $files[] = $file; + } + } + } + $stamp = '.orphan.' . date('YmdHis'); + $moved = false; + foreach($files as $file){ + if(preg_match('/\.orphan\.\d+$/', $file)){ + continue; + } + if(!$moved){ + $this->bumpEpoch($dbname); // a shard file of this name goes away: no cache of it is kept + $moved = true; + } + if(!$this->fsRename($file, $file . $stamp)){ + throw new \RuntimeException("noneDB: could not move aside the leftover file " . basename($file) . " of database '{$dbname}'"); + } + unset($this->indexCache[$file], $this->fieldIndexCache[$file], $this->spatialIndexCache[$file], $this->jsonlFormatCache[$file]); + $this->markFileNotExists($file); + } + $this->shardFilesUpTo[$base] = $upTo; // after bumpEpoch(), which drops it + } + + /** + * Records with their "key" field set + */ + private function withKeys(array $records){ + foreach($records as $key => $record){ + unset($records[$key]['key']); + $records[$key]['key'] = (int)$key; + } + return $records; + } + + /** + * Replace records ([key => full new record]) in one segment + * @return int Number of records written + */ + private function updateSegmentRecords($dbname, $shardId, array $oldRecords, array $newRecords){ + if(empty($newRecords)){ + return 0; + } + $index = $this->writableSegmentIndex($dbname, $shardId); + foreach(array_keys($newRecords) as $key){ + if(!isset($index['p'][$key])){ + unset($newRecords[$key], $oldRecords[$key]); + } + } + if(empty($newRecords)){ + $this->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $this->appendSegmentRecords($dbname, $shardId, $index, $newRecords); // counts the replaced lines + $this->storeSegmentIndex($dbname, $shardId, $index); + $this->afterRecordsChanged($dbname, $shardId, $oldRecords, $this->withKeys($newRecords)); + $this->deferCompaction($dbname, $shardId); + return count($newRecords); + } + + /** + * Delete records ([key => record]) from one segment + * @return int Number of records deleted + */ + private function deleteSegmentRecords($dbname, $shardId, array $oldRecords){ + if(empty($oldRecords)){ + return 0; + } + $index = $this->writableSegmentIndex($dbname, $shardId); + foreach(array_keys($oldRecords) as $key){ + if(!isset($index['p'][$key])){ + unset($oldRecords[$key]); + } + } + if(empty($oldRecords)){ + $this->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $keys = array_keys($oldRecords); + $this->appendTombstones($dbname, $shardId, $index, $keys); // drops the keys, counts the lines + $this->storeSegmentIndex($dbname, $shardId, $index); + $this->afterRecordsChanged($dbname, $shardId, $oldRecords, []); + + if($shardId !== null){ + $meta = $this->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + if((int)$shard['id'] === $shardId){ + $meta['shards'][$i]['count'] = max(0, $meta['shards'][$i]['count'] - count($keys)); + $meta['shards'][$i]['deleted'] += count($keys); + } + } + $meta['totalRecords'] = max(0, (int)$meta['totalRecords'] - count($keys)); + $meta['deletedCount'] = (int)($meta['deletedCount'] ?? 0) + count($keys); + $this->writeMeta($dbname, $meta); + } + + $this->deferCompaction($dbname, $shardId); + return count($keys); + } + + /** + * Records of a segment changed. Field and spatial indexes follow the data by themselves (they replay + * the data lines after their checkpoint when used, so a value given as an object is indexed as it is + * stored): nothing to do here. + */ + private function afterRecordsChanged($dbname, $shardId, array $removed, array $added){ + } + + private static function containsObject(array $value){ + foreach($value as $item){ + if(is_object($item) || (is_array($item) && self::containsObject($item))){ + return true; + } + } + return false; + } + + /** + * Compaction replaces the data file, so it runs after the write committed (see the rollback scope) + */ + private function deferCompaction($dbname, $shardId){ + $this->runIndependent($dbname, 'compact:' . ($shardId === null ? 'main' : $shardId), function() use ($dbname, $shardId){ + $this->maybeCompact($dbname, $shardId); + }); + } + + /** + * Compact a segment once garbage exceeds the threshold (relative to live records) + */ + private function maybeCompact($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null){ + return; + } + $garbage = (int)$index['d']; + if($garbage > 0 && $garbage > (count($index['p']) + $garbage) * $this->jsonlGarbageThreshold){ + $this->compactJsonl($dbname, $shardId); + } + } + + /** + * Compact a segment: rewrite live records (keys and order preserved), drop old versions and + * tombstones. Crash-safe: the data file is replaced first; if the index rename doesn't happen, + * the shorter data file is detected and the index rebuilt from it. + * @param string $dbname + * @param int|null $shardId + * @return array ['compacted' => int, 'freed' => int] + */ + private function compactJsonl($dbname, $shardId = null){ + return $this->withLock($dbname, true, function() use ($dbname, $shardId){ + $this->refuseUncommittedFile($this->segDataPath($dbname, $shardId), 'compaction'); + $this->removeStaleTemp($this->segDataPath($dbname, $shardId) . '.'); + $plan = null; + for($attempt = 0; $plan === null; $attempt++){ + if($attempt > 1){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + // A record the index points at must verify; if not, rebuild the index once and plan again + $index = $attempt === 0 ? $this->writableSegmentIndex($dbname, $shardId) : $this->repairSegmentIndex($dbname, $shardId, true); + if($index === null){ + return ['compacted' => 0, 'freed' => 0]; + } + $plan = $this->planCompaction($dbname, $shardId, $index); + } + $path = $this->segDataPath($dbname, $shardId); + $live = $plan['live']; + ksort($live, SORT_NUMERIC); + + $newIndex = $this->newJsonlIndex(); + $newIndex['created'] = $index['created'] ?? time(); + $newIndex['n'] = $index['n']; + $buffer = ''; + $offset = 0; + $maxKey = -1; + foreach($live as $key => $json){ + $newIndex['p'][$key] = $offset; + $newIndex['l'][$key] = strlen($json); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + $maxKey = $key; + } + // Keep the highest key ever used: an index rebuilt from this file must never hand it out again + $highest = ($shardId === null ? 0 : $shardId * $this->shardSizeOf($dbname)) + (int)$index['n'] - 1; + if($maxKey < $highest){ + $buffer .= '{"__nonedb_deleted":' . $highest . "}\n"; + $newIndex['t'] = 1; + } + // Lines that don't decode are dropped from the data file: their bytes go to quarantine first + $events = []; + foreach($plan['bad'] as $region){ + $quarantine = $this->quarantineBytes($dbname, $shardId, $path, $region[0], $region[0] + $region[1]); + $events[] = ['action' => 'drop-line', 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]; + } + + // The checkpoint goes first: it must never be taken for the new data file + $this->bumpEpoch($dbname); + $this->fsUnlink($path . '.jidx'); + $this->fault('compact:replace', ['db' => $dbname, 'shard' => $shardId]); + if(!$this->writeFileAtomic($path, $buffer, true)){ + throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data"); + } + $this->fault('compact:replaced', ['db' => $dbname, 'shard' => $shardId]); + $newIndex['s'] = strlen($buffer); + $newIndex['e'] = bin2hex(random_bytes(4)); + $this->indexCache[$path . '.jidx'] = $newIndex; + $this->verifiedIndexes[$path . '.jidx'] = true; + $this->writeCheckpoint($dbname, $shardId, $newIndex); + foreach($events as $event){ + $this->logRecoveryEvent($dbname, $event); + } + return ['compacted' => count($live), 'freed' => (int)$index['d']]; + }); + } + + /** + * Read a segment line by line for compaction: the JSON of every live record (verified against its + * key) and the byte ranges of lines that are neither records nor tombstones + * @return array|null ['live' => [key => json], 'bad' => [[offset, length], ...]]; null if the index + * doesn't match the data + */ + private function planCompaction($dbname, $shardId, array $index){ + $offsets = $index['p']; + asort($offsets, SORT_NUMERIC); // offsets are distinct: file order (a sort without a callback) + $entries = []; + foreach($offsets as $key => $offset){ + $entries[] = [(int)$offset, (int)$key, (int)$index['l'][$key]]; + } + $count = count($entries); + // Keys of this segment: a line with another shard's key is damaged, never a superseded record + $size = $shardId === null ? null : $this->shardSizeOf($dbname); + $base = $shardId === null ? 0 : $shardId * $size; + $limit = $shardId === null ? null : $base + $size; + $fp = @fopen($this->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return $count === 0 ? ['live' => [], 'bad' => []] : null; + } + $live = []; + $bad = []; + $next = 0; + $pos = 0; + try { + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + $end = $pos + $length; + $owned = []; + while($next < $count && $entries[$next][0] < $end){ + $owned[] = $entries[$next++]; + } + if(substr($line, -1) !== "\n"){ + if(!empty($owned)){ + return null; + } + $bad[] = [$pos, $length]; // unterminated tail + break; + } + $json = substr($line, 0, -1); + $whole = count($owned) === 1 && $owned[0][0] === $pos && $owned[0][2] === strlen($json); + foreach($owned as $entry){ + $sub = $whole ? $json : substr($line, $entry[0] - $pos, $entry[2]); + $record = json_decode($sub, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== $entry[1]){ + return null; + } + $live[$entry[1]] = $sub; + } + if(!$whole){ + if(!empty($owned)){ + $bad[] = [$pos, $length]; // a salvaged record inside other bytes: keep the line + } elseif($this->classifyLine($json, $base, $limit) === null){ + $bad[] = [$pos, $length]; // neither a record nor a tombstone of this segment + } + } + $pos = $end; + } + } finally { + fclose($fp); + } + return $next < $count ? null : ['live' => $live, 'bad' => $bad]; + } + + /** + * {db}.nonedb.migrating: written before a migration writes its first shard, removed with the old file. + * While the database isn't sharded yet, the non-orphan files of shards 0..shards-1 are that migration's + * own (shard files that were there before were set aside first). + */ + private function getMigrationMarkerPath($dbname){ + return $this->dbBase($dbname) . '.nonedb.migrating'; + } + + /** + * Remove what an interrupted migration left (exclusive lock): its shard files while the database is + * still the old file, or only the marker once the meta says the database is sharded + */ + private function discardInterruptedMigration($dbname){ + $marker = $this->getMigrationMarkerPath($dbname); + clearstatcache(true, $marker); + if(!file_exists($marker)){ + return; + } + $removed = false; + if(!$this->isSharded($dbname)){ + $info = json_decode((string)@file_get_contents($marker), true); + $shards = is_array($info) && isset($info['shards']) && is_int($info['shards']) ? $info['shards'] : 0; + for($shardId = 0; $shardId < $shards; $shardId++){ + $files = array_merge( + $this->listFilesWithPrefix($this->dbBase($dbname) . '_s' . $shardId . '.nonedb'), + $this->listFilesWithPrefix($this->dbDir . $this->hashDBName($dbname) . '_s' . $shardId . '.nonedb.sidx.') + ); + foreach($files as $file){ + if(preg_match('/\.orphan\.\d+$/', $file)){ + continue; + } + if(!$removed){ + $this->bumpEpoch($dbname); + $removed = true; + } + if(!$this->fsUnlink($file) && file_exists($file)){ + throw new \RuntimeException("noneDB: could not remove " . basename($file) . " left by an interrupted migration of database '{$dbname}'"); + } + unset($this->indexCache[$file], $this->fieldIndexCache[$file], $this->spatialIndexCache[$file], $this->jsonlFormatCache[$file]); + $this->markFileNotExists($file); + } + } + } + $this->fsUnlink($marker); + } + + /** + * Convert a non-sharded database to sharded format (keys preserved, indexes rebuilt per shard) + * @param string $dbname + * @return bool + */ + private function migrateToSharded($dbname){ + return $this->withLock($dbname, true, function() use ($dbname){ + if($this->isSharded($dbname)){ + return true; + } + $legacyPath = $this->getDataPath($dbname); + if(!$this->cachedFileExists($legacyPath)){ + return false; + } + $this->refuseUncommittedFile($legacyPath, 'migration'); + $this->flushPendingBuffers($dbname); + + $index = $this->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + $records = $this->readSegmentRecords($dbname, null); + // Journal first, also without indexes: a migration that dies after the meta was written leaves + // the old file, which recovery then removes (A-6) + $this->markIndexesPending($dbname, null, true); + // Shards an interrupted migration of this database wrote are copies of the old file: removed + $this->discardInterruptedMigration($dbname); + $size = $this->shardSize; + $maxKey = empty($records) ? -1 : max(array_keys($records)); + $nextKey = max((int)$index['n'], $maxKey + 1); + $numShards = max(1, intdiv(max($nextKey, 1) - 1, $size) + 1); + + // Shard files that are already there belong to an earlier sharded life of this name (a dropped + // database, a lost meta): set aside before the marker says which shard files are this migration's + $this->setOrphanShardsAside($dbname, range(0, $numShards - 1)); + $marker = $this->getMigrationMarkerPath($dbname); + if(!$this->writeFileAtomic($marker, json_encode(['shards' => $numShards]), true)){ + throw new \RuntimeException("noneDB: migration of database '{$dbname}' could not start (marker file)"); + } + $this->fault('migrate:marker-written', ['db' => $dbname]); + + $byShard = array_fill(0, $numShards, []); + foreach($records as $key => $record){ + $byShard[intdiv($key, $size)][$key] = $record; + } + + $meta = [ + "version" => 1, + "shardSize" => $size, + "totalRecords" => count($records), + "deletedCount" => 0, + "nextKey" => $nextKey, + "shards" => [] + ]; + + foreach($byShard as $shardId => $shardRecords){ + $slots = min($nextKey, ($shardId + 1) * $size) - $shardId * $size; + $shardIndex = $this->newJsonlIndex(); + $shardIndex['n'] = max(0, $slots); + $buffer = $this->withExactFloats(function() use ($shardRecords, &$shardIndex){ + $buffer = ''; + $offset = 0; + foreach($shardRecords as $key => $record){ + $json = $this->encodeRecord($record); + $shardIndex['p'][$key] = $offset; + $shardIndex['l'][$key] = strlen($json); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + } + return $buffer; + }); + if($shardId === $numShards - 1 && $nextKey - 1 > $maxKey){ + // Keep the highest key ever used, as compaction does: a key index rebuilt from these + // files after the meta was lost must never hand it out again + $buffer .= '{"__nonedb_deleted":' . ($nextKey - 1) . "}\n"; + $shardIndex['t'] = 1; + } + $shardPath = $this->getShardPath($dbname, $shardId); + if(!$this->writeFileAtomic($shardPath, $buffer, true)){ + throw new \RuntimeException("noneDB: migration of database '{$dbname}' failed while writing shard {$shardId}"); + } + $this->markFileExists($shardPath); + $this->jsonlFormatCache[$shardPath] = true; + $shardIndex['s'] = strlen($buffer); + $shardIndex['e'] = bin2hex(random_bytes(4)); + $this->indexCache[$shardPath . '.jidx'] = $shardIndex; + $this->verifiedIndexes[$shardPath . '.jidx'] = true; + $this->writeCheckpoint($dbname, $shardId, $shardIndex); + $this->fault('migrate:shard-written', ['db' => $dbname, 'shard' => $shardId]); + + $meta['shards'][] = [ + "id" => $shardId, + "file" => "_s" . $shardId, + "count" => count($shardRecords), + "deleted" => 0 + ]; + } + + // Flip to sharded only after every shard is safely on disk + $this->writeMeta($dbname, $meta, true); + $this->fault('migrate:meta-written', ['db' => $dbname]); + + // Verify, then remove the non-sharded files + $verified = 0; + foreach(array_keys($byShard) as $shardId){ + $verified += count($this->readSegmentRecords($dbname, $shardId)); + } + if($verified !== count($records)){ + throw new \RuntimeException("noneDB: migration of database '{$dbname}' could not be verified"); + } + $this->bumpEpoch($dbname); // the old data file goes away + $this->fsUnlink($legacyPath . '.jidx'); + $this->fault('migrate:legacy-index-deleted', ['db' => $dbname]); + $this->fsUnlink($legacyPath); + $this->fsUnlink($legacyPath . '.backup'); + $this->markFileNotExists($legacyPath); + unset($this->indexCache[$legacyPath . '.jidx'], $this->jsonlFormatCache[$legacyPath]); + $this->fsUnlink($marker); + + // Indexes: rebuilt per shard from the manifest + $this->rebuildIndexesAfterMigration($dbname); + return true; + }); + } + + // ========================================== + // WRITE BUFFER + // ========================================== + // Optional (off by default). Buffered records have no key yet; flushing renames the buffer + // first and then inserts its records through insertRecords(), so the normal indexes and + // sharding logic apply. Pending buffers (also those left by pre-3.2 versions) are flushed + // before every read and write, whether or not buffering is currently enabled. + + /** + * Get buffer file path of a database + * @param string $dbname + * @return string + */ + private function getBufferPath($dbname){ + return $this->dbBase($dbname) . ".nonedb.buffer"; + } + + /** + * Check if buffer exists and has content + * @param string $bufferPath + * @return bool + */ + private function hasBuffer($bufferPath){ + clearstatcache(true, $bufferPath); + return file_exists($bufferPath) && filesize($bufferPath) > 0; + } + + /** + * Number of records in a buffer file; counted once, then tracked by size + */ + private function getBufferRecordCount($bufferPath){ + clearstatcache(true, $bufferPath); + if(!file_exists($bufferPath)){ + return 0; + } + $size = filesize($bufferPath); + if(isset(self::$bufferCounts[$bufferPath]) && self::$bufferCounts[$bufferPath][0] === $size){ + return self::$bufferCounts[$bufferPath][1]; + } + $count = count($this->readBufferRecords($bufferPath)); + self::$bufferCounts[$bufferPath] = [$size, $count]; + return $count; + } + + /** + * Read all records from a buffer file (JSONL). An unterminated last line is not a record (its + * append never completed). + * @param string $bufferPath + * @param bool $clean Set to false when the file holds anything that isn't a complete record line + * @return array + */ + private function readBufferRecords($bufferPath, &$clean = true){ + $clean = true; + $records = []; + $fp = @fopen($bufferPath, 'rb'); + if($fp === false){ + return $records; + } + while(($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $clean = false; + break; + } + $line = trim($line); + if($line === ''){ + continue; + } + $record = json_decode($line, true); + if(is_array($record)){ + $records[] = $record; + } else { + $clean = false; + } + } + fclose($fp); + return $records; + } + + /** + * Before appending to a buffer: a torn last line (crashed append) goes to quarantine and is cut off, + * so the next record isn't glued to it. Skipped when the file has the size this process wrote last. + */ + private function repairBufferTail($dbname, $bufferPath){ + clearstatcache(true, $bufferPath); + $size = @filesize($bufferPath); + if(!$size || (self::$bufferEnds[$bufferPath] ?? null) === $size){ + return; + } + $last = $this->lastNewlineIn($bufferPath, max(0, $size - 65536), $size); + if($last !== $size - 1){ + if($last < 0){ + $last = $this->lastNewlineIn($bufferPath, 0, $size); + } + $this->quarantineAndCut($dbname, 'b', $bufferPath, $last + 1, $size); + unset(self::$bufferCounts[$bufferPath]); + } + } + + /** + * Before a flushed buffer file is deleted: anything in it that wasn't a record is kept in quarantine + */ + private function quarantineBufferRest($dbname, $file){ + clearstatcache(true, $file); + $size = (int)@filesize($file); + if($size > 0){ + $quarantine = $this->quarantineBytes($dbname, 'b', $file, 0, $size); + $this->logRecoveryEvent($dbname, ['action' => 'drop-buffer', 'id' => $quarantine['id'], 'file' => basename($file), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]); + } + } + + /** + * True if the database has buffered records waiting (current or pre-3.2 layout) + */ + private function hasPendingBuffers($dbname){ + // Under the lock the state record says it (flag 'b', set before a buffer gets records and + // cleared after they are flushed), so normal operations don't touch the file system for this + if($this->isVerified($dbname)){ + return !empty(self::$lockRegistry[$this->getLockPath($dbname)]['flags']['b']); + } + return $this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)) || !empty($this->legacyShardBufferFiles($dbname)); + } + + /** + * Buffer files whose flush was interrupted: {buffer}.flushing.{firstKey} (and the pre-3.2 {buffer}.flushing) + */ + private function flushingFiles($dbname){ + return $this->listFilesWithPrefix($this->getBufferPath($dbname) . '.flushing', '/^(\.\d+)?$/'); + } + + /** + * Pre-3.2 per-shard buffer files (and interrupted flushes) of a database + */ + private function legacyShardBufferFiles($dbname){ + if(!$this->isSharded($dbname)){ + return []; + } + return $this->listFilesWithPrefix($this->dbBase($dbname) . '_s', '/^\d+\.nonedb\.buffer(\.flushing(\.\d+)?)?$/'); + } + + /** + * Flush buffered records into the database (exclusive lock) + * @return int Number of records flushed + */ + private function flushPendingBuffers($dbname){ + $base = $this->dbBase($dbname); + // Flushing inserts records, which can migrate to shards, which flushes again: don't re-enter + if(isset(self::$flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + return (int)$this->withLock($dbname, true, function() use ($dbname, $base){ + // In place before the caller appended anything, else after the caller's write committed + return $this->runIndependent($dbname, 'flush', function() use ($dbname, $base){ + if(isset(self::$flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + self::$flushingNow[$base] = true; + try { + return $this->flushPendingBuffersLocked($dbname); + } finally { + unset(self::$flushingNow[$base]); + } + }); + }); + } + + private function flushPendingBuffersLocked($dbname){ + { + $flushed = 0; + foreach($this->legacyShardBufferFiles($dbname) as $file){ + $flushed += $this->flushLegacyShardBuffer($dbname, $file); + } + + $bufferPath = $this->getBufferPath($dbname); + // Interrupted flushes first. A flush renames the buffer to {buffer}.flushing.{firstKey}: the + // rename records atomically which keys its records get, so replaying it is idempotent. + foreach($this->flushingFiles($dbname) as $file){ + $flushed += $this->flushBufferFile($dbname, $file, preg_match('/\.flushing\.(\d+)$/', $file, $m) ? (int)$m[1] : null); + } + if($this->hasBuffer($bufferPath)){ + if(!$this->dbExists($dbname)){ + $this->createDBUnlocked($dbname); + } + $startKey = $this->nextKeyOf($dbname); + $flushingPath = $bufferPath . '.flushing.' . $startKey; + if(!$this->renameWithRetry($bufferPath, $flushingPath)){ + throw new \RuntimeException("noneDB: could not flush the write buffer of database '{$dbname}'"); + } + // The rename must survive a power loss before records get keys (else replayed twice) + $this->syncForWrite(null, $this->dbDir, 'dir'); + $flushed += $this->flushBufferFile($dbname, $flushingPath, $startKey); + } + unset(self::$bufferCounts[$bufferPath]); + $this->setBufferFlag($dbname, false); + $this->bufferLastFlush[$dbname] = time(); + return $flushed; + } + } + + /** + * Insert the records of one buffer file, starting at $startKey (null: pre-3.2 file, next free keys) + * @return int + */ + private function flushBufferFile($dbname, $file, $startKey){ + $records = $this->readBufferRecords($file, $clean); + if(!empty($records)){ + $result = $this->insertRecords($dbname, $records, $startKey); + if(isset($result['error'])){ + throw new \RuntimeException("noneDB: flushing the write buffer of '{$dbname}' failed: " . $result['error']); + } + } + $this->commitIndependentStep($dbname); // the records are stored for good before their buffer goes + if(!$clean){ + $this->quarantineBufferRest($dbname, $file); + } + $this->fsUnlink($file); + return count($records); + } + + /** + * The key the next inserted record gets + */ + private function nextKeyOf($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->getCachedMeta($dbname, true); + return $meta === null ? 0 : (int)$meta['nextKey']; + } + $index = $this->writableSegmentIndex($dbname, null); + return $index === null ? 0 : (int)$index['n']; + } + /** + * Flush a pre-3.2 shard buffer. Its records were already counted in the meta (count, totalRecords + * and nextKey) when they were buffered; they take the next free slots of their shard. + * @return int + */ + private function flushLegacyShardBuffer($dbname, $file){ + if(!preg_match('/_s(\d+)\.nonedb\.buffer(\.flushing(\.\d+)?)?$/', $file, $m)){ + return 0; + } + $shardId = (int)$m[1]; + $size = $this->shardSizeOf($dbname); + $index = $this->writableSegmentIndex($dbname, $shardId); + if($index === null){ + $this->createJsonlDatabase($dbname, $shardId); + $index = $this->writableSegmentIndex($dbname, $shardId); + } + // Same idempotent scheme as the current buffer: the first key goes into the file name + if(preg_match('/\.flushing\.(\d+)$/', $file, $km)){ + $startKey = (int)$km[1]; + } else { + $startKey = $shardId * $size + (int)$index['n']; + $renamed = preg_replace('/(\.flushing)?$/', '', $file, 1) . '.flushing.' . $startKey; + if(!$this->fsRename($file, $renamed)){ + throw new \RuntimeException("noneDB: could not flush a write buffer of database '{$dbname}'"); + } + $file = $renamed; + } + $records = array_values(array_filter($this->readBufferRecords($file, $clean), function($r) use (&$clean){ + if(array_key_exists('key', $r)){ + $clean = false; + return false; + } + return true; + })); + if(!empty($records)){ + $keyed = []; + foreach($records as $i => $record){ + $keyed[$startKey + $i] = $record; + } + foreach(array_keys($keyed) as $key){ + if(isset($index['p'][$key])){ + unset($keyed[$key]); // stored before an interruption + } + } + $this->appendSegmentRecords($dbname, $shardId, $index, $keyed); + $index['n'] = max((int)$index['n'], $startKey + count($records) - $shardId * $size); + $this->storeSegmentIndex($dbname, $shardId, $index); + $this->afterRecordsChanged($dbname, $shardId, [], $this->withKeys($keyed)); + $this->recountMeta($dbname); + } else { + $this->releaseSegmentIndex($dbname, $shardId, $index); + } + $this->commitIndependentStep($dbname); // the records are stored for good before their buffer goes + if(!$clean){ + $this->quarantineBufferRest($dbname, $file); + } + $this->fsUnlink($file); + return count($records); + } + + /** + * Append validated records to the database's write buffer (exclusive lock) + * @return array + */ + private function insertBuffered($dbname, array $validItems){ + if(!$this->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->createDBUnlocked($dbname); + } + $this->registerShutdownHandler(); + + $lines = $this->withExactFloats(function() use ($validItems){ + $lines = ''; + foreach($validItems as $item){ + $lines .= $this->encodeRecord($item) . "\n"; + } + return $lines; + }); + + $bufferPath = $this->getBufferPath($dbname); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); // one check for the flush limits, the torn tail and the count + $known = isset(self::$bufferCounts[$bufferPath]) && self::$bufferCounts[$bufferPath][0] === $size ? self::$bufferCounts[$bufferPath][1] : null; + if($size > 0 && $this->shouldFlushBuffer($bufferPath, $dbname, $size, $known)){ + $this->flushPendingBuffers($dbname); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + if($size > 0 && (self::$bufferEnds[$bufferPath] ?? null) !== $size){ + $this->repairBufferTail($dbname, $bufferPath); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + $count = $size === 0 ? 0 : (isset(self::$bufferCounts[$bufferPath]) && self::$bufferCounts[$bufferPath][0] === $size + ? self::$bufferCounts[$bufferPath][1] : $this->getBufferRecordCount($bufferPath)); + $this->setBufferFlag($dbname, true); + if($this->appendToFile($bufferPath, $lines, $dbname) === false){ + if($this->appendSyncFailed){ + throw new \RuntimeException("noneDB: fsync of the write buffer of database '{$dbname}' failed; nothing was written"); + } + return ["n" => 0, "error" => "Failed to write buffer"]; + } + $size += strlen($lines); // the append ended there (checked against the file next time) + self::$bufferCounts[$bufferPath] = [$size, $count + count($validItems)]; + self::$bufferEnds[$bufferPath] = $size; + if(!isset($this->bufferLastFlush[$dbname])){ + $this->bufferLastFlush[$dbname] = time(); + } + + if($this->shouldFlushBuffer($bufferPath, $dbname, $size, $count + count($validItems))){ + $this->flushPendingBuffers($dbname); + } + return ["n" => count($validItems)]; + } + + /** + * Check if buffer needs flushing (size, count, or time based) + * @param string $bufferPath + * @param string $flushKey Key for tracking last flush time + * @return bool + */ + private function shouldFlushBuffer($bufferPath, $flushKey, $size = null, $count = null){ + if($size === null){ + clearstatcache(true, $bufferPath); + $size = file_exists($bufferPath) ? (int)filesize($bufferPath) : 0; + } + if($size <= 0){ + return false; + } + if($size >= $this->bufferSizeLimit){ + return true; + } + if(($count ?? $this->getBufferRecordCount($bufferPath)) >= $this->bufferCountLimit){ + return true; + } + if($this->bufferFlushInterval > 0){ + $lastFlush = $this->bufferLastFlush[$flushKey] ?? time(); + if((time() - $lastFlush) >= $this->bufferFlushInterval){ + return true; + } + } + return false; + } + + /** + * Register shutdown handler for flushing all buffers + */ + private function registerShutdownHandler(){ + if($this->shutdownHandlerRegistered){ + return; + } + if($this->bufferAutoFlushOnShutdown){ + // Weak reference: the handler must not keep the instance alive until shutdown + $ref = \WeakReference::create($this); + register_shutdown_function(static function() use ($ref){ + $db = $ref->get(); + if($db !== null){ + $db->flushAllBuffers(); + } + }); + $this->shutdownHandlerRegistered = true; + } + } + + // ========================================== + // FILTER MATCHING (find / update / delete) + // ========================================== + + /** + * Normalize a "key" filter value to a list of int keys (invalid values match nothing) + * @return array + */ + private function normalizeKeyFilter($value){ + $values = is_array($value) ? $value : [$value]; + $keys = []; + foreach($values as $v){ + if(is_int($v)){ + $k = $v; + } elseif(is_string($v) && preg_match('/\A(0|[1-9][0-9]*)\z/', $v)){ // \z: "5\n" is not 5 + $k = (int)$v; + } else { + continue; + } + if($k >= 0 && !isset($keys[$k])){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + + /** + * Strict equality filter used by find(), update() and delete(): + * every field must exist in the record and be identical (===) to the filter value. + * "key" filters compare against the record key. + */ + private function matchesFilters(array $record, array $filters){ + foreach($filters as $field => $value){ + if($field === 'key'){ + if(!isset($record['key']) || !in_array($record['key'], $this->normalizeKeyFilter($value), true)){ + return false; + } + continue; + } + if(!array_key_exists($field, $record) || $record[$field] !== $value){ + return false; + } + } + return true; + } + + /** + * Records matching $filters, in key order (or in the order of a "key" filter) + * @return array [key => record] + */ + private function selectRecords($dbname, array $filters){ + // Shards the global field indexes allow (null: every segment); none: nothing to read at all + $targets = array_key_exists('key', $filters) ? null : $this->targetShardsFromGlobalIndex($dbname, $filters); + if($targets === []){ + return []; + } + $segments = $this->listSegments($dbname); + if(empty($segments)){ + return []; + } + + $keyFilter = null; + if(array_key_exists('key', $filters)){ + $keyFilter = $this->normalizeKeyFilter($filters['key']); + unset($filters['key']); + if(empty($keyFilter)){ + return []; + } + } + + $result = []; + if($keyFilter !== null){ + $bySegment = []; + foreach($keyFilter as $key){ + $segment = $this->segmentForKey($dbname, $key); + if($segment !== false){ + $bySegment[$segment === null ? '' : $segment][] = $key; + } + } + $found = []; + foreach($bySegment as $segment => $keys){ + $found += $this->readSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $keys); + } + foreach($keyFilter as $key){ + if(isset($found[$key]) && $this->matchesFilters($found[$key], $filters)){ + $result[$key] = $found[$key]; + } + } + return $result; + } + + if($targets !== null){ + // The targets that are segments (a global index may still name a removed shard), in order + $present = array_flip($segments); + $segments = []; + foreach($targets as $target){ + if(isset($present[$target])){ + $segments[] = $target; + } + } + } + foreach($segments as $segment){ + $candidates = $this->fieldIndexCandidates($dbname, $segment, $filters); + if($candidates !== null && empty($candidates)){ + continue; + } + foreach($this->readSegmentRecords($dbname, $segment, $candidates) as $key => $record){ + if(empty($filters) || $this->matchesFilters($record, $filters)){ + $result[$key] = $record; + } + } + } + return $result; + } + + /** + * Group [key => record] by the segment that stores each key + * @return array [segmentId|'' => [key => record]] + */ + private function groupBySegment($dbname, array $records){ + $groups = []; + $sharded = $this->isSharded($dbname); + $size = $this->shardSizeOf($dbname); + foreach($records as $key => $record){ + $groups[$sharded ? intdiv($key, $size) : ''][$key] = $record; + } + return $groups; + } + + // ========================================== + // SECONDARY INDEX FILES (v3.3.0) + // ========================================== + // A field index (.fx.), an R-tree (.rx.) and the global shard summaries (.gfx., .grx.) are checkpoints + // of their segment's data, like the key index: line 1 of the file is a header with the field, the + // data bytes "s" it covers, the crc32b "h" of the bytes before s and the epoch "e" of the key index it + // was made with; the body follows. An index is a function of the data alone: loading it replays the + // data lines after s (in memory), so writes never touch it, and a crash, a rollback or another + // process's writes can't leave it stale. Once its tail reached 64 KiB (or a quarter of the index file), + // the process that uses it writes a new checkpoint: a reader that loaded it, or a write session that + // has it loaded (a process that only writes never loads it). An index whose epoch or bytes don't fit + // the segment is rebuilt from the data (exclusive lock). + // Names: {base}[_sN].nonedb.{fx|rx}.{token} and {base}.nonedb.{gfx|grx}.{token}; the token is the field + // when it is [a-z0-9_]{1,40}, else a lower-case prefix and 12 hex digits of its sha256: at most 40 + // bytes, and fields that differ only in case ("city", "City") never share a file. + // Files of v3.2 (.fidx., .gfidx., .sidx., .gsidx.) are read until a writer first changes the segment + // (then its indexes are converted) or upgradeIndexes() converts them. + + private const SECONDARY_TAIL_BYTES = 65536; + + /** + * File name part of an index on $field (at most 40 bytes) + */ + private function indexToken($field){ + $field = (string)$field; + if(isset(self::$indexTokens[$field])){ + return self::$indexTokens[$field]; + } + $token = preg_match('/^[a-z0-9_]{1,40}$/', $field) === 1 ? $field + : substr(strtolower(preg_replace('/[^A-Za-z0-9_]/', '_', $field)), 0, 24) . '-' . substr(hash('sha256', $field), 0, 12); + if(count(self::$indexTokens) > 4096){ + self::$indexTokens = []; + } + return self::$indexTokens[$field] = $token; + } + + /** + * Path of a secondary index file: 'fx' / 'rx' per segment, 'gfx' / 'grx' per database + */ + private function secondaryPath($dbname, $kind, $field, $shardId = null){ + if($kind === 'gfx' || $kind === 'grx'){ + return $this->dbBase($dbname) . '.nonedb.' . $kind . '.' . $this->indexToken($field); + } + return $this->segDataPath($dbname, $shardId) . '.' . $kind . '.' . $this->indexToken($field); + } + + /** + * The key index of a segment for a secondary index check: the cached one when it is known to cover + * the data file (no function call chain on the hot path), else readJsonlIndex() + */ + private function currentKeyIndex($dbname, $shardId){ + $path = $this->segDataPath($dbname, $shardId) . '.jidx'; + if(isset($this->verifiedIndexes[$path], $this->indexCache[$path])){ + return $this->indexCache[$path]; + } + return $this->readJsonlIndex($dbname, $shardId); + } + + /** + * [header, body] of a secondary index file; null when it is missing or its header is unreadable + */ + private function readSecondaryFile($path){ + clearstatcache(true, $path); + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + $body = $line === false ? '' : (string)stream_get_contents($fp); + fclose($fp); + $header = $line === false ? null : json_decode(rtrim($line, "\n"), true); + return is_array($header) ? [$header, $body] : null; + } + + /** + * Only the header line of a secondary index file (null: missing or unreadable) + */ + private function readSecondaryHeader($path){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + fclose($fp); + $header = $line === false ? null : json_decode(rtrim($line, "\n"), true); + return is_array($header) ? $header : null; + } + + /** + * Write a secondary index file (header line + body). A checkpoint: no generation bump, its content is + * the same whoever writes it. + * @throws \RuntimeException + */ + private function writeSecondaryFile($dbname, $path, array $header, $body){ + $line = $this->encodeJson($header); + if($line === false || !$this->writeDerivedFile($path, $line . "\n" . $body)){ + throw new \RuntimeException("noneDB: failed to write index " . basename($path) . " of database '{$dbname}'"); + } + $this->markFileExists($path); + } + + /** + * The segment's key index, with an epoch: an index without one (rebuilt without a checkpoint) gets its + * checkpoint written first (exclusive lock), since secondary indexes name the epoch they belong to + * @return array|null + */ + private function keyIndexWithEpoch($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] !== null){ + return $index; + } + $lock = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$lock]) || self::$lockRegistry[$lock]['mode'] !== LOCK_EX){ + return $index; + } + $this->writeCheckpoint($dbname, $shardId, $index); + return $this->readJsonlIndex($dbname, $shardId); + } + + /** + * True when a secondary index header belongs to the segment as it is: same field, the epoch of its + * key index, and the bytes it covers unchanged (checked once per epoch and offset) + */ + private function secondaryFits($dbname, $shardId, array $header, $kind, $field){ + if(($header['kind'] ?? null) !== $kind || ($header['field'] ?? null) !== (string)$field + || !isset($header['s'], $header['e'], $header['h']) || !is_int($header['s']) || !is_string($header['e'])){ + return false; + } + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] === null || $header['e'] !== $index['e'] || $header['s'] > (int)$index['s']){ + return false; + } + $dataPath = $this->segDataPath($dbname, $shardId); + $checked = $index['e'] . ':' . $header['s']; + if((self::$secondaryChecked[$dataPath] ?? null) === $checked){ + return true; + } + if($this->segmentCrc($dataPath, $header['s']) !== $header['h']){ + return false; + } + self::$secondaryChecked[$dataPath] = $checked; + return true; + } + + /** + * Records of the complete data lines [$from, $to) of a segment, in file order: [[key, record or null + * for a tombstone], ...]. Lines that are neither are skipped (salvaged ones count, as in the key index). + */ + private function segmentTail($dbname, $shardId, $from, $to){ + if($to <= $from){ + return []; + } + $dataPath = $this->segDataPath($dbname, $shardId); + $cached = self::$tailCache[$dataPath] ?? null; + if($cached !== null && $cached[0] === $from && $cached[1] === $to){ + return $cached[2]; + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return []; + } + fseek($fp, $from); + $bytes = ''; + while(strlen($bytes) < $to - $from){ + $chunk = fread($fp, min(1048576, $to - $from - strlen($bytes))); + if($chunk === false || $chunk === ''){ + break; + } + $bytes .= $chunk; + } + fclose($fp); + $base = $shardId === null ? 0 : $shardId * $this->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->shardSizeOf($dbname); + $rows = []; + $pos = 0; + $empty = []; + while(($newline = strpos($bytes, "\n", $pos)) !== false){ + $json = substr($bytes, $pos, $newline - $pos); + $pos = $newline + 1; + $record = json_decode($json, true); + if(is_array($record) && count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $rows[] = [$record['__nonedb_deleted'], null]; + continue; + } + if(is_array($record) && isset($record['key']) && is_int($record['key'])){ + $rows[] = [$record['key'], $record]; + continue; + } + $entry = $this->salvageLine($json, $from + $pos - strlen($json) - 1, $empty, $base, $limit); + if($entry !== null){ + $rows[] = $entry[0] === 't' ? [$entry[1], null] : [$entry[1], json_decode(substr($json, $entry[2], $entry[3]), true)]; + } + } + if(count(self::$tailCache) > 64){ + self::$tailCache = []; + } + self::$tailCache[$dataPath] = [$from, $to, $rows]; + return $rows; + } + + /** + * A global index (gfx / grx) about to be rewritten, which stamps it with the current lock epoch: when + * it was written under another epoch (a data file may have been replaced since), only the entries that + * still fit their shard are kept, so the new stamp vouches for no stale entry + */ + private function freshGlobalEntries($dbname, array $global){ + $lockPath = $this->getLockPath($dbname); + if($global['le'] !== null && isset(self::$lockRegistry[$lockPath]) && $global['le'] === $this->registryEpoch($lockPath)){ + return $global; + } + foreach($global['sh'] as $shardId => $entry){ + if($this->globalEntryFits($dbname, $shardId, $entry, null)){ + continue; + } + unset($global['sh'][$shardId]); + if(isset($global['map'])){ + self::mapWithoutShard($global['map'], $shardId); + } + } + return $global; + } + + /** + * True when the tail of a secondary index (data after the bytes it covers) is long enough to write a + * new checkpoint of it + */ + private function secondaryTailIsLong($covered, $end, $fileBytes){ + return $end - $covered >= max(self::SECONDARY_TAIL_BYTES, intdiv((int)$fileBytes, 4)); + } + + /** + * A reader that loaded a secondary index and replayed a long tail writes a new checkpoint of it (and of + * its shard's global entry), so the next process doesn't replay the same lines again. Under the shared + * lock only: never inside a write session (whose appends a rollback may still cut), not for + * checkIntegrity(), explain() or a snapshot. Best effort, no generation bump: the content is a function + * of the data. + * @return bool True when one is due but was put off (checkIntegrity(), explain(), a join): the caller + * marks the cached index ("due") and the next plain read of this process writes it + */ + private function maybeSecondaryReaderCheckpoint($dbname, $shardId, $kind, $field, array $index){ + $lock = $this->getLockPath($dbname); + if($index['e'] === null || !empty($index['legacy']) + || !$this->secondaryTailIsLong((int)$index['ck'], (int)$index['s'], (int)$index['bytes'])){ + return false; + } + if(self::$noWrites > 0 || self::$multiLock > 0){ + return true; // explain(), checkIntegrity(), a join: due, the next plain read of this process writes it + } + if(!isset(self::$lockRegistry[$lock]) || self::$lockRegistry[$lock]['mode'] !== LOCK_SH){ + return false; // a write session: checkpointSegmentSecondaries() at its end + } + try { + if($kind === 'fx'){ + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } else { + $this->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: the next load replays the lines again + } + return false; + } + + /** + * End of a write session: write new checkpoints of the field and spatial indexes of a changed segment + * that this process has loaded and whose tail is long, and their global shard summaries (exclusive + * lock, best effort). An index nobody here uses isn't loaded for it: a writer that only writes keeps + * no secondary index in memory, and the next reader that loads it writes the checkpoint. + */ + private function checkpointSegmentSecondaries($dbname, $shardId){ + $manifest = $this->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; + } + $key = $this->keyIndexWithEpoch($dbname, $shardId); + if($key === null || $key['e'] === null || !$this->segmentIsConverted($dbname, $shardId)){ + return; + } + foreach(['fx' => $manifest['fields'], 'rx' => $manifest['spatial']] as $kind => $fields){ + foreach($fields as $field){ + $path = $this->secondaryPath($dbname, $kind, $field, $shardId); + $cached = $kind === 'fx' ? ($this->fieldIndexCache[$path] ?? null) : ($this->spatialIndexCache[$path] ?? null); + if($cached === null || $cached['e'] !== $key['e'] || !empty($cached['legacy'])){ + continue; // not loaded here (or of another epoch): the reader that loads it checkpoints it + } + $covered = (int)$cached['ck']; + $bytes = (int)$cached['bytes']; + if(!$this->secondaryTailIsLong($covered, (int)$key['s'], $bytes)){ + continue; + } + try { + if($kind === 'fx'){ + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); + if($index !== null && empty($index['legacy'])){ + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + } else { + $index = $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); + if(empty($index['legacy'])){ + $this->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: the next load replays a longer tail + } + } + } + } + // ========================================== + // FIELD INDEXES + // ========================================== + // Which fields are indexed is recorded in {base}.nonedb.indexes ("manifest", v2 since v3.3: + // {"v":2,"fields","spatial","spatialOptions","fmt":3}; "fmt":3 once no index file of v3.2 is left). + // A segment's field index {segment}.nonedb.fx.{token} (see SecondaryIndex) maps the exact identity of + // a value (valueKey(): "n", "t", "f", "i42", "d", "sAnkara") to the keys that hold it: + // {"v":3,"kind":"fx","field":"city","s":48213,"e":"9f2c41d0","h":"…","mk":812,"c":37,"n":800} + // {"n":[2,4],"f":10,"i-3":[1,9],"i42":7,"d4045000000000000":11,"sAnkara":3,"sIstanbul":[0,5,9]} + // Buckets are exact (1, 1.0, "1" and true are different values, as for ===) and don't depend on the + // precision, serialize_precision or locale settings. Values that are arrays are not indexed. The + // global field index {base}.nonedb.gfx.{token} of a sharded database maps each value to the shards + // that hold it as of each shard's covered bytes; a shard's newer lines are checked directly. + // v3.2's {segment}.nonedb.fidx.{name} (string buckets) is still read for segments this version + // hasn't written yet (see convertSegmentIndexes); its global .gfidx. is never used to skip shards. + + private function getIndexManifestPath($dbname){ + return $this->dbBase($dbname) . '.nonedb.indexes'; + } + + /** + * Indexed fields of a database: ['fields', 'spatial', 'spatialOptions', 'fmt' => bool (every segment + * converted), 'collides' => fields whose v3.2 files are shared with another field] + * Databases created before v3.2 have no manifest; it is derived from their index files. + */ + private function getIndexManifest($dbname){ + $path = $this->getIndexManifestPath($dbname); + if(isset($this->fieldIndexCache[$path])){ + self::$cacheTouch[$path] = ++self::$cacheTick; + return $this->fieldIndexCache[$path]; + } + clearstatcache(true, $path); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? false : json_decode($content, true); + if($content === false && $this->isVerified($dbname)){ + $manifest = ['fmt' => 3]; // no index list: no indexes (pre-3.2 databases got theirs when verified) + } elseif(!is_array($manifest)){ + // Corrupt, or zero-length after a power loss: derived from the index files that exist + $manifest = $this->deriveIndexManifest($dbname); + } + $fields = array_values(array_unique(array_map('strval', $manifest['fields'] ?? []))); + $manifest = [ + 'fields' => $fields, + 'spatial' => array_values(array_unique(array_map('strval', $manifest['spatial'] ?? []))), + 'spatialOptions' => is_array($manifest['spatialOptions'] ?? null) ? $manifest['spatialOptions'] : [], + 'fmt' => ($manifest['fmt'] ?? null) === 3, + 'collides' => $this->collidingFields($fields), + ]; + $this->fieldIndexCache[$path] = $manifest; + self::$cacheGrew = true; + return $manifest; + } + + private function saveIndexManifest($dbname, array $manifest){ + $file = [ + 'v' => 2, + 'fields' => array_values(array_unique($manifest['fields'] ?? [])), + 'spatial' => array_values(array_unique($manifest['spatial'] ?? [])), + ]; + $file['spatialOptions'] = array_intersect_key($manifest['spatialOptions'] ?? [], array_flip($file['spatial'])); + if(!empty($manifest['fmt'])){ + $file['fmt'] = 3; // v3.2 ignores it, and drops it when it writes the list (its files are back then) + } + $path = $this->getIndexManifestPath($dbname); + if(empty($file['fields']) && empty($file['spatial'])){ + $this->fsUnlink($path); + } elseif(!$this->atomicWrite($path, $file, false, 'durable')){ + throw new \RuntimeException("noneDB: failed to write index list of database '{$dbname}'"); + } + $this->fieldIndexCache[$path] = ['fields' => $file['fields'], 'spatial' => $file['spatial'], + 'spatialOptions' => $file['spatialOptions'], 'fmt' => !empty($manifest['fmt']), + 'collides' => $this->collidingFields($file['fields'])]; + } + + /** + * Databases without a readable index list: field names from the index files that exist (headers of + * v3.3 files, "field" of v3.2 files; v3.2 file names may have been lossy) + */ + private function deriveIndexManifest($dbname){ + $base = $this->dbBase($dbname); + $hashBase = $this->dbDir . $this->hashDBName($dbname); + $fields = []; + $spatial = []; + $options = []; + foreach(array_merge($this->listFilesWithPrefix($base . '.nonedb.'), $this->listFilesWithPrefix($base . '_s', '/^\d+\.nonedb\./')) as $file){ + if(preg_match('/\.nonedb\.(fx|rx)\.[^.]+$/', $file, $m)){ + $header = $this->readSecondaryHeader($file); + if(is_array($header) && isset($header['field']) && is_string($header['field'])){ + if($m[1] === 'fx'){ + $fields[] = $header['field']; + } else { + $spatial[] = $header['field']; + if(isset($header['ns'])){ + $options[$header['field']] = ['nodeSize' => (int)$header['ns']]; + } + } + } + } elseif(preg_match('/\.fidx\.(.+)$/', $file, $m)){ + $index = $this->atomicRead($file, null); + $fields[] = is_array($index) && isset($index['field']) && is_string($index['field']) ? $index['field'] : $this->decodeIndexFieldName($m[1]); + } + } + foreach(array_merge($this->listFilesWithPrefix($hashBase . '.nonedb.sidx.'), $this->listFilesWithPrefix($hashBase . '_s', '/^\d+\.nonedb\.sidx\./')) as $file){ + $index = $this->atomicRead($file, null); + if(is_string($index)){ + $index = json_decode($index, true); // pre-3.2 files were JSON-encoded twice + } + if(is_array($index) && isset($index['field']) && is_string($index['field'])){ + $spatial[] = $index['field']; + } elseif(preg_match('/\.sidx\.(.+)$/', $file, $m)){ + $spatial[] = $this->decodeIndexFieldName($m[1]); + } + } + return ['fields' => array_values(array_unique($fields)), 'spatial' => array_values(array_unique($spatial)), 'spatialOptions' => $options]; + } + + /** + * File name part of a v3.2 index file: the field itself when it matches [A-Za-z0-9_]+, else "~" + hex + */ + private function encodeIndexFieldName($field){ + $field = (string)$field; + return preg_match('/^[A-Za-z0-9_]+$/', $field) ? $field : '~' . bin2hex($field); + } + + private function decodeIndexFieldName($name){ + if(isset($name[0]) && $name[0] === '~' && ctype_xdigit(substr($name, 1)) && strlen($name) % 2 === 1){ + return hex2bin(substr($name, 1)); + } + return $name; + } + + /** + * Fields whose v3.2 index files are shared with another indexed field: same name ignoring case (one + * file on case-insensitive file systems). Their v3.2 files are never used. + * @return array field => true + */ + private function collidingFields(array $fields){ + $collides = []; + $byLower = []; + foreach($fields as $field){ + $byLower[strtolower($field)][] = $field; + } + foreach($byLower as $group){ + if(count($group) > 1){ + foreach($group as $field){ + $collides[$field] = true; + } + } + } + return $collides; + } + + /** + * Error for a database whose index file names would be longer than file systems allow (255 bytes): + * the field takes at most 40 bytes (indexToken), the database name the rest. Null when fine. + */ + private function indexNameError($dbname, $field){ + $longest = 20 + 1 + strlen($dbname) + 2 + 10 + strlen('.nonedb.gfx.') + strlen($this->indexToken($field)) + 4 + 8; + if($longest > 255){ + return "Index file names of database '{$dbname}' would be longer than 255 bytes (database name too long)"; + } + return null; + } + + /** + * Path of a segment's field index (v3.3) + */ + private function getFieldIndexPath($dbname, $field, $shardId = null){ + return $this->secondaryPath($dbname, 'fx', $field, $shardId); + } + + /** + * Path of a segment's v3.2 field index + */ + private function getLegacyFieldIndexPath($dbname, $field, $shardId = null){ + return $this->segDataPath($dbname, $shardId) . ".fidx." . $this->encodeIndexFieldName($field); + } + + /** + * Exact bucket of a value (see valueKey()); arrays are not indexed + */ + private function fieldIndexValueKey($value){ + return self::valueKey($value); + } + + /** + * Bucket of a v3.2 field index: values of different types share one (1, "1", 1.0), floats depend on + * the precision setting + */ + private function legacyFieldIndexKey($value){ + if($value === null){ + return '__null__'; + } + if(is_bool($value)){ + return $value ? '__true__' : '__false__'; + } + if(is_float($value) && $value == 0.0){ + return '0'; + } + return (string)$value; + } + + /** + * True if a value can be stored in a field index + */ + private function isIndexableValue($value){ + return is_scalar($value) || is_null($value); + } + + private function newFieldIndex($field){ + return ['kind' => 'fx', 'field' => (string)$field, 's' => 0, 'ck' => -1, 'e' => null, 'b' => [], 'mk' => -1, + 'rev' => null, 'views' => null, 'bytes' => 0, 'legacy' => false]; + } + + /** + * Build a segment's field index from its data (covers the key index's bytes) + * @return array + */ + private function buildSegmentFieldIndex($dbname, $shardId, $field){ + $key = $this->keyIndexWithEpoch($dbname, $shardId); + $index = $this->newFieldIndex($field); + // fieldIndexApply() inlined: every key occurs once, so a value is only ever added + $field = (string)$field; + $buckets = []; + foreach($this->readSegmentRecords($dbname, $shardId) as $k => $record){ + $k = (int)$k; + if($k > $index['mk']){ + $index['mk'] = $k; + } + if(!isset($record[$field]) && !array_key_exists($field, $record)){ + continue; + } + $value = $record[$field]; + if(is_string($value)){ + $buckets['s' . $value][] = $k; + } elseif(is_scalar($value) || $value === null){ + $buckets[self::valueKey($value)][] = $k; + } + } + foreach($buckets as $bucket => $keys){ + $index['b'][$bucket] = count($keys) === 1 ? $keys[0] : $keys; + } + $index['mk'] = $key === null ? -1 : max($index['mk'], empty($key['p']) ? -1 : max(array_keys($key['p']))); + $index['s'] = $key === null ? 0 : (int)$key['s']; + $index['e'] = $key === null ? null : $key['e']; + $index['rev'] = null; + return $index; + } + + /** + * Apply one data line to a field index: the key now holds $record (null: deleted) + */ + private function fieldIndexApply(array &$index, $key, $record){ + $field = $index['field']; + $new = $record !== null && array_key_exists($field, $record) && $this->isIndexableValue($record[$field]) + ? self::valueKey($record[$field]) : null; + if($key > $index['mk']){ + // A key never seen: nothing to remove + $index['mk'] = $key; + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + if($index['rev'] !== null){ + $index['rev'][$key] = $new; + } + } + return; + } + if($index['rev'] === null){ + $rev = []; + foreach($index['b'] as $bucket => $keys){ + foreach((array)$keys as $k){ + $rev[$k] = (string)$bucket; + } + } + $index['rev'] = $rev; + } + $old = $index['rev'][$key] ?? null; + if($old === $new){ + return; // the value didn't change: nothing to do (T2-6) + } + if($old !== null){ + $keys = $index['b'][$old]; + if(is_int($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + } else { + $at = array_search($key, $keys, true); + if($at !== false){ + unset($keys[$at]); + } + $keys = array_values($keys); + if(count($keys) === 1){ + $index['b'][$old] = $keys[0]; + } elseif(empty($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + } else { + $index['b'][$old] = $keys; + } + } + unset($index['rev'][$key]); + } + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + $index['rev'][$key] = $new; + } + } + + private function fieldIndexAdd(array &$index, $bucket, $key){ + if(!isset($index['b'][$bucket])){ + $index['b'][$bucket] = $key; + $index['views'] = null; + } elseif(is_int($index['b'][$bucket])){ + $index['b'][$bucket] = [$index['b'][$bucket], $key]; + } else { + $index['b'][$bucket][] = $key; + } + } + + /** + * A segment's field index, current (the data lines after the bytes it covers are replayed). Null when + * there is none that fits the segment (missing, of another epoch, damaged): see ensureSegmentFieldIndex. + * @return array|null + */ + private function readFieldIndex($dbname, $field, $shardId = null){ + // The segment's key index (currentKeyIndex() inlined: this is the hot path of every lookup) + $dataPath = $this->segDataPath($dbname, $shardId); + $key = isset($this->verifiedIndexes[$dataPath . '.jidx']) ? ($this->indexCache[$dataPath . '.jidx'] ?? null) : null; + if($key === null && ($key = $this->readJsonlIndex($dbname, $shardId)) === null){ + return null; + } + $path = $dataPath . '.fx.' . $this->indexToken($field); + self::$cacheTouch[$path] = ++self::$cacheTick; + if(isset($this->fieldIndexCache[$path])){ + $index = $this->fieldIndexCache[$path]; + if($index['s'] === (int)$key['s'] && $index['e'] === $key['e'] && $index['e'] !== null){ + if(isset($index['due']) && self::$noWrites === 0 && self::$multiLock === 0){ + unset($index['due'], $this->fieldIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + return $this->fieldIndexCache[$path]; + } + return $index; // the usual case: nothing written since + } + if($index['e'] !== null && $index['e'] === $key['e'] && $index['s'] <= (int)$key['s']){ + if($index['s'] < (int)$key['s']){ + unset($this->fieldIndexCache[$path], $index['due']); // sole owner while it changes + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + } + return $this->fieldIndexCache[$path]; + } + unset($this->fieldIndexCache[$path]); + } + $file = $this->readSecondaryFile($path); + if($file === null || !$this->secondaryFits($dbname, $shardId, $file[0], 'fx', $field)){ + return null; + } + $buckets = json_decode($file[1], true); + if(!is_array($buckets)){ + return null; + } + $index = $this->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['ck'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $index['bytes'] = strlen($file[1]); + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + self::$cacheGrew = true; + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + return $this->fieldIndexCache[$path]; + } + + private function replayFieldIndex($dbname, $shardId, array &$index, $to){ + foreach($this->segmentTail($dbname, $shardId, $index['s'], $to) as [$k, $record]){ + $this->fieldIndexApply($index, $k, $record); + } + $index['s'] = $to; + } + + /** + * Read a segment's v3.2 field index (only for a segment this version hasn't written yet) + * @return array|null ['legacy' => true, 'values' => [legacy bucket => keys]] + */ + private function readLegacyFieldIndex($dbname, $field, $shardId){ + if(isset($this->getIndexManifest($dbname)['collides'][(string)$field])){ + return null; // shares its file with another field + } + $path = $this->getLegacyFieldIndexPath($dbname, $field, $shardId); + if(array_key_exists($path, $this->fieldIndexCache)){ + return $this->fieldIndexCache[$path]; + } + $index = $this->atomicRead($path, null); + if(!is_array($index) || ($index['v'] ?? 1) < 2 || !isset($index['values']) || !is_array($index['values']) + || (isset($index['field']) && $index['field'] !== (string)$field)){ + $index = null; + } else { + $index = ['legacy' => true, 'field' => (string)$field, 'values' => $index['values']]; + } + $this->fieldIndexCache[$path] = $index; + self::$cacheGrew = true; + return $index; + } + + /** + * Write a field index checkpoint (exclusive lock or a reader: the content is the data's) + */ + private function writeFieldIndex($dbname, $field, array &$index, $shardId = null){ + if($index['e'] === null){ + return false; + } + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $crc = $this->segmentCrc($this->segDataPath($dbname, $shardId), (int)$index['s']); + if($crc === null){ + return false; + } + $buckets = self::canonicalBuckets($index['b']); + $body = $this->encodeJson(empty($buckets) ? new \stdClass() : $buckets); + if($body === false){ + return false; + } + $n = 0; + foreach($buckets as $keys){ + $n += is_int($keys) ? 1 : count($keys); + } + $header = ['v' => 3, 'kind' => 'fx', 'field' => (string)$field, 's' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc, + 'mk' => (int)$index['mk'], 'c' => count($buckets), 'n' => $n, 'created' => time()]; + $this->writeSecondaryFile($dbname, $path, $header, $body . "\n"); + $index['b'] = $buckets; + $index['ck'] = (int)$index['s']; + $index['bytes'] = strlen($body) + 1; + $index['views'] = null; + $this->fieldIndexCache[$path] = $index; + self::$cacheGrew = true; + return true; + } + + /** + * Buckets in the fixed order of an index file, so the file is the same whoever writes it: bucket ids + * in byte order (ksort, in C), keys of a bucket ascending. Range queries sort their own views + * (fieldIndexViews), so no order by value is needed here. + */ + private static function canonicalBuckets(array $buckets){ + ksort($buckets, SORT_STRING); + foreach($buckets as &$keys){ + if(is_array($keys)){ + sort($keys); + } + } + unset($keys); + return $buckets; + } + + /** + * The number of an "i…" or "d…" bucket + */ + private static function bucketNumber($bucket){ + if($bucket[0] === 'i'){ + return (int)substr($bucket, 1); + } + $unpacked = unpack('E', (string)hex2bin(substr($bucket, 1))); + return $unpacked === false ? 0.0 : $unpacked[1]; + } + + /** + * Delete a field's index files of a segment (v3.3 and v3.2) + */ + private function deleteFieldIndexFile($dbname, $field, $shardId = null){ + $ok = true; + foreach([$this->getFieldIndexPath($dbname, $field, $shardId), $this->getLegacyFieldIndexPath($dbname, $field, $shardId)] as $path){ + unset($this->fieldIndexCache[$path]); + $this->markFileNotExists($path); + clearstatcache(true, $path); + if(file_exists($path) && !$this->fsUnlink($path)){ + $ok = false; + } + } + return $ok; + } + + /** + * Get list of indexed fields for a database + * @return array + */ + private function getIndexedFields($dbname, $shardId = null){ + return $this->getIndexManifest($dbname)['fields']; + } + + /** + * Check if a field has an index + * @return bool + */ + private function isFieldIndexed($dbname, $field, $shardId = null){ + return in_array((string)$field, $this->getIndexedFields($dbname), true); + } + + /** + * Invalidate field index cache for a database + */ + private function invalidateFieldIndexCache($dbname, $field = null, $shardId = null){ + $prefix = $this->dbDir . $this->hashDBName($dbname); + foreach(array_keys($this->fieldIndexCache) as $key){ + if(strpos($key, $prefix) === 0){ + unset($this->fieldIndexCache[$key]); + } + } + } + + /** + * A segment's field index: the current one, a v3.2 one while the segment isn't converted, or built + * from the data (exclusive lock). Null when none can be used (the caller scans). + * @return array|null + */ + private function ensureSegmentFieldIndex($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index !== null){ + return $index; + } + if(!$this->segmentIsConverted($dbname, $shardId)){ + $legacy = $this->readLegacyFieldIndex($dbname, $field, $shardId); + if($legacy !== null){ + return $legacy; + } + } + if(self::$noWrites > 0 || self::$multiLock > 0){ + return null; // checkIntegrity(), explain(), a snapshot: nothing is built, the segment is scanned + } + return $this->withLock($dbname, true, function() use ($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + return $index; + }); + } + + /** + * Keys of a field index with exactly this value (v3.2 index: candidates of the same loose bucket; + * null for a float there, whose bucket depends on the precision setting) + * @return array|null + */ + private function fieldIndexKeys(array $index, $value){ + if(!empty($index['legacy'])){ + if(is_float($value)){ + return null; + } + return array_map('intval', (array)($index['values'][$this->legacyFieldIndexKey($value)] ?? [])); + } + $keys = $index['b'][is_string($value) ? 's' . $value : self::valueKey($value)] ?? []; + return is_int($keys) ? [$keys] : $keys; + } + + /** + * Candidate keys of a segment for equality filters, from field indexes. + * Null means "no usable index, scan the segment". + * @return array|null + */ + private function fieldIndexCandidates($dbname, $shardId, array $filters){ + if(!$this->fieldIndexEnabled || empty($filters)){ + return null; + } + $indexed = $this->getIndexedFields($dbname); + if(empty($indexed)){ + return null; + } + $candidates = null; + foreach($filters as $field => $value){ + if($field === 'key' || !$this->isIndexableValue($value) || !in_array((string)$field, $indexed, true)){ + continue; + } + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; // never read without the lock + } + $index = null; // can't be built: scan (B-1) + } + $keys = $index === null ? null : $this->fieldIndexKeys($index, $value); + if($keys === null){ + continue; + } + $candidates = $candidates === null ? $keys : array_values(array_intersect($candidates, $keys)); + if(empty($candidates)){ + return []; + } + } + return $candidates; + } + + /** + * Field indexes follow the data by themselves (replayed when used): kept for the write paths + */ + private function applyFieldIndexChanges($dbname, $shardId, array $removed, array $added){ + } + + // ==================== RANGE VIEWS (I3) ==================== + + /** + * Candidate keys of a segment whose field value may satisfy every range condition ($gt, $gte, $lt, + * $lte => operand): a superset, the caller checks every record. Null: no usable index (scan). + * @param array $conditions [operator => operand] + * @return array|null + */ + private function fieldIndexRangeCandidates($dbname, $shardId, $field, array $conditions){ + if(!$this->fieldIndexEnabled || !$this->isFieldIndexed($dbname, $field)){ + return null; + } + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); + } catch(\Throwable $e){ + if($e->getCode() === self::LOCK_FAILED){ + throw $e; + } + return null; + } + if($index === null || !empty($index['legacy'])){ + return null; + } + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $views = $this->fieldIndexViews($index); + if(isset($this->fieldIndexCache[$path])){ + $this->fieldIndexCache[$path]['views'] = $views; + } + $buckets = null; // bucket => true + foreach($conditions as $op => $operand){ + if(!in_array($op, ['$gt', '$gte', '$lt', '$lte'], true)){ + return null; + } + $found = $this->rangeBuckets($views, $op, $operand); + if($found === null){ + return null; // an operand nothing compares with: no help from the index + } + $buckets = $buckets === null ? $found : array_intersect_key($buckets, $found); + if(empty($buckets)){ + return []; + } + } + if($buckets === null){ + return null; + } + $keys = []; + foreach(array_keys($buckets) as $bucket){ + $bucketKeys = $index['b'][$bucket] ?? []; + if(is_int($bucketKeys)){ + $keys[] = $bucketKeys; + } else { + foreach($bucketKeys as $k){ + $keys[] = $k; + } + } + } + return $keys; + } + + /** + * Sorted views of a field index's buckets: numbers (ints, floats and numeric strings, by value) and + * strings (byte order) + * @return array ['num' => [[float value, bucket], ...], 'str' => [[string, bucket], ...]] + */ + /** + * The first and last values of fieldIndexViews() (numbers: ints, floats and numeric strings by value; + * strings by byte order) without building the views + * @return array [num low, num high, string low, string high], nulls when there are none + */ + private static function fieldIndexBounds(array $index){ + if(!empty($index['views'])){ + $num = $index['views']['num']; + $str = $index['views']['str']; + return [$num ? $num[0][0] : null, $num ? $num[count($num) - 1][0] : null, + $str ? $str[0][0] : null, $str ? $str[count($str) - 1][0] : null]; + } + $nlo = $nhi = $slo = $shi = null; + $keys = array_keys($index['b']); + $other = preg_grep('/\Ai/', $keys, PREG_GREP_INVERT); + $ints = $other ? array_diff_key($keys, $other) : $keys; + if($ints){ + // Without the "i": min()/max() compare integer strings as numbers, in C + $values = substr_replace($ints, '', 0, 1); + $nlo = (float)(int)min($values); + $nhi = (float)(int)max($values); + } + foreach(preg_grep('/\Ad/', $other) as $bucket){ + $number = (float)self::bucketNumber($bucket); + if($nlo === null || $number < $nlo){ + $nlo = $number; + } + if($nhi === null || $number > $nhi){ + $nhi = $number; + } + } + $strings = preg_grep('/\As/', $other); + if($strings){ + sort($strings, SORT_STRING); + $slo = substr(reset($strings), 1); + $shi = substr(end($strings), 1); + // Numeric strings start with optional white space and sign, then a digit or ".digit" + foreach(preg_grep('/\As[ \t\n\r\v\f]*[+-]?\.?[0-9]/', $strings) as $bucket){ + $trimmed = rtrim(substr($bucket, 1), " \t\n\r\v\f"); + if(!is_numeric($trimmed)){ + continue; + } + $number = (float)$trimmed; + if($nlo === null || $number < $nlo){ + $nlo = $number; + } + if($nhi === null || $number > $nhi){ + $nhi = $number; + } + } + } + return [$nlo, $nhi, $slo, $shi]; + } + + private function fieldIndexViews(array $index){ + if($index['views'] !== null){ + return $index['views']; + } + // Sort keys first, then one array_multisort per view (a comparator per pair is far slower) + $numValues = []; + $numIds = []; + $strValues = []; + $strIds = []; + foreach(array_keys($index['b']) as $bucket){ + $bucket = (string)$bucket; + $type = $bucket[0]; + if($type === 'i' || $type === 'd'){ + $numValues[] = (float)self::bucketNumber($bucket); + $numIds[] = $bucket; + } elseif($type === 's'){ + $s = substr($bucket, 1); + $strValues[] = $s; + $strIds[] = $bucket; + $trimmed = rtrim($s, " \t\n\r\v\f"); + if(is_numeric($trimmed)){ + $numValues[] = (float)$trimmed; + $numIds[] = $bucket; + } + } + } + array_multisort($numValues, SORT_NUMERIC, $numIds, SORT_STRING); + array_multisort($strValues, SORT_STRING, $strIds, SORT_STRING); + $num = []; + foreach($numValues as $i => $value){ + $num[] = [$value, $numIds[$i]]; + } + $str = []; + foreach($strValues as $i => $value){ + $str[] = [$value, $strIds[$i]]; + } + return ['num' => $num, 'str' => $str]; + } + + /** + * Buckets whose values may compare as $op with $operand (noneDBQuery::compareValues semantics: numbers + * and numeric strings numerically, other strings by byte order; null, booleans and arrays compare + * with nothing). Ties of different types are included (rechecked by the caller). + * @return array|null bucket => true; null when the operand can't be compared + */ + private function rangeBuckets(array $views, $op, $operand){ + if(is_float($operand) && is_nan($operand)){ + return []; // NAN matches nothing + } + if(is_int($operand) || is_float($operand) || (is_string($operand) && is_numeric(rtrim($operand, " \t\n\r\v\f")))){ + $x = (float)(is_string($operand) ? rtrim($operand, " \t\n\r\v\f") : $operand); + $list = $views['num']; + $found = []; + // A wider band at the boundary: values above 2^53 compare as floats here and as ints there + $slack = abs($x) >= 9007199254740992.0 ? max(1.0, abs($x) * 4.0e-16) : 0.0; + foreach($this->rangeSlice($list, $op, $x, $slack) as $entry){ + $found[$entry[1]] = true; + } + if(is_string($operand)){ + // A numeric string operand compares with non-numeric strings by byte order too + foreach($this->rangeSliceStrings($views['str'], $op, $operand) as $entry){ + $found[$entry[1]] = true; + } + } + return $found; + } + if(is_string($operand)){ + $found = []; + foreach($this->rangeSliceStrings($views['str'], $op, $operand) as $entry){ + $found[$entry[1]] = true; + } + return $found; + } + return null; + } + + /** + * Entries of a sorted numeric view on the operator's side of $x (binary search; $slack widens it). + * Without slack ($x below 2^53, where these floats are exact) a strict operator leaves out the values + * equal to $x: compareValues() finds them equal too, so they never match + */ + private function rangeSlice(array $list, $op, $x, $slack){ + $n = count($list); + $strict = $slack == 0.0 && ($op === '$gt' || $op === '$lt'); + if($op === '$gt' || $op === '$gte'){ + $bound = $x - $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid][0] <= $bound : $list[$mid][0] < $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return array_slice($list, $lo); + } + $bound = $x + $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid][0] < $bound : $list[$mid][0] <= $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return array_slice($list, 0, $lo); + } + + /** + * Entries of the sorted string view on the operator's side of $x (the string $x itself only for $gte + * and $lte) + */ + private function rangeSliceStrings(array $list, $op, $x){ + $n = count($list); + $lo = 0; + $hi = $n; + $strict = $op === '$gt' || $op === '$lt'; + if($op === '$gt' || $op === '$gte'){ + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid][0], $x); + if($strict ? $cmp <= 0 : $cmp < 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return array_slice($list, $lo); + } + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid][0], $x); + if($strict ? $cmp < 0 : $cmp <= 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return array_slice($list, 0, $lo); + } + + // ==================== GLOBAL FIELD INDEX (shard skip) ==================== + // {base}.nonedb.gfx.{token}: header {"v":3,"kind":"gfx","field","le": lock epoch when written, + // "sh": {shard: {"s","e","h","nlo","nhi","slo","shi"}}}, body {value id: shard | [shards]}. An entry says which + // values the shard's data bytes [0, s) hold. While the lock epoch is the one recorded, no data file was + // replaced, so every entry still describes its shard; otherwise each entry is checked against its + // shard (epoch and checksum). A shard is skipped only when it has an entry that fits, the value isn't + // in it, and the shard's lines after s don't have it either. No entry: never skipped. Entries are + // updated when a shard's field index is checkpointed, never per write. + + private function getGlobalFieldIndexPath($dbname, $field){ + return $this->secondaryPath($dbname, 'gfx', $field); + } + + private function getLegacyGlobalFieldIndexPath($dbname, $field){ + return $this->dbBase($dbname) . ".nonedb.gfidx." . $this->encodeIndexFieldName($field); + } + + /** + * Read the global field index: ['le', 'sh' => [shard => entry], 'map' => [value id => [shards]]] + * @return array|null + */ + private function readGlobalFieldIndex($dbname, $field, $path = null){ + $path = $path ?? $this->getGlobalFieldIndexPath($dbname, $field); + self::$cacheTouch[$path] = ++self::$cacheTick; + if(array_key_exists($path, $this->fieldIndexCache)){ + return $this->fieldIndexCache[$path]; + } + $file = $this->readSecondaryFile($path); + $global = null; + if($file !== null && ($file[0]['kind'] ?? null) === 'gfx' && ($file[0]['field'] ?? null) === (string)$field && is_array($file[0]['sh'] ?? null)){ + $map = json_decode($file[1], true); + if(is_array($map)){ + $global = ['le' => $file[0]['le'] ?? null, 'sh' => [], 'map' => $map]; + foreach($file[0]['sh'] as $shardId => $entry){ + if(is_array($entry) && isset($entry['s'], $entry['e'], $entry['h'])){ + $global['sh'][(int)$shardId] = $entry; + } + } + } + } + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$this->getLockPath($dbname)][$path]); + self::$cacheGrew = true; + return $global; + } + + /** + * Remove a shard from a global field index map (value bucket => shard id, or [shard ids] when several + * shards hold the value: a unique value costs no array), except for the buckets in $keep + */ + private static function mapWithoutShard(array &$map, $shardId, ?array $keep = null){ + foreach($map as $bucket => $held){ + if($keep !== null && isset($keep[$bucket])){ + continue; + } + if($held === $shardId){ + unset($map[$bucket]); + } elseif(is_array($held) && in_array($shardId, $held, true)){ + $held = array_values(array_diff($held, [$shardId])); + if(count($held) === 1){ + $map[$bucket] = $held[0]; + } else { + $map[$bucket] = $held; + } + } + } + } + + /** + * Write the global field index (exclusive lock) + */ + private function writeGlobalFieldIndex($dbname, $field, array $global){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $lock = self::$lockRegistry[$this->getLockPath($dbname)] ?? null; + $global['le'] = $lock === null ? null : $this->registryEpoch($this->getLockPath($dbname)); + $map = $global['map']; + ksort($map, SORT_STRING); + $body = $this->encodeJson(empty($map) ? new \stdClass() : $map); + if($body === false){ + throw new \RuntimeException("noneDB: failed to write global field index '{$field}' of database '{$dbname}'"); + } + $sh = []; + foreach($global['sh'] as $shardId => $entry){ + $sh[(string)$shardId] = $entry; + } + $this->writeSecondaryFile($dbname, $path, ['v' => 3, 'kind' => 'gfx', 'field' => (string)$field, 'le' => $global['le'], + 'sh' => empty($sh) ? new \stdClass() : $sh], $body . "\n"); + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$this->getLockPath($dbname)][$path], self::$lockRegistry[$this->getLockPath($dbname)]['globals'][$path]); + } + + /** + * Record a shard's values (from its field index, at the bytes it covers) in the global field index + */ + private function updateGlobalFieldShard($dbname, $field, $shardId, array $index){ + $global = $this->freshGlobalEntries($dbname, $this->readGlobalFieldIndex($dbname, $field) ?? ['le' => null, 'sh' => [], 'map' => []]); + if($this->applyGlobalFieldShard($global, $dbname, $shardId, $index, false)){ + $this->deferGlobalWrite($dbname, 'gfx', $field, $global); + } + } + + /** + * Keep a changed global field or spatial index in the cache and write it when the lock session ends + * (flushPendingGlobals()); without a lock at once + */ + private function deferGlobalWrite($dbname, $kind, $field, array $global){ + $lock = $this->getLockPath($dbname); + if(!isset(self::$lockRegistry[$lock])){ + if($kind === 'gfx'){ + $this->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + return; + } + if($kind === 'gfx'){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$lock][$path]); + } else { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $this->globalSpatialIndexCache[$path] = $global; + } + self::$lockRegistry[$lock]['globals'][$path] = [$kind, $field, $this->registryEpoch($lock)]; + } + + /** + * Record a shard's values in a global field index in memory. $fresh: a rebuild (the shard isn't in it + * yet, shards come in ascending order): no shard list is searched and every bucket gets a list + * @return bool False when the index can't be recorded (a v3.2 index, no epoch, data file gone) + */ + private function applyGlobalFieldShard(array &$global, $dbname, $shardId, array $index, $fresh){ + if(!empty($index['legacy']) || $index['e'] === null){ + return false; + } + $crc = $this->segmentCrc($this->segDataPath($dbname, $shardId), (int)$index['s']); + if($crc === null){ + return false; + } + if(!$fresh && isset($global['sh'][$shardId])){ + self::mapWithoutShard($global['map'], $shardId, $index['b']); + } + // No local copy of a shard list: appending to an array also held by a variable copies it first + // (with a value in every shard, O(shards) per append) + $map = &$global['map']; + if($fresh){ + // Lists only; rebuildGlobalFieldIndex() turns the single ones into ints at the end + foreach(array_keys($index['b']) as $bucket){ + $map[$bucket][] = $shardId; + } + } else { + foreach(array_keys($index['b']) as $bucket){ + if(!isset($map[$bucket])){ + $map[$bucket] = $shardId; + } elseif(is_int($map[$bucket])){ + if($map[$bucket] !== $shardId){ + $map[$bucket] = [$map[$bucket], $shardId]; + } + } elseif(!in_array($shardId, $map[$bucket], true)){ + $map[$bucket][] = $shardId; + } + } + } + unset($map); + $entry = ['s' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc]; + [$nlo, $nhi, $slo, $shi] = self::fieldIndexBounds($index); + if($nlo !== null){ + $entry['nlo'] = bin2hex(pack('E', $nlo)); + $entry['nhi'] = bin2hex(pack('E', $nhi)); + } + if($slo !== null){ + $entry['slo'] = $slo; + $entry['shi'] = $shi; + } + $global['sh'][$shardId] = $entry; + return true; + } + + /** + * Rebuild the global field index from the shard field indexes (exclusive lock): built in memory and + * written once (once per shard was O(shards²)) + */ + private function rebuildGlobalFieldIndex($dbname, $field){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + unset($this->fieldIndexCache[$path], $this->globalSkip[$this->getLockPath($dbname)][$path]); + $global = ['le' => null, 'sh' => [], 'map' => []]; + $shards = $this->listSegments($dbname); + sort($shards); + foreach($shards as $shardId){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + } + $this->applyGlobalFieldShard($global, $dbname, $shardId, $index, true); + } + foreach($global['map'] as &$held){ + if(count($held) === 1){ + $held = $held[0]; + } + } + unset($held); + $this->writeGlobalFieldIndex($dbname, $field, $global); + return $this->fieldIndexCache[$path] ?? $global; + } + + /** + * True when a global entry still describes its shard's data bytes [0, s) + */ + private function globalEntryFits($dbname, $shardId, array $entry, $lockEpoch){ + $lock = self::$lockRegistry[$this->getLockPath($dbname)] ?? null; + $dataPath = $this->segDataPath($dbname, $shardId); + if($lockEpoch !== null && $lock !== null && $this->registryEpoch($this->getLockPath($dbname)) === $lockEpoch){ + return true; // no data file was replaced since the entry was written + } + if(($this->dataEpochs[$dataPath] ?? null) === null){ + $this->dataEpochs[$dataPath] = $this->segmentEpochOnDisk($dbname, $shardId); + } + if($this->dataEpochs[$dataPath] !== $entry['e']){ + return false; + } + $checked = $entry['e'] . ':' . $entry['s']; + if((self::$secondaryChecked[$dataPath] ?? null) === $checked){ + return true; + } + if($this->segmentCrc($dataPath, (int)$entry['s']) !== $entry['h']){ + return false; + } + self::$secondaryChecked[$dataPath] = $checked; + return true; + } + + /** + * A segment's data epoch without loading its key index: the cached index, else the checkpoint header + */ + private function segmentEpochOnDisk($dbname, $shardId){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + if(isset($this->indexCache[$path])){ + return $this->indexCache[$path]['e']; + } + $fp = @fopen($path, 'rb'); + if($fp === false){ + return false; + } + $head = (string)fread($fp, 512); + fclose($fp); + return preg_match('/"e":"([0-9a-f]{8})"/', $head, $m) === 1 ? $m[1] : false; + } + + /** + * Current data size of a shard: its key index's when this process has it verified (its own writes + * keep it current), else the file's (cached until a write) + */ + private function shardDataSize($dbname, $shardId, $dataPath = null){ + $dataPath = $dataPath ?? $this->segDataPath($dbname, $shardId); + if(isset($this->verifiedIndexes[$dataPath . '.jidx'], $this->indexCache[$dataPath . '.jidx'])){ + return (int)$this->indexCache[$dataPath . '.jidx']['s']; + } + if(!isset($this->dataSizes[$dataPath])){ + clearstatcache(true, $dataPath); + $this->dataSizes[$dataPath] = (int)@filesize($dataPath); + } + return $this->dataSizes[$dataPath]; + } + + /** + * The shards a lookup in a global field index looks at besides the value's holders: [shard => false] + * has no entry that fits its data (always read), [shard => true] has lines after its entry (read when + * they may hold the value). Worked out over all shards once per state of the data (another process's + * write or a new epoch starts over: invalidateDbCaches(), bumpEpoch()) and kept current with this + * process's own writes (noteWrite()), so a lookup costs O(holders + such shards), not O(shards). + * @return array shard id => bool + */ + private function globalCheckShards($dbname, array $global, $path, $lockPath){ + $seq = $this->segWrites[$lockPath]['seq'] ?? 0; + $state = $this->globalSkip[$lockPath][$path] ?? null; + if($state !== null && $state['seq'] === $seq){ + return $state['check']; + } + if($state === null){ + // While the lock epoch is the recorded one no data file was replaced: every entry fits + $fresh = $global['le'] !== null && isset(self::$lockRegistry[$lockPath]) && $global['le'] === $this->registryEpoch($lockPath); + $check = []; + $prefix = $this->dbBase($dbname) . '_s'; + foreach($this->listSegments($dbname) as $shardId){ + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || (!$fresh && !$this->globalEntryFits($dbname, $shardId, $entry, null))){ + $check[$shardId] = false; + } elseif($this->shardDataSize($dbname, $shardId, $prefix . $shardId . '.nonedb') > (int)$entry['s']){ + $check[$shardId] = true; + } + } + $state = ['seq' => $seq, 'check' => $check]; + } else { + // This process appended to some shards since: their entries still fit (the bytes they cover + // didn't change), their newer lines are checked; a new shard has no entry + $prefix = $this->dbBase($dbname) . '_s'; + $len = strlen($prefix); + foreach($this->segWrites[$lockPath]['paths'] as $dataPath => $at){ + if($at > $state['seq'] && strncmp($dataPath, $prefix, $len) === 0){ + $shardId = (int)substr($dataPath, $len); + if(!isset($state['check'][$shardId])){ + $state['check'][$shardId] = isset($global['sh'][$shardId]); + } + } + } + $state['seq'] = $seq; + } + $this->globalSkip[$lockPath][$path] = $state; + return $state['check']; + } + + /** + * True when a shard's data lines after a global entry (whose bytes are covered) may hold a value + * matching $match (callable(record) => bool) + */ + private function shardTailMayMatch($dbname, $shardId, array $entry, callable $match){ + if($this->shardDataSize($dbname, $shardId) <= (int)$entry['s']){ + return false; + } + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null){ + return true; + } + foreach($this->segmentTail($dbname, $shardId, (int)$entry['s'], (int)$index['s']) as [$k, $record]){ + if($record !== null && $match($record)){ + return true; + } + } + return false; + } + + /** + * Shards that can contain matches for the equality filters (null = all shards / not sharded) + * @return array|null + */ + private function targetShardsFromGlobalIndex($dbname, array $filters){ + if(!$this->fieldIndexEnabled || !$this->isSharded($dbname) || empty($filters)){ + return null; + } + $manifest = $this->getIndexManifest($dbname); + $targets = null; + $base = null; + foreach($filters as $field => $value){ + if($field === 'key' || !(is_scalar($value) || $value === null) || !in_array((string)$field, $manifest['fields'], true)){ + continue; + } + // Paths built here, not by the helpers: this runs for every lookup of a sharded database + $base = $base ?? $this->dbBase($dbname); + $lockPath = $base . '.nonedb.lock'; + $path = $base . '.nonedb.gfx.' . (self::$indexTokens[(string)$field] ?? $this->indexToken($field)); + if(isset($this->fieldIndexCache[$path])){ + $global = $this->fieldIndexCache[$path]; // readGlobalFieldIndex()'s cache hit, inlined + self::$cacheTouch[$path] = ++self::$cacheTick; + } else { + $global = $this->readGlobalFieldIndex($dbname, $field, $path); + } + if($global === null){ + if(self::$noWrites > 0 || self::$multiLock > 0 || !$manifest['fmt']){ + continue; // checkIntegrity(), a snapshot, or v3.2 files not all converted: build nothing, skip nothing + } + $global = $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildGlobalFieldIndex($dbname, $field); + }); + if($global === null){ + continue; + } + } + // The shards whose entry can't rule a value out (worked out once per state of the data) + $state = $this->globalSkip[$lockPath][$path] ?? null; + $check = $state !== null && $state['seq'] === ($this->segWrites[$lockPath]['seq'] ?? 0) + ? $state['check'] : $this->globalCheckShards($dbname, $global, $path, $lockPath); + // The value's holders, and those of the shards to check that may hold it + $held = $global['map'][is_string($value) ? 's' . $value : self::valueKey($value)] ?? null; + if(empty($check)){ + $shards = $held === null ? [] : (is_int($held) ? [$held] : $held); + if(count($shards) > 1){ + sort($shards); + } + } else { + $shards = $held === null ? [] : (is_int($held) ? [$held => true] : array_fill_keys($held, true)); + $match = null; + foreach($check as $shardId => $tail){ + if(isset($shards[$shardId])){ + continue; + } + $match = $match ?? function($record) use ($field, $value){ + return array_key_exists($field, $record) && $record[$field] === $value; + }; + if(!$tail || $this->shardTailMayMatch($dbname, $shardId, $global['sh'][$shardId], $match)){ + $shards[$shardId] = true; + } + } + $shards = array_keys($shards); + sort($shards); + } + $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); + } + return $targets; + } + + /** + * Delete the global field index files (v3.3 and v3.2) + */ + private function deleteGlobalFieldIndex($dbname, $field){ + foreach([$this->getGlobalFieldIndexPath($dbname, $field), $this->getLegacyGlobalFieldIndexPath($dbname, $field)] as $path){ + clearstatcache(true, $path); + if(file_exists($path)){ + $this->fsUnlink($path); + } + unset($this->fieldIndexCache[$path], $this->globalSkip[$this->getLockPath($dbname)][$path]); + } + } + + // ==================== CONVERSION OF v3.2 INDEX FILES ==================== + + /** + * True when no index file of v3.2 is left for the segment (or the index list says none is left) + */ + private function segmentIsConverted($dbname, $shardId){ + $manifest = $this->getIndexManifest($dbname); + if($manifest['fmt'] || (empty($manifest['fields']) && empty($manifest['spatial']))){ + return true; + } + $dataPath = $this->segDataPath($dbname, $shardId); + if(!isset($this->convertedSegments[$dataPath])){ + $legacy = false; + foreach($manifest['fields'] as $field){ + $legacy = $legacy || $this->cachedFileExists($this->getLegacyFieldIndexPath($dbname, $field, $shardId)); + } + foreach($manifest['spatial'] as $field){ + $legacy = $legacy || $this->cachedFileExists($this->getLegacySpatialIndexPath($dbname, $field, $shardId)); + } + $this->convertedSegments[$dataPath] = !$legacy; + } + return $this->convertedSegments[$dataPath]; + } + + /** + * Before a writer first changes a segment that still has v3.2 index files (exclusive lock): its field + * indexes and R-trees are built in the v3.3 formats from the data, then its v3.2 files go. The first + * conversion of a database also removes the v3.2 global field indexes (v3.2 trusts them to skip shards, + * but this version doesn't update them; it rebuilds missing ones), the last one the global spatial + * indexes, and marks the index list converted ("fmt": 3). + * @return bool True when the segment was converted now + */ + private function convertSegmentIndexes($dbname, $shardId){ + if($this->segmentIsConverted($dbname, $shardId)){ + return false; + } + $manifest = $this->getIndexManifest($dbname); + foreach($manifest['fields'] as $field){ + $this->deleteLegacyGlobalFieldIndex($dbname, $field); + } + $this->keyIndexWithEpoch($dbname, $shardId); + foreach($manifest['fields'] as $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId) ?? $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + $legacy = $this->getLegacyFieldIndexPath($dbname, $field, $shardId); + $this->fsUnlink($legacy); + $this->markFileNotExists($legacy); + unset($this->fieldIndexCache[$legacy]); + } + foreach($manifest['spatial'] as $field){ + $index = $this->readSpatialIndex($dbname, $field, $shardId) ?? $this->buildSegmentSpatialIndex($dbname, $field, $shardId, + isset($manifest['spatialOptions'][$field]['nodeSize']) ? ['nodeSize' => $manifest['spatialOptions'][$field]['nodeSize']] : []); + $this->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + $legacy = $this->getLegacySpatialIndexPath($dbname, $field, $shardId); + $this->fsUnlink($legacy); + $this->markFileNotExists($legacy); + unset($this->spatialIndexCache[$legacy]); + } + $this->convertedSegments[$this->segDataPath($dbname, $shardId)] = true; + $this->fault('convert:segment', ['db' => $dbname, 'shard' => $shardId]); + foreach($this->listSegments($dbname) as $segment){ + if(!$this->segmentIsConverted($dbname, $segment)){ + return true; + } + } + // Every segment converted: the v3.2 global spatial indexes go last, then the index list says so + foreach($manifest['spatial'] as $field){ + $legacy = $this->getLegacyGlobalSpatialIndexPath($dbname, $field); + $this->fsUnlink($legacy); + unset($this->globalSpatialIndexCache[$legacy]); + } + $manifest['fmt'] = true; + $this->saveIndexManifest($dbname, $manifest); + return true; + } + + /** + * A state record without an epoch (exclusive lock): v3.2 may have used the database and built its own + * index files again, which it trusts but this version doesn't update. With v3.2 files of a segment + * left, the index list stops saying "converted": each segment is checked again and converted before + * it is next written to. Only v3.2 global files left: they go (v3.2 rebuilds missing ones from its + * segment files, none of which are left). + */ + private function forgetIndexConversion($dbname){ + $manifest = $this->getIndexManifest($dbname); + if(!$manifest['fmt'] || (empty($manifest['fields']) && empty($manifest['spatial']))){ + return; + } + foreach($this->listSegments($dbname) as $segment){ + foreach($manifest['fields'] as $field){ + if(is_file($this->getLegacyFieldIndexPath($dbname, $field, $segment))){ + $manifest['fmt'] = false; + } + } + foreach($manifest['spatial'] as $field){ + if(is_file($this->getLegacySpatialIndexPath($dbname, $field, $segment))){ + $manifest['fmt'] = false; + } + } + } + if(!$manifest['fmt']){ + $this->saveIndexManifest($dbname, $manifest); + return; + } + foreach($manifest['fields'] as $field){ + $this->deleteLegacyGlobalFieldIndex($dbname, $field); + } + foreach($manifest['spatial'] as $field){ + $legacy = $this->getLegacyGlobalSpatialIndexPath($dbname, $field); + if(is_file($legacy)){ + $this->fsUnlink($legacy); + } + unset($this->globalSpatialIndexCache[$legacy]); + } + } + + private function deleteLegacyGlobalFieldIndex($dbname, $field){ + $path = $this->getLegacyGlobalFieldIndexPath($dbname, $field); + if($this->cachedFileExists($path)){ + $this->fsUnlink($path); + $this->markFileNotExists($path); + } + unset($this->fieldIndexCache[$path]); + } + + /** + * After a migration to sharded format: the non-sharded index files go; every shard's field and + * spatial indexes are built from the data + */ + private function rebuildIndexesAfterMigration($dbname){ + $manifest = $this->getIndexManifest($dbname); + foreach($manifest['fields'] as $field){ + $this->deleteFieldIndexFile($dbname, $field, null); + $this->deleteGlobalFieldIndex($dbname, $field); + foreach($this->listSegments($dbname) as $shardId){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + } + $this->rebuildGlobalFieldIndex($dbname, $field); + } + foreach($manifest['spatial'] as $field){ + $this->rebuildSpatialIndexForDb($dbname, $field); + } + $manifest['fmt'] = true; + $this->saveIndexManifest($dbname, $manifest); + } + + // ========================================== + // QUERY PLANS AND CURSORS (v3.3.0) + // ========================================== + // A plan is a tree of what a query's filters ask for that indexes can answer: + // ['eq', field, value] ['in', field, values] ['range', field, operator, operand] + // ['keys', [key, ...]] ['any'] ['and', [nodes]] ['or', [nodes]] + // For each segment it gives candidate keys (a superset of the matches: the caller checks every record + // with the whole filter) or null (scan the segment); shards it can rule out (global field indexes, + // key ranges) are skipped. Negations (whereNot, $ne, $nin, notLike, ...) never use indexes. + // scanRecords() reads the candidates of every segment in key order, in batches (32, growing to 4096), + // and stops as soon as the visitor says so: first(), exists() and limit() read what they need. + + /** + * Candidate keys of a segment for a plan node; null when the node can't narrow the segment down + * @return int[]|null sorted + */ + private function planCandidates($dbname, $shardId, array $node){ + switch($node[0]){ + case 'eq': + if(!$this->fieldIndexEnabled || !$this->isIndexableValue($node[2]) || !$this->isFieldIndexed($dbname, $node[1])){ + return null; + } + return $this->fieldIndexCandidates($dbname, $shardId, [$node[1] => $node[2]]); + case 'in': + if(!$this->fieldIndexEnabled || !$this->isFieldIndexed($dbname, $node[1])){ + return null; + } + $keys = []; + foreach($node[2] as $value){ + if(!$this->isIndexableValue($value)){ + return null; + } + $found = $this->fieldIndexCandidates($dbname, $shardId, [$node[1] => $value]); + if($found === null){ + return null; + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'range': + return $this->fieldIndexRangeCandidates($dbname, $shardId, $node[1], [$node[2] => $node[3]]); + case 'keys': + $index = $this->readJsonlIndex($dbname, $shardId); + $keys = []; + foreach($node[1] as $k){ + if($index !== null && isset($index['p'][$k])){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + continue; + } + $result = $result === null ? $found : array_keys(array_intersect_key(array_flip($result), array_flip($found))); + if(empty($result)){ + return []; + } + } + return $result; + case 'or': + $keys = []; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + return null; // one branch needs a scan: the whole segment + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + return null; + } + + /** + * Shards a plan node can match in (null: all). Only skips a shard when it can't hold a match. + * @return array|null shard id => true + */ + private function planShards($dbname, array $node){ + if(!$this->isSharded($dbname)){ + return null; + } + switch($node[0]){ + case 'eq': + if(!$this->isIndexableValue($node[2])){ + return null; + } + $shards = $this->targetShardsFromGlobalIndex($dbname, [$node[1] => $node[2]]); + return $shards === null ? null : array_fill_keys($shards, true); + case 'in': + $all = []; + foreach($node[2] as $value){ + $found = $this->planShards($dbname, ['eq', $node[1], $value]); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + case 'keys': + $size = $this->shardSizeOf($dbname); + $shards = []; + foreach($node[1] as $k){ + $shards[intdiv($k, $size)] = true; + } + return $shards; + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found !== null){ + $result = $result === null ? $found : array_intersect_key($result, $found); + } + } + return $result; + case 'or': + $all = []; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + } + return null; + } + + /** + * Read the records a plan allows, segment by segment in key order, in batches: $visit([key => record]) + * gets each batch in key order and returns false to stop. Caller holds the lock. + * @param bool $whole The visitor never stops (count, top-k): each segment is read in one go + * @return array ['segments', 'skipped', 'indexed', 'scanned', 'recordsRead', 'plan'] + */ + private function scanRecords($dbname, array $plan, callable $visit, $whole = false){ + $stats = ['segments' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0, 'recordsRead' => 0, 'plan' => []]; + $segments = $this->listSegments($dbname); + if(empty($segments)){ + return $stats; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + foreach($segments as $segment){ + $stats['segments']++; + if($shards !== null && !isset($shards[$segment])){ + $stats['skipped']++; + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } + continue; + } + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + $access = $keys === null ? 'scan' : 'index'; + if($keys === null){ + $stats['scanned']++; + $index = $this->readJsonlIndex($dbname, $segment); + $keys = $index === null ? [] : array_keys($index['p']); + } else { + $stats['indexed']++; + } + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => $access, 'candidates' => count($keys)]; + } + if(empty($keys)){ + continue; + } + // Keys come in ascending order almost always (they are assigned in order, an update keeps a + // key's place in the index): sorted only when they aren't + $previous = PHP_INT_MIN; + foreach($keys as $k){ + if($k < $previous){ + sort($keys, SORT_NUMERIC); + break; + } + $previous = $k; + } + $n = count($keys); + $batch = $whole ? $n : 32; + for($i = 0; $i < $n; $i += $batch, $batch = min(4096, $batch * 4)){ + $records = $this->readSegmentRecords($dbname, $segment, $access === 'scan' && $i === 0 && $batch >= $n ? null : array_slice($keys, $i, $batch)); + $stats['recordsRead'] += count($records); + if($visit($records) === false){ + return $stats; + } + } + } + return $stats; + } + + /** + * explain(): what scanRecords() would do, without reading records or building anything (an index that + * a read would build counts as a scan) + */ + private function explainPlan($dbname, array $plan){ + $explain = function() use ($dbname, $plan){ + $out = ['sharded' => $this->dbExists($dbname) && $this->isSharded($dbname), 'segments' => ['total' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0], + 'plan' => [], 'estimatedRead' => 0]; + if(!$this->dbExists($dbname)){ + return $out; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + foreach($this->listSegments($dbname) as $segment){ + $out['segments']['total']++; + if($shards !== null && !isset($shards[$segment])){ + $out['segments']['skipped']++; + $entry = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } else { + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + if($keys === null){ + $out['segments']['scanned']++; + $index = $this->readJsonlIndex($dbname, $segment); + $entry = ['segment' => $segment, 'access' => 'scan', 'candidates' => $index === null ? 0 : count($index['p'])]; + } else { + $out['segments']['indexed']++; + $entry = ['segment' => $segment, 'access' => 'index', 'candidates' => count($keys)]; + } + } + $out['estimatedRead'] += $entry['candidates']; + if(count($out['plan']) < 50){ + $out['plan'][] = $entry; + } + } + $out['planTruncated'] = $out['segments']['total'] > 50; + return $out; + }; + self::$noWrites++; + try { + return $this->hasLockFile($dbname) ? $this->withLock($dbname, false, $explain) : $explain(); + } finally { + self::$noWrites--; + } + } + + + /** + * Visit the records matching find()-style filters (field => value) in key order, stopping when + * $visit returns false. False when it can't (a key filter keeps its own order: find() handles it; + * a missing database without autoCreateDB): then the caller uses find(). Caller holds the lock. + */ + private function scanMatches($dbname, array $filters, callable $visit, $whole = false){ + if(array_key_exists('key', $filters)){ + return false; + } + if(!$this->dbExists($dbname)){ + return !$this->autoCreateDB ? false : true; + } + $and = []; + foreach($filters as $field => $value){ + $and[] = is_array($value) ? ['any'] : ['eq', (string)$field, $value]; + } + $plan = count($and) === 1 ? $and[0] : ['and', $and]; + if($plan[0] === 'and'){ + $plan[1] = array_values(array_filter($plan[1], function($node){ return $node[0] !== 'any'; })); + $plan = empty($plan[1]) ? ['any'] : (count($plan[1]) === 1 ? $plan[1][0] : $plan); + } + $this->scanRecords($dbname, $plan, function(array $records) use ($filters, $visit){ + foreach($records as $key => $record){ + if($this->matchesFilters($record, $filters) && $visit($key, $record) === false){ + return false; + } + } + return true; + }, $whole); + return true; + } + + // ========================================== + // PUBLIC INDEX API + // ========================================== + + /** + * Enable or disable indexing (kept for backward compatibility; key lookups always use the .jidx index) + * @param bool $enable + */ + public function enableIndexing($enable = true){ + $this->indexEnabled = (bool)$enable; + } + + /** + * Check if indexing is enabled + * @return bool + */ + public function isIndexingEnabled(){ + return $this->indexEnabled; + } + + /** + * Rebuild the key indexes (.jidx) of a database from its data files. + * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. + * @param string $dbname + * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] + */ + public function rebuildIndex($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname){ + return $this->rebuildIndexUnlocked($dbname); + }); + } + + private function rebuildIndexUnlocked($dbname){ + $start = microtime(true); + $this->flushPendingBuffers($dbname); + $segments = $this->listSegments($dbname); + if(empty($segments)){ + return ['success' => false, 'error' => 'Database does not exist']; + } + $total = 0; + foreach($segments as $segment){ + $this->ensureJsonlFormat($dbname, $segment); + $total += count($this->rebuildSegmentIndexFromData($dbname, $segment)['p']); + } + if($this->isSharded($dbname)){ + $this->recountMeta($dbname); + } + // After a manual restore the field and spatial indexes are as stale as the key index (B-8) + $this->rebuildSecondaryIndexes($dbname, $segments); + return [ + 'success' => true, + 'totalRecords' => $total, + 'time' => round((microtime(true) - $start) * 1000, 2) . 'ms' + ]; + } + + /** + * Get index information for a database + * @param string $dbname + * @return array|null + */ + public function getIndexInfo($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname){ + return $this->getIndexInfoUnlocked($dbname); + }); + } + + private function getIndexInfoUnlocked($dbname){ + $segments = $this->listSegments($dbname); + if(empty($segments)){ + return null; + } + $total = 0; + $created = null; + $updated = null; + foreach($segments as $segment){ + $index = $this->readJsonlIndex($dbname, $segment); + if($index === null){ + continue; + } + $total += count($index['p']); + $created = $created === null ? ($index['created'] ?? null) : min($created, $index['created'] ?? $created); + $updated = max((int)$updated, (int)($index['updated'] ?? 0)); + } + $sharded = $this->isSharded($dbname); + return [ + 'exists' => true, + 'version' => 4, + 'created' => $created, + 'updated' => $updated, + 'totalRecords' => $total, + 'sharded' => $sharded, + 'path' => $sharded ? $this->getMetaPath($dbname) : $this->getJsonlIndexPath($dbname, null) + ]; + } + + /** + * Recovery log of a database (v3.3.0): what crash recovery and maintenance did to its files, oldest + * first. Every entry has "t" (time) and "action": "cut-tail" / "drop-line" (bytes removed from a data + * file, kept in the "quarantine" file), "salvaged" / "skip-line" (a damaged line found while + * rebuilding an index), "deferred-failed" (compaction, buffer flush or migration after a committed + * write failed; it runs again later). What reads of this process found and no write session logged + * yet is included with "persisted" => false (reads never write). + * @param string $dbname + * @param int $limit Newest entries to return + * @return array + */ + /** + * Check a database against its own data, read-only (v3.3.0): every line of every data file is + * decoded; the key index, field indexes, spatial indexes and shard meta are compared with what the + * data says. Nothing is written, repaired, flushed or recovered (rebuildIndex() repairs). + * @param string $dbname + * @return array ['ok' => bool, 'segments', 'lines', 'records', 'badLines' => [[segment, offset, length]], + * 'tornTail', 'indexMismatches', 'bufferedRecords', 'pendingRecovery', 'quarantineFiles', + * 'recoveryLog' => last entries] + */ + public function checkIntegrity($dbname){ + $dbname = $this->normalizeDbName($dbname); + $check = function() use ($dbname){ + return $this->checkIntegrityLocked($dbname); + }; + self::$noWrites++; + try { + // Without a lock file (pre-3.2 or missing database) nothing is locked: taking it would create one + return $this->hasLockFile($dbname) ? $this->withLock($dbname, false, $check) : $check(); + } finally { + self::$noWrites--; + } + } + + private function checkIntegrityLocked($dbname){ + $report = ['ok' => true, 'database' => $dbname, 'exists' => false, 'segments' => 0, 'lines' => 0, 'records' => 0, + 'badLines' => [], 'tornTail' => [], 'indexMismatches' => [], 'bufferedRecords' => 0, 'pendingRecovery' => false, + 'quarantineFiles' => [], 'recoveryLog' => []]; + $lock = self::$lockRegistry[$this->getLockPath($dbname)] ?? null; + if($lock !== null && (!empty($lock['flags']['j']) || empty($lock['flags']['v']))){ + $report['pendingRecovery'] = true; // the next writer or reader recovers (journal / pre-3.2 checks) + } + $base = $this->dbBase($dbname); + $metaPath = $this->getMetaPath($dbname); + clearstatcache(true, $metaPath); + $sharded = file_exists($metaPath); + $meta = $sharded ? $this->readMeta($dbname) : null; + if($sharded && $meta === null){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'meta file unreadable']; + } + $segments = $sharded ? array_map(function($f) use ($base){ return (int)substr($f, strlen($base) + 2); }, + $this->listFilesWithPrefix($base . '_s', '/^\d+\.nonedb$/')) : (is_file($this->getDataPath($dbname)) ? [null] : []); + sort($segments); + $report['exists'] = !empty($segments) || $sharded; + $size = $meta !== null ? (int)$meta['shardSize'] : $this->shardSize; + if($sharded && $meta === null){ + try { + $size = (int)$this->deriveShardMeta($dbname)['shardSize']; // as recovery would rebuild it + } catch(\RuntimeException $e){ + } + } + $manifest = $this->deriveOrReadManifest($dbname); + $liveTotal = 0; + $maxKey = -1; + $byShard = []; + + foreach($segments as $shardId){ + $report['segments']++; + $seg = $shardId === null ? 'main' : $shardId; + $dataPath = $this->segDataPath($dbname, $shardId); + $live = []; + $offset = 0; + $fp = @fopen($dataPath, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $report['tornTail'][] = ['segment' => $seg, 'offset' => $offset, 'length' => strlen($line)]; + break; + } + $report['lines']++; + $entry = $this->classifyLine(substr($line, 0, -1), $shardId === null ? 0 : $shardId * $size, + $shardId === null ? null : ($shardId + 1) * $size); + if($entry === null){ + $report['badLines'][] = [$seg, $offset, strlen($line)]; + } elseif($entry[0] === 't'){ + unset($live[$entry[1]]); + $maxKey = max($maxKey, $entry[1]); + } else { + $live[$entry[1]] = [$offset, $entry[3]]; + $maxKey = max($maxKey, $entry[1]); + } + $offset += strlen($line); + } + if($fp !== false){ + fclose($fp); + } + ksort($live, SORT_NUMERIC); + $liveTotal += count($live); + $byShard[$shardId === null ? 'main' : $shardId] = count($live); + + // Key index: its checkpoint (or v3.2 index) fits the data file, and with the complete lines after it + // gives the same records at the same places. A missing one is rebuilt from the data on the next + // access (normal: nothing lost), a broken one too, but that is reported. + $file = $this->atomicRead($dataPath . '.jidx', null); + clearstatcache(true, $dataPath); + $fileSize = (int)@filesize($dataPath); + $index = null; + if($file === null && @filesize($dataPath . '.jidx') > 0){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'unreadable (rebuilt on next access)']; + } elseif(is_array($file) && isset($file['p'], $file['l'], $file['s']) && is_array($file['p']) && is_array($file['l'])){ + if(!$this->checkpointFits($dataPath, (int)$file['s'], $fileSize, $file['h'] ?? null)){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'checkpoint does not fit the data file (rebuilt on next access)']; + } else { + $index = $file; + } + } elseif(is_array($file) && isset($file['o']) && is_array($file['o'])){ + if(!isset($file['s'])){ + $index = ['p' => [], 'l' => []] + $file; // before v3.2: compared as it is + foreach($file['o'] as $k => $location){ + $index['p'][(int)$k] = (int)($location[0] ?? -1); + $index['l'][(int)$k] = (int)($location[1] ?? -1); + } + $index['s'] = $offset; + } elseif((int)$file['s'] !== $offset){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => "covers {$file['s']} bytes, data has $offset complete bytes"]; + } else { + $index = $this->indexFromV3($file); + } + } elseif($file !== null){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'not an index (rebuilt on next access)']; + } + if($index !== null){ + $index += ['n' => 0, 'd' => 0, 't' => 0, 'tl' => 0]; + $this->replaySegment($dbname, $shardId, $index, $fileSize, null, false); + $indexed = $this->indexEntries($index); + if($indexed != $live){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => count(array_diff_key($live, $indexed)) . ' records not indexed, ' + . count(array_diff_key($indexed, $live)) . ' indexed records not in the data or elsewhere']; + } + } + + // Field and spatial indexes against the records (read in file order: compared in key order) + $records = $this->readJsonlRecordsBatch($dataPath, array_map(function($location){ return $location[0]; }, $live), + array_map(function($location){ return $location[1]; }, $live)); + ksort($records, SORT_NUMERIC); + // Field and spatial indexes: one that fits the segment (its epoch, the bytes it covers), with the + // lines after it replayed, must hold exactly the records; one that doesn't fit is rebuilt when + // used (not damage). v3.2 files of a segment this version hasn't written are compared as well. + $keyEpoch = is_array($index) && isset($index['e']) && is_string($index['e']) ? $index['e'] : null; + foreach($manifest['fields'] as $field){ + $expected = []; + $legacyExpected = []; + foreach($records as $key => $record){ + if(array_key_exists($field, $record) && $this->isIndexableValue($record[$field])){ + $expected[self::valueKey($record[$field])][] = (int)$key; + $legacyExpected[$this->legacyFieldIndexKey($record[$field])][] = (int)$key; + } + } + $actual = $this->integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + $legacy = $this->atomicRead($this->getLegacyFieldIndexPath($dbname, $field, $shardId), null); + if(!is_array($legacy) || !isset($legacy['values']) || !is_array($legacy['values'])){ + continue; // built on first use + } + $expected = $legacyExpected; + $actual = []; + foreach($legacy['values'] as $bucket => $keys){ + if(!empty($keys)){ + $actual[(string)$bucket] = array_map('intval', (array)$keys); + } + } + } + foreach([&$expected, &$actual] as &$buckets){ + foreach($buckets as $bucket => $keys){ + sort($keys); + $buckets[$bucket] = $keys; + } + ksort($buckets, SORT_STRING); + } + unset($buckets); + if($actual !== $expected){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'field ' . $field, 'detail' => 'buckets differ from the records']; + } + } + foreach($manifest['spatial'] as $field){ + $expected = []; + foreach($records as $key => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->validateGeoJSON($geometry)['valid'] && $this->calculateMBR($geometry) !== null){ + $expected[] = (int)$key; + } + } + $actual = $this->integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + $legacy = $this->atomicRead($this->getLegacySpatialIndexPath($dbname, $field, $shardId), null); + if(!is_array($legacy) || !isset($legacy['leafOf'])){ + continue; + } + $actual = array_map('intval', array_keys($legacy['leafOf'])); + } + sort($actual); + if($actual !== $expected){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'spatial ' . $field, 'detail' => 'entries differ from the records']; + } + } + } + $report['records'] = $liveTotal; + + if($meta !== null){ + $listed = array_map('intval', array_column($meta['shards'], 'id')); + sort($listed); + if($listed !== $segments){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'shards listed in the meta differ from the shard files']; + } + if((int)($meta['totalRecords'] ?? -1) !== $liveTotal){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "totalRecords {$meta['totalRecords']}, data has $liveTotal"]; + } + if((int)$meta['nextKey'] <= $maxKey){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "nextKey {$meta['nextKey']} would reuse key $maxKey"]; + } + } + + foreach(array_merge([$this->getBufferPath($dbname)], $this->flushingFiles($dbname), $this->legacyShardBufferFiles($dbname)) as $buffer){ + if(is_file($buffer)){ + $report['bufferedRecords'] += count($this->readBufferRecords($buffer)); + } + } + $report['quarantineFiles'] = array_map('basename', $this->listFilesWithPrefix($this->dbDir . $this->hashDBName($dbname) . '.nonedb.q.', '/\.bin$/')); + $report['recoveryLog'] = array_slice($this->getRecoveryLogEntries($dbname), -20); + $report['ok'] = empty($report['badLines']) && empty($report['tornTail']) && empty($report['indexMismatches']) && !$report['pendingRecovery']; + return $report; + } + + /** + * The index list without writing anything (checkIntegrity) + */ + private function deriveOrReadManifest($dbname){ + $path = $this->getIndexManifestPath($dbname); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? [] : json_decode($content, true); + if(!is_array($manifest)){ + $manifest = $this->deriveIndexManifest($dbname); + } + return ['fields' => array_values($manifest['fields'] ?? []), 'spatial' => array_values($manifest['spatial'] ?? [])]; + } + + /** + * checkIntegrity(): buckets of a segment's field index (v3.3) that fits the key checkpoint, with the + * data lines up to $complete replayed; null when there is none that fits + */ + private function integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->readSecondaryFile($this->getFieldIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'fx', $field, $keyEpoch, $complete)){ + return null; + } + $buckets = json_decode($file[1], true); + if(!is_array($buckets)){ + return null; + } + $index = $this->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $this->replayFieldIndex($dbname, $shardId, $index, $complete); + $out = []; + foreach($index['b'] as $bucket => $keys){ + $out[(string)$bucket] = is_int($keys) ? [$keys] : array_values($keys); + } + return $out; + } + + /** + * checkIntegrity(): keys of a segment's R-tree (v3.3) that fits the key checkpoint, replayed up to + * $complete; null when there is none that fits + */ + private function integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->readSecondaryFile($this->getSpatialIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'rx', $field, $keyEpoch, $complete)){ + return null; + } + $nodes = json_decode($file[1], true); + if(!is_array($nodes) || !isset($file[0]['root'], $file[0]['next'])){ + return null; + } + $tree = $this->newRTree($field, (int)($file[0]['ns'] ?? $this->rtreeNodeSize)); + $tree['nodes'] = []; + foreach($nodes as $id => $node){ + $tree['nodes'][(int)$id] = $node; + } + $tree['root'] = (int)$file[0]['root']; + $tree['next'] = (int)$file[0]['next']; + $tree['mk'] = (int)($file[0]['mk'] ?? -1); + $tree['s'] = $file[0]['s']; + $tree['e'] = $file[0]['e']; + $this->replaySpatialIndex($dbname, $shardId, $tree, $complete); + $this->rtreeMaps($tree); + return array_keys($tree['leafOf']); + } + + private function integrityHeaderFits($dbname, $shardId, array $header, $kind, $field, $keyEpoch, $complete){ + return ($header['kind'] ?? null) === $kind && ($header['field'] ?? null) === (string)$field && $keyEpoch !== null + && ($header['e'] ?? null) === $keyEpoch && isset($header['s']) && is_int($header['s']) && $header['s'] <= $complete + && $this->segmentCrc($this->segDataPath($dbname, $shardId), $header['s']) === ($header['h'] ?? null); + } + + private function getRecoveryLogEntries($dbname){ + $path = $this->getRecoveryLogPath($dbname); + $entries = []; + foreach([$path . '.1', $path] as $file){ + $fp = @fopen($file, 'rb'); + if($fp === false){ + continue; + } + while(($line = fgets($fp)) !== false){ + $entry = substr($line, -1) === "\n" ? json_decode($line, true) : null; + if(is_array($entry)){ + $entries[] = $entry; + } + } + fclose($fp); + } + return $entries; + } + + public function getRecoveryLog($dbname, $limit = 100){ + $dbname = $this->normalizeDbName($dbname); + $read = function() use ($dbname, $limit){ + // What readers of this process found and no write session logged yet: "persisted" => false + $pending = []; + foreach(self::$pendingRecoveryEvents[$this->getLockPath($dbname)] ?? [] as $event){ + $pending[] = $event + ['persisted' => false]; + } + $limit = max(0, (int)$limit); + return $limit === 0 ? [] : array_slice(array_merge($this->getRecoveryLogEntries($dbname), $pending), -$limit); + }; + return $this->hasLockFile($dbname) ? $this->withLock($dbname, false, $read) : $read(); + } + + /** + * check db + * if auto create db is true will be create db + * if auto create db is false and is not in db dir return false + * @param string $dbname + * @return bool + * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] + */ + public function checkDB($dbname=null){ + if($dbname === null || $dbname === '' || $dbname === false){ + return false; + } + $dbname = $this->normalizeDbName($dbname); + if(!$this->autoCreateDB){ + // Only a check: a database that doesn't exist creates nothing (no directory, no lock file) + return $this->withReadLock($dbname, function() use ($dbname){ + return $this->dbExists($dbname); + }); + } + return $this->withLock($dbname, true, function() use ($dbname){ + return $this->checkDBUnlocked($dbname); + }); + } + + private function checkDBUnlocked($dbname){ + $this->ensureDbDir(); + + // Existing database (non-sharded data file or sharded meta) + $fullDBPath = $this->getDataPath($dbname); + if($this->cachedFileExists($fullDBPath) || $this->cachedFileExists($this->getMetaPath($dbname))){ + return true; + } + + if($this->autoCreateDB){ + return $this->createDBUnlocked($dbname); + } + return false; + } + + /** + * create db function + * @param string $dbname + * @return bool True if created, false if it already existed + */ + public function createDB($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname){ + return $this->createDBUnlocked($dbname); + }); + } + + private function createDBUnlocked($dbname){ + $this->ensureDbDir(); + $fullDBPath = $this->getDataPath($dbname); + if($this->cachedFileExists($fullDBPath) || $this->cachedFileExists($this->getMetaPath($dbname))){ + return false; + } + + // Info file: the creation time (getDBs()) and an existence marker; its content never matters for + // the data, so without fsync a plain write as in 3.2 (an atomic write costs a rename per new database) + if($this->fsync){ + $this->writeFileAtomic($fullDBPath . "info", (string)time(), 'durable'); + } else { + $this->fsPutContents($fullDBPath . "info", (string)time()); + } + + // v3.0: Create empty JSONL format database + if(!$this->fsTouch($fullDBPath)){ + throw new \RuntimeException("noneDB: cannot create database file for '{$dbname}'"); + } + $this->syncForWrite(null, $this->dbDir, 'dir'); + + // Mark file as existing in cache + $this->markFileExists($fullDBPath); + $this->jsonlFormatCache[$fullDBPath] = true; + + // Empty index, checkpointed at offset 0 + $index = $this->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[$fullDBPath . '.jidx'] = $index; + $this->verifiedIndexes[$fullDBPath . '.jidx'] = true; + $this->writeCheckpoint($dbname, null, $index); + + return true; + } + + /** + * Delete a database and every file that belongs to it (data, shards, indexes, buffers, metadata) + * @param string $dbname + * @return array ['success' => bool, 'files' => int deleted file count] + */ + public function dropDB($dbname){ + $dbname = $this->normalizeDbName($dbname); + // Any file but the lock file counts: also what an interrupted dropDB left behind + if(empty($this->listDbFiles($dbname))){ + return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; + } + return $this->withLock($dbname, true, function() use ($dbname){ + $this->refuseInUncommittedWrite($dbname, 'dropDB'); + $base = $this->dbBase($dbname); + $lockPath = $this->getLockPath($dbname); + // Other processes drop all their caches of it even if this dies half-way + $this->bumpEpoch($dbname); + // Buffers first, so an interrupted drop never leaves records that a later flush would bring back; + // the files that make the database exist last, so an interrupted drop can be repeated + $files = $this->listDbFiles($dbname); + $last = [$base . '.nonedb.jidx', $base . '.nonedb', $base . '.nonedb.meta', $base . '.nonedbinfo']; + $buffers = array_values(array_filter($files, function($file){ + return strpos($file, '.nonedb.buffer') !== false; + })); + $rest = array_values(array_diff($files, $buffers, $last)); + $deleted = 0; + $failed = []; + foreach([$buffers, $rest, array_values(array_intersect($last, $files))] as $step => $group){ + foreach($group as $file){ + $this->fault('drop:unlink', ['db' => $dbname, 'file' => $file]); + if($this->fsUnlink($file)){ + $deleted++; + } elseif(is_file($file)){ + $failed[] = basename($file); + } + } + if($step === 0){ + unset(self::$lockRegistry[$lockPath]['flags']['b']); + $this->saveLockState($lockPath); + } + } + $this->invalidateDbCaches($dbname); + $this->syncForWrite(null, $this->dbDir, 'dir'); + if(!empty($failed)){ + return ['success' => false, 'files' => $deleted, 'error' => 'Could not delete: ' . implode(', ', $failed)]; + } + // Nothing is left: no buffer, no journal, no pre-3.2 files + self::$lockRegistry[$lockPath]['flags'] = ['v' => true]; + $this->saveLockState($lockPath); + return ['success' => true, 'files' => $deleted]; + }); + } + + /** + * Name, creation time and total size of a database (all shards included) + * @return array|false + */ + private function getDBInfo($dbname){ + $base = $this->dbBase($dbname); + $infoPath = $base . '.nonedbinfo'; + clearstatcache(); + if(!file_exists($infoPath)){ + return false; + } + $created = (int)@file_get_contents($infoPath); + $size = 0; + if(file_exists($base . '.nonedb')){ + $size += (int)filesize($base . '.nonedb'); + } + foreach($this->listFilesWithPrefix($base . '_s', '/^\d+\.nonedb$/') as $shardFile){ + $size += (int)filesize($shardFile); + } + $info = array("name" => $dbname, "createdTime" => $created, "size" => $this->fileSizeConvert($size)); + // Shard files found without a meta entry are kept aside (see setOrphanShardsAside) + $orphans = array_map('basename', $this->listFilesWithPrefix($base . '_s', '/\.orphan\.\d+$/')); + if(!empty($orphans)){ + $info['orphanFiles'] = $orphans; + } + return $info; + } + /** + * Convert bytes to human readable format + * @param float $bytes + * @return string + */ + private function fileSizeConvert($bytes){ + $bytes = floatval($bytes); + $arBytes = array( + 0 => array("UNIT" => "TB", "VALUE" => pow(1024, 4)), + 1 => array("UNIT" => "GB", "VALUE" => pow(1024, 3)), + 2 => array("UNIT" => "MB", "VALUE" => pow(1024, 2)), + 3 => array("UNIT" => "KB", "VALUE" => 1024), + 4 => array("UNIT" => "B", "VALUE" => 1), + ); + $result = "0 B"; + foreach($arBytes as $arItem){ + if($bytes >= $arItem["VALUE"]){ + $result = $bytes / $arItem["VALUE"]; + $result = str_replace(".", "," , strval(round($result, 2)))." ".$arItem["UNIT"]; + break; + } + } + return $result; + } + + public function getDBs($info=false){ + // Handle three cases: false (names only), true (with metadata), string (specific db) + $withMetadata = false; + $specificDb = null; + + if(is_bool($info)){ + $withMetadata = $info; + }else{ + $specificDb = $this->normalizeDbName($info); // reading information never creates the database + } + + // If specific database requested + if($specificDb !== null){ + return $this->getDBInfo($specificDb); + } + + // List all databases: one entry per {hash}-{name}.nonedb (non-sharded) or {hash}-{name}.nonedb.meta (sharded) + $dbs = []; + if(!is_dir($this->dbDir)){ + return $dbs; + } + $seen = []; + foreach(new DirectoryIterator($this->dbDir) as $item) { + if($item->isDot() || !$item->isFile()) { + continue; + } + if(!preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+)\\.nonedb(\\.meta)?$/", $item->getFilename(), $m)){ + continue; + } + $dbname = $m[2]; + // Only databases that belong to this secretKey + if($this->hashDBName($dbname) !== $m[1] || isset($seen[$dbname])){ + continue; + } + $seen[$dbname] = true; + + if($withMetadata){ + $info = $this->getDBInfo($dbname); + if($info !== false){ + $dbs[] = $info; + } + }else{ + $dbs[] = $dbname; + } + } + return $dbs; + } + + + /** + * limit function + * @param array $array Extract a slice of the array + * @param integer $limit + * @return array|false False for invalid arguments + */ + public function limit($array, $limit=0){ + if(!is_array($array) || !is_int($limit) || $limit <= 0){ + return false; + } + if(empty($array)){ + return []; + } + // Must be a list of records + if(count($array) === count($array, COUNT_RECURSIVE)) { + return false; + } + return array_slice($array, 0, $limit); + } + + /** + * Run a read operation: shared lock while the database exists, no lock (and no files created) + * when it doesn't. Buffered records are flushed first. + */ + private function withReadLock($dbname, callable $fn){ + if(!$this->hasLockFile($dbname) && !$this->dbFilesExist($dbname)){ + return $fn(); + } + return $this->withLock($dbname, false, function() use ($dbname, $fn){ + $this->flushPendingBuffers($dbname); + return $fn(); + }); + } + + /** + * Uncached existence check for a database without a lock file, used before taking a lock. Since 3.2 + * every database has a lock file (never deleted) and an interrupted flush is {buffer}.flushing.{key}; + * older versions had no lock file and named it {buffer}.flushing. So no directory listing is needed + * here (a read of a missing database would list the whole directory each time). + */ + private function dbFilesExist($dbname){ + $base = $this->dbBase($dbname); + foreach(['.nonedb', '.nonedb.meta', '.nonedb.buffer', '.nonedb.buffer.flushing'] as $suffix){ + clearstatcache(true, $base . $suffix); + if(file_exists($base . $suffix)){ + return true; + } + } + return false; + } + + /** + * True when the database's lock file exists (the database may have been dropped since). Taking + * that lock creates no file, and the operation itself finds out whether the database exists. + */ + private function hasLockFile($dbname){ + $path = $this->getLockPath($dbname); + if(isset(self::$lockHandles[$path]) || isset(self::$lockRegistry[$path])){ + return true; + } + clearstatcache(true, $path); + return file_exists($path); + } + + /** + * read db all data + * @param string $dbname + * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); + * a "key" entry selects by record key (int or list of ints) + * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + */ + public function find($dbname, $filters=0){ + $dbname = $this->normalizeDbName($dbname); + if($filters === null || is_int($filters)){ + $filters = []; + } + if(!is_array($filters)){ + return false; + } + return $this->withReadLock($dbname, function() use ($dbname, $filters){ + return $this->findUnlocked($dbname, $filters); + }); + } + + private function findUnlocked($dbname, $filters=0){ + if($filters === null || is_int($filters)){ + $filters = []; + } + if(!$this->dbExists($dbname)){ + // Reads never create databases + return $this->autoCreateDB ? [] : false; + } + return array_values($this->selectRecords($dbname, $filters)); + } + + /** + * Check if 'key' exists at top level of data (not nested) + * @param array $data + * @return bool + */ + private function hasReservedKeyField($data){ + return is_array($data) && array_key_exists("key", $data); + } + + /** + * Check if array is a list of records: sequential 0..n-1 keys, every element an array + * @param array $data + * @return bool + */ + private function isRecordList($data){ + if(!is_array($data) || count($data) === 0){ + return false; + } + $i = 0; + foreach($data as $k => $v){ + if($k !== $i++ || !is_array($v)){ + return false; + } + } + return true; + } + + /** + * Validate insert data + * @return array [items, error|null] + */ + private function validateInsertData($data){ + if(!is_array($data)){ + return [[], "insert data must be array"]; + } + if(empty($data)){ + return [[], "insert data must not be empty"]; + } + $keys = array_keys($data); + $looksLikeList = is_int($keys[0]) && is_array($data[$keys[0]]); + if($looksLikeList && !$this->isRecordList($data)){ + return [[], "insert data must be either one record or a list of records"]; + } + $items = $looksLikeList ? array_values($data) : [$data]; + return $this->withExactFloats(function() use ($items){ + foreach($items as $item){ + if($this->hasReservedKeyField($item)){ + return [[], "You cannot set key name to key"]; + } + if($this->encodeJson($item) === false){ + return [[], "insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"]; + } + } + return [$items, null]; + }); + } + + /** + * insert to db + * @param string $dbname + * @param array $data One record, or a list of records + * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] + */ + public function insert($dbname, $data){ + $dbname = $this->normalizeDbName($dbname); + list($items, $error) = $this->validateInsertData($data); + if($error !== null){ + return array("n" => 0, "error" => $error); + } + return $this->withLock($dbname, true, function() use ($dbname, $items){ + if($this->bufferEnabled){ + return $this->insertBuffered($dbname, $items); + } + $this->flushPendingBuffers($dbname); + return $this->insertRecords($dbname, $items); + }); + } + + /** + * delete function + * @param string $dbname + * @param array $data Filters (same rules as find); [] deletes every record + * @return array ['n' => deleted count] + */ + public function delete($dbname, $data){ + $dbname = $this->normalizeDbName($dbname); + if(!is_array($data)){ + return array("n" => 0, "error" => "Please check your delete parameters"); + } + return $this->withLock($dbname, true, function() use ($dbname, $data){ + return $this->deleteUnlocked($dbname, $data); + }); + } + + private function deleteUnlocked($dbname, $data){ + $this->flushPendingBuffers($dbname); + if(!$this->dbExists($dbname)){ + return array("n" => 0); + } + $deleted = 0; + foreach($this->groupBySegment($dbname, $this->selectRecords($dbname, $data)) as $segment => $records){ + $deleted += $this->deleteSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $records); + } + return array("n" => $deleted); + } + + /** + * update function + * @param string $dbname + * @param array $data [filters, ['set' => [field => value, ...]]] + * @return array ['n' => updated count] + */ + public function update($dbname, $data){ + $dbname = $this->normalizeDbName($dbname); + if(!is_array($data) || !array_key_exists(0, $data) || !is_array($data[0]) + || !isset($data[1]) || !is_array($data[1]) || !isset($data[1]['set']) || !is_array($data[1]['set']) + || array_key_exists("key", $data[1]['set'])){ + return array("n" => 0, "error" => "Please check your update parameters"); + } + if($this->encodeJson($data[1]['set']) === false){ + return array("n" => 0, "error" => "update data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"); + } + return $this->withLock($dbname, true, function() use ($dbname, $data){ + return $this->updateUnlocked($dbname, $data); + }); + } + + private function updateUnlocked($dbname, $data){ + $this->flushPendingBuffers($dbname); + if(!$this->dbExists($dbname)){ + return array("n" => 0); + } + $set = $data[1]['set']; + if(empty($set)){ + return array("n" => count($this->selectRecords($dbname, $data[0]))); + } + $updated = 0; + foreach($this->groupBySegment($dbname, $this->selectRecords($dbname, $data[0])) as $segment => $records){ + $new = []; + foreach($records as $key => $record){ + foreach($set as $field => $value){ + $record[$field] = $value; + } + $new[$key] = $record; + } + $updated += $this->updateSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $records, $new); + } + return array("n" => $updated); + } + + /** + * Replace whole records by key (used by noneDBQuery::removeFields); caller holds the exclusive lock + * @param array $records [key => new record without "key"] + * @return int + */ + private function replaceRecordsByKey($dbname, array $records){ + if(empty($records) || !$this->dbExists($dbname)){ + return 0; + } + $old = $this->selectRecords($dbname, ['key' => array_keys($records)]); + $written = 0; + foreach($this->groupBySegment($dbname, $old) as $segment => $oldRecords){ + $new = []; + foreach(array_keys($oldRecords) as $key){ + $new[$key] = $records[$key]; + } + $written += $this->updateSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $oldRecords, $new); + } + return $written; + } + + // ========================================== + // QUERY METHODS + // ========================================== + + /** + * Get unique values for a field + * @param string $dbname + * @param string $field + * @return array|false + */ + public function distinct($dbname, $field){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field){ + return $this->distinctUnlocked($dbname, $field); + }); + } + + private function distinctUnlocked($dbname, $field){ + $all = $this->findUnlocked($dbname, []); + if($all === false) return false; + return self::uniqueValues($all, $field); + } + + /** + * Distinct values of a field (null excluded) in first-seen order, same identity as + * in_array($value, $values, true), in linear time (a hash set of valueKey()s) + */ + private static function uniqueValues(array $records, $field){ + $seen = []; + $values = []; + foreach($records as $record){ + if(isset($record[$field])){ + $id = self::valueKey($record[$field]); + if(!isset($seen[$id])){ + $seen[$id] = true; + $values[] = $record[$field]; + } + } + } + return $values; + } + + /** + * A string that is equal for two values exactly when they are identical (===): typed, -0.0 is 0.0, + * arrays by their typed keys and values in order (lengths make the encoding unambiguous). + * Floats by their bits, so neither precision nor locale settings matter. + */ + private static function valueKey($v){ + if($v === null) return 'n'; + if($v === true) return 't'; + if($v === false) return 'f'; + if(is_int($v)) return 'i' . $v; + if(is_float($v)) return 'd' . bin2hex(pack('E', $v + 0.0)); + if(is_string($v)) return 's' . $v; + if(is_array($v)){ + $key = 'a' . count($v) . ':'; + foreach($v as $k => $item){ + $kk = (is_int($k) ? 'i' : 's') . $k; + $id = self::valueKey($item); + $key .= strlen($kk) . ':' . $kk . strlen($id) . ':' . $id; + } + return $key; + } + return is_object($v) ? 'o' . spl_object_id($v) : 'r' . (int)$v; // identical only to itself + } + + /** + * Sort array by field + * @param array $array Result from find() + * @param string $field Field to sort by + * @param string $order 'asc' or 'desc' + * @return array|false + */ + public function sort($array, $field, $order = 'asc'){ + if(!is_array($array)) return false; + if(count($array) === 0) return []; + $desc = strtolower((string)$order) === 'desc'; + $items = []; + foreach(array_values($array) as $i => $row){ + $items[] = [self::sortKey(is_array($row) && array_key_exists($field, $row) ? $row[$field] : null), $i, $row]; + } + usort($items, function($a, $b) use ($desc){ + $result = $a[0] <=> $b[0]; + if($desc){ + $result = -$result; + } + return $result !== 0 ? $result : $a[1] <=> $b[1]; // stable + }); + return array_column($items, 2); + } + + /** + * Total order used by sort(), min() and max() (and the query builder): missing/null < booleans < + * numbers < numeric strings (by their number) < other strings (byte order) < arrays. + * @return int + */ + private static function compareSortValues($a, $b){ + return self::sortKey($a) <=> self::sortKey($b); + } + + /** + * Sort key of a value: [kind, comparable value]. Two keys compare with <=> in the order above + * (kind first, then the value; arrays by their JSON), so sorts compute each key once instead of + * classifying both values in every comparison. + * @return array + */ + private static function sortKey($v){ + if($v === null) return [0, 0]; + if(is_bool($v)) return [1, $v]; + if(is_int($v) || is_float($v)) return [2, $v]; + if(is_string($v)){ + // One definition of "numeric string" on every PHP version ("5 " is numeric since 8.0 only). + // Numeric strings never meet other strings, so <=> never mixes numeric and byte comparison + // (which made "9" < "10" < "1a" < "9" a cycle before v3.3). + $trimmed = rtrim($v, " \t\n\r\v\f"); + return is_numeric($trimmed) ? [3, $trimmed + 0] : [4, $v]; + } + return [5, (string)json_encode($v)]; + } + + /** + * sortKey() for the query builder + * @return \Closure + */ + private function sortKeyFunction(){ + return \Closure::fromCallable([self::class, 'sortKey']); + } + + /** + * Count records matching filter + * @param string $dbname + * @param mixed $filter + * @return int + */ + public function count($dbname, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->countUnlocked($dbname, $filter); + }); + } + + private function countUnlocked($dbname, $filter = 0){ + $filters = $this->normalizeFilter($filter); + if($filters === null){ + return 0; // a filter find() rejects matches nothing (a TypeError before v3.3) + } + // Fast-path: no filter = use index/meta counters + if(empty($filters)){ + return $this->countFast($dbname); + } + $count = 0; + if($this->scanMatches($dbname, $filters, function() use (&$count){ $count++; return true; }, true)){ + return $count; // counted record by record, none kept (v3.3) + } + $result = $this->findUnlocked($dbname, $filters); + return $result === false ? 0 : count($result); + } + + /** + * Count without reading records: index entries (non-sharded) or meta totalRecords (sharded) + * @param string $dbname Already normalized + * @return int + */ + private function countFast($dbname){ + if(!$this->dbExists($dbname)){ + return 0; + } + if($this->isSharded($dbname)){ + $meta = $this->getCachedMeta($dbname); + return $meta !== null ? (int)($meta['totalRecords'] ?? 0) : 0; + } + $index = $this->readJsonlIndex($dbname, null); + return $index !== null ? count($index['p']) : 0; + } + + /** + * Pattern matching search (LIKE) + * @param string $dbname + * @param string $field + * @param string $pattern Use ^ for starts with, $ for ends with + * @return array|false + */ + public function like($dbname, $field, $pattern){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $pattern){ + return $this->likeUnlocked($dbname, $field, $pattern); + }); + } + + private function likeUnlocked($dbname, $field, $pattern){ + if(!is_scalar($pattern)){ + return false; // "Array to string" before v3.3 + } + $all = $this->findUnlocked($dbname, []); + if($all === false) return false; + + $result = []; + $like = noneDBQuery::compileLike((string)$pattern); + foreach($all as $record){ + if(!isset($record[$field])) continue; + $value = $record[$field]; + // Only strings and numbers (true used to match "1", like in the query builder it never does) + if(!is_scalar($value) || is_bool($value)) continue; + if(noneDBQuery::likeMatch($like, (string)$value)){ + $result[] = $record; + } + } + return $result; + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Sum numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function sum($dbname, $field, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->sumUnlocked($dbname, $field, $filter); + }); + } + + private function sumUnlocked($dbname, $field, $filter = 0){ + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false) return 0; + $sum = 0; + foreach($result as $record){ + $number = \noneDBQuery::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + } + } + return $sum; + } + + /** + * Average of numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function avg($dbname, $field, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->avgUnlocked($dbname, $field, $filter); + }); + } + + private function avgUnlocked($dbname, $field, $filter = 0){ + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false || count($result) === 0) return 0; + $sum = 0; + $count = 0; + foreach($result as $record){ + $number = \noneDBQuery::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + $count++; + } + } + return $count > 0 ? $sum / $count : 0; + } + + /** + * Get minimum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function min($dbname, $field, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->minUnlocked($dbname, $field, $filter); + }); + } + + private function minUnlocked($dbname, $field, $filter = 0){ + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false || count($result) === 0) return null; + // The order of sort() (and of the query builder's min()); loose < depended on the insertion order + $min = null; + foreach($result as $record){ + if(isset($record[$field])){ + if($min === null || self::compareSortValues($record[$field], $min) < 0){ + $min = $record[$field]; + } + } + } + return $min; + } + + /** + * Get maximum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function max($dbname, $field, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->maxUnlocked($dbname, $field, $filter); + }); + } + + private function maxUnlocked($dbname, $field, $filter = 0){ + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false || count($result) === 0) return null; + $max = null; + foreach($result as $record){ + if(isset($record[$field])){ + if($max === null || self::compareSortValues($record[$field], $max) > 0){ + $max = $record[$field]; + } + } + } + return $max; + } + + // ========================================== + // UTILITY METHODS + // ========================================== + + /** + * Get first matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function first($dbname, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->firstUnlocked($dbname, $filter); + }); + } + + private function firstUnlocked($dbname, $filter = 0){ + $filters = $this->normalizeFilter($filter); + $first = null; + if($filters !== null && $this->scanMatches($dbname, $filters, function($key, $record) use (&$first){ $first = $record; return false; })){ + return $first; // stopped at the first match (v3.3) + } + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false || count($result) === 0) return null; + return $result[0]; + } + + /** + * Get last matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function last($dbname, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->lastUnlocked($dbname, $filter); + }); + } + + private function lastUnlocked($dbname, $filter = 0){ + $result = $this->filteredUnlocked($dbname, $filter); + if($result === false || count($result) === 0) return null; + return $result[count($result) - 1]; + } + + /** + * Check if records exist matching filter + * @param string $dbname + * @param mixed $filter + * @return bool + */ + public function exists($dbname, $filter){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->existsUnlocked($dbname, $filter); + }); + } + + private function existsUnlocked($dbname, $filter){ + $filters = $this->normalizeFilter($filter); + $found = false; + if($filters !== null && $this->scanMatches($dbname, $filters, function() use (&$found){ $found = true; return false; })){ + return $found; + } + $result = $this->filteredUnlocked($dbname, $filter); + return $result !== false && count($result) > 0; + } + + /** + * Range query (min <= value <= max) + * @param string $dbname + * @param string $field + * @param mixed $min + * @param mixed $max + * @param mixed $filter Additional filter + * @return array|false + */ + public function between($dbname, $field, $min, $max, $filter = 0){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $min, $max, $filter){ + return $this->betweenUnlocked($dbname, $field, $min, $max, $filter); + }); + } + + private function betweenUnlocked($dbname, $field, $min, $max, $filter = 0){ + // An indexed field: only the records whose value the field index puts in range are read (v3.3) + $result = $this->rangeCandidateRecords($dbname, $field, ['$gte' => $min, '$lte' => $max], $filter); + if($result === null){ + $result = $this->filteredUnlocked($dbname, $filter); + } + if($result === false) return false; + // Same comparison as query()->between(): numbers (and numeric strings) numerically, strings + // lexically; null, booleans, arrays and number-vs-text are never in range + $filtered = []; + foreach($result as $record){ + if(!array_key_exists($field, $record)){ + continue; + } + $lo = noneDBQuery::compareValues($record[$field], $min); + $hi = noneDBQuery::compareValues($record[$field], $max); + if($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0){ + $filtered[] = $record; + } + } + return $filtered; + } + + + /** + * Records matching a filter parameter whose field value may satisfy the range conditions, read + * through the field index of every segment that has one (a superset: the caller checks the range). + * Null when the field has no index or a key filter decides (the caller reads everything). + * @return array|false|null + */ + private function rangeCandidateRecords($dbname, $field, array $conditions, $filter){ + $filters = $this->normalizeFilter($filter); + if($filters === null){ + return false; + } + if(array_key_exists('key', $filters) || !$this->dbExists($dbname) || !$this->fieldIndexEnabled || !$this->isFieldIndexed($dbname, $field)){ + return null; + } + $result = []; + foreach($this->listSegments($dbname) as $segment){ + $keys = $this->fieldIndexRangeCandidates($dbname, $segment, $field, $conditions); + if($keys !== null && empty($keys)){ + continue; + } + foreach($this->readSegmentRecords($dbname, $segment, $keys) as $key => $record){ + if(empty($filters) || $this->matchesFilters($record, $filters)){ + $result[$key] = $record; + } + } + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + // ========================================== + // SHARDING PUBLIC METHODS + // ========================================== + + /** + * Get sharding information for a database + * @param string $dbname + * @return array|false + */ + public function getShardInfo($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname){ + return $this->getShardInfoUnlocked($dbname); + }); + } + + private function getShardInfoUnlocked($dbname){ + if(!$this->dbExists($dbname)){ + return false; + } + if(!$this->isSharded($dbname)){ + $index = $this->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + return array( + "sharded" => false, + "shards" => 0, + "totalRecords" => count($index['p']), + "shardSize" => $this->shardSize + ); + } + + $meta = $this->getCachedMeta($dbname); + if($meta === null){ + return false; + } + return array( + "sharded" => true, + "shards" => count($meta['shards']), + "totalRecords" => (int)$meta['totalRecords'], + "deletedCount" => (int)($meta['deletedCount'] ?? 0), + "shardSize" => (int)$meta['shardSize'], + "nextKey" => (int)$meta['nextKey'] + ); + } + // ========================================== + // FIELD INDEX PUBLIC API (v3.0.0) + // ========================================== + + /** + * Create an index on a field for faster equality lookups (find / where with field => value) + * @param string $dbname Database name + * @param string $field Field name (any name except "key") + * @return array ['success' => bool, 'indexed' => int, 'values' => int, 'error' => ?string] + */ + public function createFieldIndex($dbname, $field){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->createFieldIndexUnlocked($dbname, $field); + }); + } + + private function createFieldIndexUnlocked($dbname, $field){ + if(!is_string($field) || $field === '' || $field === 'key'){ + return ['success' => false, 'indexed' => 0, 'values' => 0, 'error' => 'Invalid field name']; + } + $nameError = $this->indexNameError($dbname, $field); + if($nameError !== null){ + return ['success' => false, 'indexed' => 0, 'values' => 0, 'error' => $nameError]; + } + if($this->dbExists($dbname) && $this->isFieldIndexed($dbname, $field)){ + $this->dropFieldIndexUnlocked($dbname, $field); // creating it again rebuilds it (as in 3.2) + } + $this->flushPendingBuffers($dbname); + if(!$this->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ['success' => false, 'indexed' => 0, 'values' => 0, 'error' => 'Database does not exist']; + } + $this->createDBUnlocked($dbname); + } + + // Distinct values per type: 1, 1.0, "1" and true count as four (v3.3: indexes are exact) + $indexed = 0; + $values = []; + $segments = $this->listSegments($dbname); + foreach($segments as $segment){ + $this->deleteFieldIndexFile($dbname, $field, $segment); // leftovers of an earlier index of this field + $index = $this->buildSegmentFieldIndex($dbname, $segment, $field); + $this->writeFieldIndex($dbname, $field, $index, $segment); + foreach($index['b'] as $bucket => $keys){ + $indexed += is_int($keys) ? 1 : count($keys); + $values[(string)$bucket] = true; + } + } + if($this->isSharded($dbname)){ + $this->deleteGlobalFieldIndex($dbname, $field); + $this->rebuildGlobalFieldIndex($dbname, $field); + } + + $manifest = $this->getIndexManifest($dbname); + $manifest['fields'][] = $field; + $this->saveIndexManifest($dbname, $manifest); + + $result = ['success' => true, 'indexed' => $indexed, 'values' => count($values), 'error' => null]; + if($this->isSharded($dbname)){ + $result['shards'] = count($segments); + } + return $result; + } + + /** + * Drop a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => string|null] + */ + public function dropFieldIndex($dbname, $field){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropFieldIndexUnlocked($dbname, $field); + }); + } + + private function dropFieldIndexUnlocked($dbname, $field){ + $manifest = $this->getIndexManifest($dbname); + if(!in_array((string)$field, $manifest['fields'], true)){ + return ['success' => false, 'error' => 'Index does not exist']; + } + $segments = $this->listSegments($dbname); + foreach($segments as $segment){ + $this->deleteFieldIndexFile($dbname, $field, $segment); + // Also remove a lossy pre-3.2 file name for this field, if any + $legacy = $this->segDataPath($dbname, $segment) . '.fidx.' . preg_replace('/[^a-zA-Z0-9_]/', '_', (string)$field); + if($legacy !== $this->getLegacyFieldIndexPath($dbname, $field, $segment) && is_file($legacy)){ + $old = $this->atomicRead($legacy, null); + if(is_array($old) && ($old['field'] ?? null) === (string)$field){ + $this->fsUnlink($legacy); + } + } + } + $this->deleteGlobalFieldIndex($dbname, $field); + $manifest['fields'] = array_values(array_diff($manifest['fields'], [(string)$field])); + $this->saveIndexManifest($dbname, $manifest); + $this->invalidateFieldIndexCache($dbname, $field); + return ['success' => true, 'error' => null]; + } + + /** + * Check if a field has an index + * @param string $dbname Database name + * @param string $field Field name + * @return bool + */ + public function hasFieldIndex($dbname, $field){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field){ + return $this->dbExists($dbname) && $this->isFieldIndexed($dbname, $field); + }); + } + + /** + * Get list of field indexes for a database + * @param string $dbname Database name + * @return array ['fields' => array, 'sharded' => bool] + */ + public function getFieldIndexes($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname){ + return $this->getFieldIndexesUnlocked($dbname); + }); + } + + private function getFieldIndexesUnlocked($dbname){ + if(!$this->dbExists($dbname)){ + return ['fields' => [], 'sharded' => false]; + } + return ['fields' => $this->getIndexedFields($dbname), 'sharded' => $this->isSharded($dbname)]; + } + + /** + * Rebuild a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createFieldIndex + */ + public function rebuildFieldIndex($dbname, $field){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildFieldIndexUnlocked($dbname, $field); + }); + } + + private function rebuildFieldIndexUnlocked($dbname, $field){ + if($this->isFieldIndexed($dbname, $field)){ + $this->dropFieldIndexUnlocked($dbname, $field); + } + return $this->createFieldIndexUnlocked($dbname, $field); + } + + /** + * Get keys matching a field value using field index + * Returns null if no index exists (caller should fall back to scan) + * @param string $dbname Database name + * @param string $field Field name + * @param mixed $value Value to match + * @param int|null $shardId Shard ID for sharded databases (null = all shards) + * @return array|null Array of matching keys, or null if no index + */ + public function getKeysFromFieldIndex($dbname, $field, $value, $shardId = null){ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $value, $shardId){ + return $this->getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId); + }); + } + + private function getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId = null){ + if(!$this->dbExists($dbname) || !$this->isFieldIndexed($dbname, $field) || !$this->isIndexableValue($value)){ + return null; + } + $segments = $shardId !== null ? [(int)$shardId] : $this->listSegments($dbname); + $keys = []; + foreach($segments as $segment){ + if(!in_array($segment, $this->listSegments($dbname), true)){ + continue; + } + $index = $this->ensureSegmentFieldIndex($dbname, $segment, $field); + $found = $index === null ? null : $this->fieldIndexKeys($index, $value); + if($found === null || !empty($index['legacy'])){ + // No usable index (or a v3.2 one, whose buckets mix types): the records decide, exactly + foreach($found === null ? $this->readSegmentRecords($dbname, $segment) : $this->readSegmentRecords($dbname, $segment, $found) as $key => $record){ + if(array_key_exists($field, $record) && $record[$field] === $value){ + $keys[] = (int)$key; + } + } + continue; + } + foreach($found as $key){ + $keys[] = (int)$key; + } + } + sort($keys); + return $keys; + } + + + /** + * Convert the index files of v3.2 that are still in use (v3.3.0): one segment at a time, each in a + * short exclusive session of its own, so other processes never wait long. Safe to run again or to stop + * half-way: the next run continues. Without $dbname: every database of this secret key in dbDir. + * Not needed for correctness: a segment is converted when it is first written; until then its v3.2 + * files are read (their global shard summaries are not used to skip shards). + * @param string|null $dbname + * @return array ['databases' => int, 'segments' => int, 'converted' => int, 'ms' => float] + */ + public function upgradeIndexes($dbname = null){ + $start = microtime(true); + $names = $dbname === null ? (array)$this->getDBs(false) : [$dbname]; + $report = ['databases' => 0, 'segments' => 0, 'converted' => 0, 'ms' => 0.0]; + foreach($names as $name){ + $name = $this->normalizeDbName($name); + $segments = $this->withReadLock($name, function() use ($name){ + return $this->dbExists($name) ? $this->listSegments($name) : []; + }); + if(empty($segments)){ + continue; + } + $report['databases']++; + foreach($segments as $segment){ + $report['segments']++; + $converted = $this->withLock($name, true, function() use ($name, $segment){ + return in_array($segment, $this->listSegments($name), true) && $this->convertSegmentIndexes($name, $segment); + }); + if($converted){ + $report['converted']++; + } + } + // Nothing left to convert (or nothing ever was): the index list says so, no file is looked for again + $this->withLock($name, true, function() use ($name){ + $manifest = $this->getIndexManifest($name); + if($manifest['fmt'] || (empty($manifest['fields']) && empty($manifest['spatial']))){ + return; + } + foreach($this->listSegments($name) as $segment){ + if(!$this->segmentIsConverted($name, $segment)){ + return; + } + } + $manifest['fmt'] = true; + $this->saveIndexManifest($name, $manifest); + }); + } + $report['ms'] = round((microtime(true) - $start) * 1000, 2); + return $report; + } + + // ========================================== + // SPATIAL INDEX - GEOJSON SUPPORT (v3.1.0) + // ========================================== + + /** + * Validate a GeoJSON geometry object + * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection + * + * @param array $geometry GeoJSON geometry object + * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] + */ + public function validateGeoJSON($geometry): array { + self::$geometryValidations++; + // Never throws: a stored record that fails here is simply not indexed (v3.3.0: before, a malformed + // geometry made every later operation on the database fail) + if (!is_array($geometry)) { + return ['valid' => false, 'error' => 'Geometry must be an array', 'type' => null]; + } + if (!isset($geometry['type'])) { + return ['valid' => false, 'error' => 'Missing type property', 'type' => null]; + } + + $type = $geometry['type']; + if (!is_string($type)) { + return ['valid' => false, 'error' => 'Geometry type must be a string', 'type' => null]; + } + + // GeometryCollection has different structure + if ($type === 'GeometryCollection') { + return $this->validateGeometryCollection($geometry); + } + + if (!isset($geometry['coordinates'])) { + return ['valid' => false, 'error' => 'Missing coordinates property', 'type' => null]; + } + + $coords = $geometry['coordinates']; + + // Each validator checks the lists it needs (a Point needs none: values at 0 and 1) + if (!is_array($coords)) { + return ['valid' => false, 'error' => 'Coordinates must be an array', 'type' => null]; + } + + // Strict comparison (a switch compared loosely: 0 matched 'Point' before PHP 8) + if ($type === 'Point') { + return $this->validateGeoJSONPoint($coords); + } + if ($type === 'LineString') { + return $this->validateGeoJSONLineString($coords); + } + if ($type === 'Polygon') { + return $this->validateGeoJSONPolygon($coords); + } + if ($type === 'MultiPoint') { + return $this->validateGeoJSONMultiPoint($coords); + } + if ($type === 'MultiLineString') { + return $this->validateGeoJSONMultiLineString($coords); + } + if ($type === 'MultiPolygon') { + return $this->validateGeoJSONMultiPolygon($coords); + } + return ['valid' => false, 'error' => "Unknown geometry type: $type", 'type' => null]; + } + + /** + * An array with keys 0..n-1 in order (JSON array), PHP 7.4 compatible array_is_list() + */ + private static function isList($value): bool { + if (!is_array($value)) { + return false; + } + if (function_exists('array_is_list')) { + return array_is_list($value); + } + $i = 0; + foreach ($value as $key => $unused) { + if ($key !== $i++) { + return false; + } + } + return true; + } + + /** + * Validate Point coordinates [lon, lat] or [lon, lat, altitude] + */ + private function validateGeoJSONPoint($coords): array { + // A position: values at 0 and 1 (checked below) are enough, no list check per position + if (!is_array($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'Point must have at least 2 coordinates', 'type' => null]; + } + if (!isset($coords[0], $coords[1]) || !is_numeric($coords[0]) || !is_numeric($coords[1]) + || is_string($coords[0]) || is_string($coords[1]) || !is_finite((float)$coords[0]) || !is_finite((float)$coords[1])) { + return ['valid' => false, 'error' => 'Point coordinates must be finite numbers', 'type' => null]; + } + $lon = (float)$coords[0]; + $lat = (float)$coords[1]; + if ($lon < -180 || $lon > 180) { + return ['valid' => false, 'error' => 'Longitude must be between -180 and 180', 'type' => null]; + } + if ($lat < -90 || $lat > 90) { + return ['valid' => false, 'error' => 'Latitude must be between -90 and 90', 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Point']; + } + + /** + * Validate LineString coordinates [[lon,lat], [lon,lat], ...] + */ + private function validateGeoJSONLineString($coords): array { + if (!self::isList($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'LineString must have at least 2 positions', 'type' => null]; + } + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid position at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'LineString']; + } + + /** + * Validate Polygon coordinates [[[lon,lat], ...], ...] + * First ring is outer boundary, subsequent rings are holes + */ + private function validateGeoJSONPolygon($coords): array { + if (!self::isList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'Polygon must have at least one ring', 'type' => null]; + } + // Rings in order. The holes before the first failing ring are checked against the outer ring + // together (one sweep for all their vertices); the first error is the same as ring by ring. + $failure = null; + $holes = []; + foreach ($coords as $ringIndex => $ring) { + if (!self::isList($ring) || count($ring) < 4) { + $failure = "Ring $ringIndex must have at least 4 positions"; + break; + } + // Validate each position + foreach ($ring as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + $failure = "Invalid position at ring $ringIndex, index $i: " . $result['error']; + break 2; + } + } + // Check if ring is closed (first == last, compared numerically: 29 and 29.0 are the same) + $first = $ring[0]; + $last = $ring[count($ring) - 1]; + if ((float)$first[0] != (float)$last[0] || (float)$first[1] != (float)$last[1]) { + $failure = "Ring $ringIndex must be closed (first and last positions must be identical)"; + break; + } + if ($this->ringSelfIntersects($ring)) { + $failure = "Ring $ringIndex must not intersect itself"; + break; + } + if ($ringIndex > 0) { + $holes[$ringIndex] = $ring; + } + } + if ($holes) { + // Every vertex of a hole inside the outer ring or on it + $points = []; + $owner = []; + foreach ($holes as $ringIndex => $ring) { + foreach ($ring as $position) { + $points[] = $position; + $owner[] = $ringIndex; + } + } + foreach ($this->locatePointsInRing($coords[0], $points) as $k => $where) { + if ($where < 0) { + return ['valid' => false, 'error' => "Hole {$owner[$k]} must lie inside the outer ring", 'type' => null]; + } + } + } + if ($failure !== null) { + return ['valid' => false, 'error' => $failure, 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Polygon']; + } + + /** + * True when two non-adjacent edges of a closed ring share a point (they cross, touch or overlap; + * exact arithmetic). First a sort and sweep by x that compares the edges whose x ranges overlap: + * fast for usual rings (a 2000-vertex polygon took seconds per query before v3.3). When that work + * passes 16 edge pairs per edge (rings built against it, e.g. a comb whose edges all overlap in x), + * a sweep line over a balanced tree finishes the check: O(n log n) for every ring. + */ + private function ringSelfIntersects(array $ring): bool { + $n = count($ring) - 1; // edges; edge i runs from vertex i to vertex i + 1 + if ($n < 4) { + return false; // every pair of a triangle's edges is adjacent + } + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; + } + $budget = self::$sweepBudget ?? 16 * $n + 256; + if ($budget === 0) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + $minX = []; + $maxX = []; + for ($i = 0; $i < $n; $i++) { + $a = $xs[$i]; + $b = $xs[$i + 1]; + $minX[$i] = $a < $b ? $a : $b; + $maxX[$i] = $a < $b ? $b : $a; + } + asort($minX, SORT_NUMERIC); + $active = []; // edge => max x, edges whose x range may still overlap the next ones + $work = 0; + foreach ($minX as $i => $from) { + foreach ($active as $j => $to) { + if ($to < $from) { + unset($active[$j]); + continue; + } + $d = $i > $j ? $i - $j : $j - $i; + if ($d === 1 || $d === $n - 1) { + continue; // neighbours share a vertex (the first and the last edge share the closing one) + } + if (++$work > $budget) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + self::$segmentPairTests++; + if (self::segmentsMeet($xs[$i], $ys[$i], $xs[$i + 1], $ys[$i + 1], $xs[$j], $ys[$j], $xs[$j + 1], $ys[$j + 1])) { + return true; + } + } + $active[$i] = $maxX[$i]; + } + return false; + } + + /** + * ringSelfIntersects() as a sweep line (Shamos-Hoey) over an AVL tree of the edges that cross it, + * ordered from bottom to top: O(n log n). Events are the vertices in (x, y) order. A ring that doesn't + * intersect itself has at every event point exactly one vertex, so exactly its two edges end or + * start there and no other edge passes through it; between events two edges can only cross after + * they became neighbours in the tree, and every new pair of neighbours is tested. + */ + private function ringSelfIntersectsSweep(array $xs, array $ys, int $n): bool { + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $tree = self::avlTree(array_fill(0, $n, 0)); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi, self::$segmentPairTests); + $meet = function (int $a, int $b) use ($xs, $ys, $n): bool { + if ($a < 0 || $b < 0) { + return false; + } + $d = $a > $b ? $a - $b : $b - $a; + if ($d === 1 || $d === $n - 1) { + return false; + } + self::$segmentPairTests++; + return self::segmentsMeet($xs[$a], $ys[$a], $xs[$a + 1], $ys[$a + 1], $xs[$b], $ys[$b], $xs[$b + 1], $ys[$b + 1]); + }; + $count = count($ee); + for ($k = 0; $k < $count;) { + $px = $ex[$k]; + $py = $ey[$k]; + $edges = []; + $starts = []; + $ends = []; + for (; $k < $count && $ex[$k] == $px && $ey[$k] == $py; $k++) { + $edge = $ee[$k] >> 1; + $edges[$edge] = true; + if ($ee[$k] & 1) { + $ends[] = $edge; + } else { + $starts[] = $edge; + } + } + // One vertex: its two edges, adjacent (a repeated vertex or a zero-length edge gives more) + if (count($edges) !== 2) { + return true; + } + [$e1, $e2] = array_keys($edges); + $d = $e1 > $e2 ? $e1 - $e2 : $e2 - $e1; + if ($d !== 1 && $d !== $n - 1) { + return true; + } + foreach ($ends as $edge) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $edge, $cmp); + } + // No other edge through the vertex; the edges below and above it become neighbours + $below = -1; + $above = -1; + for ($node = $tree['root']; $node >= 0;) { + self::$segmentPairTests++; + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $px, $py); + if ($c === 0) { + return true; // the vertex lies on another edge + } + if ($c > 0) { + $below = $node; + $node = $tree['r'][$node]; + } else { + $above = $node; + $node = $tree['l'][$node]; + } + } + if (!$starts) { + if ($meet($below, $above)) { + return true; + } + continue; + } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + if ($meet($pred, $edge) || $meet($edge, $succ)) { + return true; + } + } + } + return false; + } + + /** + * Sweep events of a ring's edges: [lo, hi, x, y, code] with lo/hi the vertex numbers of each edge's + * smaller and larger end in (x, y) order, and the events (both ends of every edge, code edge * 2 + * for the start, edge * 2 + 1 for the end) sorted by x, then y + */ + private static function sweepEvents(array $xs, array $ys, int $n): array { + $lo = []; + $hi = []; + $ex = []; + $ey = []; + $ee = []; + for ($i = 0; $i < $n; $i++) { + $a = $i; + $b = $i + 1; + if ($xs[$b] < $xs[$a] || ($xs[$b] == $xs[$a] && $ys[$b] < $ys[$a])) { + $a = $i + 1; + $b = $i; + } + $lo[$i] = $a; + $hi[$i] = $b; + $ex[] = $xs[$a]; + $ey[] = $ys[$a]; + $ee[] = $i * 2; + $ex[] = $xs[$b]; + $ey[] = $ys[$b]; + $ee[] = $i * 2 + 1; + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + return [$lo, $hi, $ex, $ey, $ee]; + } + + /** + * Order of two edges that cross the sweep line at the same time and don't cross each other (bottom + * to top): compared where the later one starts, then by direction; equal edges by number. $counter + * counts the comparisons. + */ + private static function sweepOrder(array $xs, array $ys, array $lo, array $hi, int &$counter): \Closure { + return function (int $s, int $t) use ($xs, $ys, $lo, $hi, &$counter): int { + $counter++; + $sx = $xs[$lo[$s]]; + $sy = $ys[$lo[$s]]; + $tx = $xs[$lo[$t]]; + $ty = $ys[$lo[$t]]; + if ($sx > $tx || ($sx == $tx && $sy > $ty)) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $sx, $sy); + if ($c === 0) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } + } elseif ($sx == $tx && $sy == $ty) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } else { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $tx, $ty); + if ($c === 0) { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $xs[$hi[$t]], $ys[$hi[$t]]); + } + } + return $c !== 0 ? $c : ($s <=> $t); + }; + } + + /** + * Where points are against a ring that doesn't intersect itself: 1 inside, 0 on its boundary, -1 + * outside (exact). Every point against every edge when that is cheap; otherwise one sweep line over + * the edges with the points as events, O((n + m) log n): the edges below a point, each counted +1 or + * -1 by its direction, give its winding number (a ray straight down). + */ + private function locatePointsInRing(array $ring, array $points): array { + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; + } + $n = count($xs) - 1; + $m = count($points); + $budget = self::$sweepBudget ?? (int)(16 * ($n + $m) * log($n + $m + 2, 2)); + $out = []; + if ($n < 4 || ($budget > 0 && $n * $m <= $budget)) { + foreach ($points as $point) { + $out[] = self::locatePointInRing((float)$point[0], (float)$point[1], $xs, $ys, $n); + } + return $out; + } + + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $vertices = []; + for ($i = 0; $i < $n; $i++) { + $vertices[pack('dd', $xs[$i] + 0.0, $ys[$i] + 0.0)] = true; + } + $weights = []; + for ($i = 0; $i < $n; $i++) { + $weights[$i] = $lo[$i] === $i ? 1 : -1; // run from its smaller end to its larger one + } + foreach ($points as $k => $point) { + $x = (float)$point[0]; + $y = (float)$point[1]; + if (isset($vertices[pack('dd', $x + 0.0, $y + 0.0)])) { + $out[$k] = 0; + continue; + } + $out[$k] = -1; + $ex[] = $x; + $ey[] = $y; + $ee[] = -1 - $k; // point events: negative codes + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + $tree = self::avlTree($weights); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi, self::$pointEdgeTests); + $count = count($ee); + for ($k = 0; $k < $count;) { + if ($ee[$k] < 0) { + // A point (never at a vertex): the edges below it + $x = $ex[$k]; + $y = $ey[$k]; + $winding = 0; + for ($node = $tree['root']; $node >= 0;) { + self::$pointEdgeTests++; + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $x, $y); + if ($c === 0) { + $winding = null; // on this edge + break; + } + if ($c > 0) { + $winding += $tree['s'][$tree['l'][$node]] + $weights[$node]; + $node = $tree['r'][$node]; + } else { + $node = $tree['l'][$node]; + } + } + $out[-1 - $ee[$k]] = $winding === null ? 0 : ($winding !== 0 ? 1 : -1); + $k++; + continue; + } + // A vertex: its edges that end here leave the tree, then the ones that start here join it + $px = $ex[$k]; + $py = $ey[$k]; + $starts = []; + for (; $k < $count && $ee[$k] >= 0 && $ex[$k] == $px && $ey[$k] == $py; $k++) { + if ($ee[$k] & 1) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $ee[$k] >> 1, $cmp); + } else { + $starts[] = $ee[$k] >> 1; + } + } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + } + } + ksort($out); + return $out; + } + + /** + * One point against a ring, exact: 1 inside, 0 on its boundary, -1 outside (winding number) + */ + private static function locatePointInRing(float $x, float $y, array $xs, array $ys, int $n): int { + self::$pointEdgeTests += $n; + $winding = 0; + for ($i = 0; $i < $n; $i++) { + $x1 = $xs[$i]; + $y1 = $ys[$i]; + $x2 = $xs[$i + 1]; + $y2 = $ys[$i + 1]; + $o = null; + if (($x1 < $x2 ? $x1 <= $x && $x <= $x2 : $x2 <= $x && $x <= $x1) + && ($y1 < $y2 ? $y1 <= $y && $y <= $y2 : $y2 <= $y && $y <= $y1)) { + $o = self::orient($x1, $y1, $x2, $y2, $x, $y); + if ($o === 0) { + return 0; + } + } + if ($y1 <= $y) { + if ($y2 > $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) > 0) { + $winding++; + } + } elseif ($y2 <= $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) < 0) { + $winding--; + } + } + return $winding !== 0 ? 1 : -1; + } + + /** + * True when two closed segments share a point (exact) + */ + private static function segmentsMeet(float $ax, float $ay, float $bx, float $by, float $cx, float $cy, float $dx, float $dy): bool { + if (($ax < $bx ? $ax : $bx) > ($cx > $dx ? $cx : $dx) || ($cx < $dx ? $cx : $dx) > ($ax > $bx ? $ax : $bx) + || ($ay < $by ? $ay : $by) > ($cy > $dy ? $cy : $dy) || ($cy < $dy ? $cy : $dy) > ($ay > $by ? $ay : $by)) { + return false; // bounding boxes apart + } + $o1 = self::orient($ax, $ay, $bx, $by, $cx, $cy); + $o2 = self::orient($ax, $ay, $bx, $by, $dx, $dy); + $o3 = self::orient($cx, $cy, $dx, $dy, $ax, $ay); + $o4 = self::orient($cx, $cy, $dx, $dy, $bx, $by); + if ($o1 * $o2 < 0 && $o3 * $o4 < 0) { + return true; + } + // Touching or collinear: an end of one on the other (inside the boxes, which overlap) + return ($o1 === 0 && self::inBox($ax, $ay, $bx, $by, $cx, $cy)) || ($o2 === 0 && self::inBox($ax, $ay, $bx, $by, $dx, $dy)) + || ($o3 === 0 && self::inBox($cx, $cy, $dx, $dy, $ax, $ay)) || ($o4 === 0 && self::inBox($cx, $cy, $dx, $dy, $bx, $by)); + } + + private static function inBox(float $ax, float $ay, float $bx, float $by, float $px, float $py): bool { + return ($ax < $bx ? $ax <= $px && $px <= $bx : $bx <= $px && $px <= $ax) + && ($ay < $by ? $ay <= $py && $py <= $by : $by <= $py && $py <= $ay); + } + + /** + * Sign of the orientation of c against the directed line a -> b: 1 left, -1 right, 0 on the line. + * Exact for all finite doubles: the floating-point determinant decides when it is clearly away from + * zero (Shewchuk's error bound), otherwise it is summed exactly. + */ + private static function orient(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + $detLeft = ($ax - $cx) * ($by - $cy); + $detRight = ($ay - $cy) * ($bx - $cx); + $det = $detLeft - $detRight; + if ($detLeft > 0.0) { + if ($detRight <= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = $detLeft + $detRight; + } elseif ($detLeft < 0.0) { + if ($detRight >= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = -$detLeft - $detRight; + } else { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $bound = 3.3306690738754716e-16 * $detSum; // (3 + 16 eps) eps, eps = 2^-53 + if ($det >= $bound || -$det >= $bound) { + return $det > 0.0 ? 1 : -1; + } + return self::orientExact($ax, $ay, $bx, $by, $cx, $cy); + } + + /** + * Exact sign of (a - c) x (b - c): the differences and products as exact float sums (Dekker, + * Knuth), added into a nonoverlapping expansion whose largest part has the sign + */ + private static function orientExact(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + [$acx, $acxTail] = self::twoDiff($ax, $cx); + [$bcy, $bcyTail] = self::twoDiff($by, $cy); + [$acy, $acyTail] = self::twoDiff($ay, $cy); + [$bcx, $bcxTail] = self::twoDiff($bx, $cx); + $sum = []; + foreach ([[$acx, $bcy, 1.0], [$acx, $bcyTail, 1.0], [$acxTail, $bcy, 1.0], [$acxTail, $bcyTail, 1.0], + [$acy, $bcx, -1.0], [$acy, $bcxTail, -1.0], [$acyTail, $bcx, -1.0], [$acyTail, $bcxTail, -1.0]] as [$p, $q, $sign]) { + if ($p == 0.0 || $q == 0.0) { + continue; + } + [$product, $error] = self::twoProduct($p, $q); + $sum = self::growExpansion(self::growExpansion($sum, $sign * $product), $sign * $error); + } + for ($i = count($sum) - 1; $i >= 0; $i--) { + if ($sum[$i] != 0.0) { + return $sum[$i] > 0.0 ? 1 : -1; + } + } + return 0; + } + + /** Add $b to a nonoverlapping expansion (smallest part first), zero parts dropped */ + private static function growExpansion(array $parts, float $b): array { + if ($b == 0.0) { + return $parts; + } + $out = []; + $q = $b; + foreach ($parts as $part) { + [$q, $error] = self::twoSum($q, $part); + if ($error != 0.0) { + $out[] = $error; + } + } + $out[] = $q; + return $out; + } + + /** a + b = x + y exactly */ + private static function twoSum(float $a, float $b): array { + $x = $a + $b; + $bVirtual = $x - $a; + $aVirtual = $x - $bVirtual; + return [$x, ($a - $aVirtual) + ($b - $bVirtual)]; + } + + /** a - b = x + y exactly */ + private static function twoDiff(float $a, float $b): array { + $x = $a - $b; + $bVirtual = $a - $x; + $aVirtual = $x + $bVirtual; + return [$x, ($a - $aVirtual) + ($bVirtual - $b)]; + } + + /** a * b = x + y exactly (Dekker's split into 26-bit halves) */ + private static function twoProduct(float $a, float $b): array { + $x = $a * $b; + $c = 134217729.0 * $a; // 2^27 + 1 + $aHigh = $c - ($c - $a); + $aLow = $a - $aHigh; + $c = 134217729.0 * $b; + $bHigh = $c - ($c - $b); + $bLow = $b - $bHigh; + $error = (($x - $aHigh * $bHigh) - $aLow * $bHigh) - $aHigh * $bLow; + return [$x, $aLow * $bLow - $error]; + } + + /** + * An empty AVL tree for the sweeps; $weights[node] is summed per subtree ('s'). Index -1 is the + * empty child (height 0, sum 0). + */ + private static function avlTree(array $weights): array { + return ['l' => [-1 => -1], 'r' => [-1 => -1], 'h' => [-1 => 0], 's' => [-1 => 0], 'w' => $weights, 'root' => -1]; + } + + /** Insert $node below $x, return the new subtree root; $pred/$succ get its neighbours */ + private static function avlInsert(array &$t, int $x, int $node, \Closure $cmp, int &$pred, int &$succ): int { + if ($x < 0) { + $t['l'][$node] = -1; + $t['r'][$node] = -1; + $t['h'][$node] = 1; + $t['s'][$node] = $t['w'][$node]; + return $node; + } + if ($cmp($node, $x) < 0) { + $succ = $x; + $t['l'][$x] = self::avlInsert($t, $t['l'][$x], $node, $cmp, $pred, $succ); + } else { + $pred = $x; + $t['r'][$x] = self::avlInsert($t, $t['r'][$x], $node, $cmp, $pred, $succ); + } + return self::avlBalance($t, $x); + } + + /** Remove $node (it is in the tree) below $x, return the new subtree root */ + private static function avlDelete(array &$t, int $x, int $node, \Closure $cmp): int { + if ($x < 0) { + return -1; + } + if ($x !== $node) { + if ($cmp($node, $x) < 0) { + $t['l'][$x] = self::avlDelete($t, $t['l'][$x], $node, $cmp); + } else { + $t['r'][$x] = self::avlDelete($t, $t['r'][$x], $node, $cmp); + } + return self::avlBalance($t, $x); + } + $left = $t['l'][$x]; + $right = $t['r'][$x]; + if ($left < 0) { + return $right; + } + if ($right < 0) { + return $left; + } + $next = $right; + while ($t['l'][$next] >= 0) { + $next = $t['l'][$next]; + } + $t['r'][$next] = self::avlDeleteMin($t, $right); + $t['l'][$next] = $left; + return self::avlBalance($t, $next); + } + + private static function avlDeleteMin(array &$t, int $x): int { + if ($t['l'][$x] < 0) { + return $t['r'][$x]; + } + $t['l'][$x] = self::avlDeleteMin($t, $t['l'][$x]); + return self::avlBalance($t, $x); + } + + private static function avlBalance(array &$t, int $x): int { + self::avlUpdate($t, $x); + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $diff = $t['h'][$left] - $t['h'][$right]; + if ($diff > 1) { + if ($t['h'][$t['l'][$left]] < $t['h'][$t['r'][$left]]) { + $t['l'][$x] = self::avlRotate($t, $left, true); + } + return self::avlRotate($t, $x, false); + } + if ($diff < -1) { + if ($t['h'][$t['r'][$right]] < $t['h'][$t['l'][$right]]) { + $t['r'][$x] = self::avlRotate($t, $right, false); + } + return self::avlRotate($t, $x, true); + } + return $x; + } + + /** Rotate: $left = the right child goes up, else the left child; returns the new subtree root */ + private static function avlRotate(array &$t, int $x, bool $left): int { + if ($left) { + $y = $t['r'][$x]; + $t['r'][$x] = $t['l'][$y]; + $t['l'][$y] = $x; + } else { + $y = $t['l'][$x]; + $t['l'][$x] = $t['r'][$y]; + $t['r'][$y] = $x; + } + self::avlUpdate($t, $x); + self::avlUpdate($t, $y); + return $y; + } + + private static function avlUpdate(array &$t, int $x): void { + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $t['h'][$x] = ($t['h'][$left] > $t['h'][$right] ? $t['h'][$left] : $t['h'][$right]) + 1; + $t['s'][$x] = $t['s'][$left] + $t['s'][$right] + $t['w'][$x]; + } + + /** + * True when a point lies on a ring's boundary + */ + private function pointOnRing(float $lon, float $lat, array $ring): bool { + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + if ($this->crossProductDirection($ring[$i], $ring[$i + 1], [$lon, $lat]) == 0 + && $this->onSegment($ring[$i], $ring[$i + 1], [$lon, $lat])) { + return true; + } + } + return false; + } + + /** + * Validate MultiPoint coordinates [[lon,lat], [lon,lat], ...] + */ + private function validateGeoJSONMultiPoint($coords): array { + if (!self::isList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPoint must have at least one point', 'type' => null]; + } + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid point at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiPoint']; + } + + /** + * Validate MultiLineString coordinates [[[lon,lat], ...], ...] + */ + private function validateGeoJSONMultiLineString($coords): array { + if (!self::isList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiLineString must have at least one linestring', 'type' => null]; + } + foreach ($coords as $i => $lineString) { + $result = $this->validateGeoJSONLineString($lineString); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid linestring at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiLineString']; + } + + /** + * Validate MultiPolygon coordinates [[[[lon,lat], ...], ...], ...] + */ + private function validateGeoJSONMultiPolygon($coords): array { + if (!self::isList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPolygon must have at least one polygon', 'type' => null]; + } + foreach ($coords as $i => $polygon) { + $result = $this->validateGeoJSONPolygon($polygon); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid polygon at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiPolygon']; + } + + /** + * Validate GeometryCollection + */ + private function validateGeometryCollection(array $geometry): array { + if (!isset($geometry['geometries']) || !self::isList($geometry['geometries'])) { + return ['valid' => false, 'error' => 'GeometryCollection must have geometries array', 'type' => null]; + } + foreach ($geometry['geometries'] as $i => $geom) { + if (!is_array($geom)) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: must be an object", 'type' => null]; + } + $result = $this->validateGeoJSON($geom); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'GeometryCollection']; + } + + // ========================================== + // SPATIAL INDEX - GEOMETRY OPERATIONS + // ========================================== + + /** + * Calculate Minimum Bounding Rectangle (MBR) for any GeoJSON geometry + * @param array $geometry GeoJSON geometry object + * @return array|null [minLon, minLat, maxLon, maxLat]; null for a geometry without any position + * (an empty GeometryCollection): it has no location and is never indexed or matched + */ + private function calculateMBR(array $geometry): ?array { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + + switch ($type) { + case 'Point': + return [$coords[0], $coords[1], $coords[0], $coords[1]]; + + case 'LineString': + case 'MultiPoint': + return $this->mbrFromPoints($coords); + + case 'Polygon': + // Use outer ring (first ring) + return $this->mbrFromPoints($coords[0]); + + case 'MultiLineString': + $allPoints = []; + foreach ($coords as $lineString) { + $allPoints = array_merge($allPoints, $lineString); + } + return $this->mbrFromPoints($allPoints); + + case 'MultiPolygon': + $allPoints = []; + foreach ($coords as $polygon) { + $allPoints = array_merge($allPoints, $polygon[0]); // outer ring + } + return $this->mbrFromPoints($allPoints); + + case 'GeometryCollection': + $mbr = null; + foreach ($geometry['geometries'] as $geom) { + $geomMBR = $this->calculateMBR($geom); + if ($geomMBR !== null) { + $mbr = $mbr === null ? $geomMBR : $this->mbrUnion($mbr, $geomMBR); + } + } + return $mbr; + + default: + return null; + } + } + + /** + * Calculate MBR from array of points + * @param array $points Array of [lon, lat] positions + * @return array [minLon, minLat, maxLon, maxLat] + */ + private function mbrFromPoints(array $points): ?array { + if (empty($points)) { + return null; + } + + $minLon = $maxLon = $points[0][0]; + $minLat = $maxLat = $points[0][1]; + + foreach ($points as $point) { + $minLon = min($minLon, $point[0]); + $maxLon = max($maxLon, $point[0]); + $minLat = min($minLat, $point[1]); + $maxLat = max($maxLat, $point[1]); + } + + return [$minLon, $minLat, $maxLon, $maxLat]; + } + + /** + * Union two MBRs + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return array [minLon, minLat, maxLon, maxLat] + */ + private function mbrUnion(array $mbr1, array $mbr2): array { + return [ + min($mbr1[0], $mbr2[0]), // minLon + min($mbr1[1], $mbr2[1]), // minLat + max($mbr1[2], $mbr2[2]), // maxLon + max($mbr1[3], $mbr2[3]) // maxLat + ]; + } + + /** + * Check if two MBRs overlap + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return bool + */ + private function mbrOverlaps(array $mbr1, array $mbr2): bool { + // Check if one MBR is completely to the left, right, above, or below the other + if ($mbr1[2] < $mbr2[0] || $mbr2[2] < $mbr1[0]) return false; // No horizontal overlap + if ($mbr1[3] < $mbr2[1] || $mbr2[3] < $mbr1[1]) return false; // No vertical overlap + return true; + } + + /** + * Calculate area of an MBR (in degrees squared) + * @param array $mbr [minLon, minLat, maxLon, maxLat] + * @return float + */ + private function mbrArea(array $mbr): float { + $width = $mbr[2] - $mbr[0]; + $height = $mbr[3] - $mbr[1]; + return $width * $height; + } + + /** + * Calculate enlargement needed if entry is added to an MBR + * @param array $mbr Existing MBR + * @param array $entryMBR New entry's MBR + * @return float Area increase + */ + private function mbrEnlargement(array $mbr, array $entryMBR): float { + $unionMBR = $this->mbrUnion($mbr, $entryMBR); + return $this->mbrArea($unionMBR) - $this->mbrArea($mbr); + } + + /** + * Calculate Haversine distance between two points (in meters) + * (v3.3.0: no memo any more: its string keys depended on the precision setting, so two points could + * share a key, it almost never hit, and the key cost more than the formula) + * @param float $lon1 Longitude of point 1 + * @param float $lat1 Latitude of point 1 + * @param float $lon2 Longitude of point 2 + * @param float $lat2 Latitude of point 2 + * @return float Distance in meters + */ + public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float { + $earthRadius = 6371000.0; // meters + + $latDelta = deg2rad($lat2 - $lat1); + $lonDelta = deg2rad($lon2 - $lon1); + + $a = sin($latDelta / 2) * sin($latDelta / 2) + + cos(deg2rad($lat1)) * cos(deg2rad($lat2)) * + sin($lonDelta / 2) * sin($lonDelta / 2); + + $c = 2 * atan2(sqrt($a), sqrt(1 - $a)); + + return $earthRadius * $c; + } + + /** + * Kept for compatibility: there is no distance cache any more (v3.3.0) + */ + public function clearDistanceCache(): void { + } + + /** + * Point-in-Polygon test using Winding Number algorithm + * More robust than ray casting for edge cases + * + * @param float $lon Point longitude + * @param float $lat Point latitude + * @param array $polygon GeoJSON Polygon coordinates [[[lon,lat], ...], ...] + * @return bool + */ + private function pointInPolygon(float $lon, float $lat, array $polygon): bool { + // Handle GeoJSON Polygon object or raw coordinates + $coords = isset($polygon['coordinates']) ? $polygon['coordinates'] : $polygon; + + // Check outer ring + if (!$this->pointInRing($lon, $lat, $coords[0])) { + return false; + } + + // Check holes (if any) + for ($i = 1; $i < count($coords); $i++) { + if ($this->pointInRing($lon, $lat, $coords[$i])) { + return false; // Inside a hole + } + } + + return true; + } + + /** + * Check if point is inside a ring using Winding Number algorithm + */ + private function pointInRing(float $lon, float $lat, array $ring): bool { + $windingNumber = 0; + $n = count($ring); + + for ($i = 0; $i < $n - 1; $i++) { + $x1 = $ring[$i][0]; + $y1 = $ring[$i][1]; + $x2 = $ring[$i + 1][0]; + $y2 = $ring[$i + 1][1]; + + if ($y1 <= $lat) { + if ($y2 > $lat) { + // Upward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) > 0) { + $windingNumber++; + } + } + } else { + if ($y2 <= $lat) { + // Downward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) < 0) { + $windingNumber--; + } + } + } + } + + return $windingNumber !== 0; + } + + /** + * Determine if point is left of a line (using cross product) + * Returns: >0 if left, <0 if right, =0 if on line + */ + private function isLeftOfLine(float $x1, float $y1, float $x2, float $y2, float $px, float $py): float { + return (($x2 - $x1) * ($py - $y1)) - (($px - $x1) * ($y2 - $y1)); + } + + /** + * Cross product direction calculation + */ + private function crossProductDirection(array $p1, array $p2, array $p3): float { + return ($p3[0] - $p1[0]) * ($p2[1] - $p1[1]) - ($p2[0] - $p1[0]) * ($p3[1] - $p1[1]); + } + + /** + * Check if point p is on segment p1-p2 + */ + private function onSegment(array $p1, array $p2, array $p): bool { + return min($p1[0], $p2[0]) <= $p[0] && $p[0] <= max($p1[0], $p2[0]) && + min($p1[1], $p2[1]) <= $p[1] && $p[1] <= max($p1[1], $p2[1]); + } + + /** + * Kept for compatibility: no centroid cache since v3.3 (the code that used it was never called) + */ + public function clearCentroidCache(): void { + } + + // ========================================== + // SPATIAL INDEX - R-TREE ENGINE + // ========================================== + // One R-tree per segment and indexed field: {segment}.nonedb.rx.{token} (see SecondaryIndex), a + // checkpoint like the field indexes: + // {"v":4,"kind":"rx","field":"loc","q":10000000,"ns":32,"s":…,"e":…,"h":…,"mk":…,"cnt":…,"root":3,"next":9} + // {"0":[0,[minX,minY,maxX,maxY],[[key,minX,minY,maxX,maxY],…]],"3":[1,[…],[0,2]],…} + // A node is [level, box, payload]: level 0 holds entries [key, box], a higher level child node ids. + // Boxes are integers: degrees * 1e7 rounded outwards (1e-7° is about 1 cm), so they don't depend on + // serialize_precision, and every query re-checks the exact geometry. Built with sort-tile-recursive + // bulk loading; changes after the checkpoint are replayed from the data when the index is used. + // Sharded databases keep {base}.nonedb.grx.{token}: each shard's box (null = empty) as of the bytes + // its entry covers; a shard is skipped only when its entry fits, its box misses the query and the + // shard's newer lines hold no geometry in it. v3.2's .sidx./.gsidx. files are read until the segment + // is converted (convertSegmentIndexes); .gsidx. never skips a shard. + + private const RTREE_SCALE = 10000000; + + /** + * Path of a segment's R-tree (v3.3) + */ + private function getSpatialIndexPath(string $dbname, string $field, ?int $shardId = null): string { + return $this->secondaryPath($dbname, 'rx', $field, $shardId); + } + + /** + * Path of a segment's v3.2 R-tree + */ + private function getLegacySpatialIndexPath(string $dbname, string $field, ?int $shardId = null): string { + $hash = $this->hashDBName($dbname); + $name = $this->encodeIndexFieldName($field); + return $this->dbDir . $hash . ($shardId !== null ? '_s' . $shardId : '') . '.nonedb.sidx.' . $name; + } + + private function getGlobalSpatialIndexPath(string $dbname, string $field): string { + return $this->secondaryPath($dbname, 'grx', $field); + } + + private function getLegacyGlobalSpatialIndexPath(string $dbname, string $field): string { + return $this->dbDir . $this->hashDBName($dbname) . '.nonedb.gsidx.' . $this->encodeIndexFieldName($field); + } + + /** + * Integer box of a geometry (outwards), or null when it has no position + */ + private function quantizedMBR(array $geometry): ?array { + $mbr = $this->calculateMBR($geometry); + return $mbr === null ? null : $this->quantizeBox($mbr); + } + + /** + * Integer box of [minLon, minLat, maxLon, maxLat] (clamped to the globe, rounded outwards) + */ + private function quantizeBox(array $box): array { + $minX = max(-180.0, min(180.0, (float)$box[0])); + $minY = max(-90.0, min(90.0, (float)$box[1])); + $maxX = max(-180.0, min(180.0, (float)$box[2])); + $maxY = max(-90.0, min(90.0, (float)$box[3])); + return [(int)floor($minX * self::RTREE_SCALE), (int)floor($minY * self::RTREE_SCALE), + (int)ceil($maxX * self::RTREE_SCALE), (int)ceil($maxY * self::RTREE_SCALE)]; + } + + private static function boxUnion(?array $a, array $b): array { + if($a === null){ + return $b; + } + return [$a[0] < $b[0] ? $a[0] : $b[0], $a[1] < $b[1] ? $a[1] : $b[1], $a[2] > $b[2] ? $a[2] : $b[2], $a[3] > $b[3] ? $a[3] : $b[3]]; + } + + private static function boxOverlaps(?array $a, array $b): bool { + return $a !== null && $a[0] <= $b[2] && $b[0] <= $a[2] && $a[1] <= $b[3] && $b[1] <= $a[3]; + } + + private static function boxArea(array $a): float { + return ((float)$a[2] - $a[0]) * ((float)$a[3] - $a[1]); + } + + private function newRTree(string $field, int $nodeSize): array { + return ['kind' => 'rx', 'field' => $field, 'ns' => $nodeSize, 's' => 0, 'ck' => -1, 'e' => null, 'bytes' => 0, 'legacy' => false, + 'nodes' => [0 => [0, null, []]], 'root' => 0, 'next' => 1, 'mk' => -1, 'cnt' => 0, 'leafOf' => null, 'parent' => null]; + } + + /** + * Bulk load (sort-tile-recursive): leaves 3/4 full, ties broken by key (the same tree on every PHP) + * @param array $entries list of [key, box] + */ + private function rtreeBulkLoad(array &$tree, array $entries): void { + $capacity = max(2, intdiv($tree['ns'] * 3, 4)); + $tree['nodes'] = []; + $tree['next'] = 0; + $tree['cnt'] = count($entries); + $tree['leafOf'] = null; + $tree['parent'] = null; + if(empty($entries)){ + $tree['nodes'][0] = [0, null, []]; + $tree['root'] = 0; + $tree['next'] = 1; + return; + } + $level = 0; + $items = []; + foreach($entries as $entry){ + $items[] = [$entry[1], $entry]; + } + while(true){ + $count = count($items); + $groups = (int)ceil($count / $capacity); + $slices = (int)ceil(sqrt($groups)); + usort($items, function($a, $b){ + return ($a[0][0] + $a[0][2]) <=> ($b[0][0] + $b[0][2]) ?: self::rtreeTie($a, $b); + }); + $perSlice = $slices * $capacity; + $next = []; + for($i = 0; $i < $count; $i += $perSlice){ + $slice = array_slice($items, $i, $perSlice); + usort($slice, function($a, $b){ + return ($a[0][1] + $a[0][3]) <=> ($b[0][1] + $b[0][3]) ?: self::rtreeTie($a, $b); + }); + foreach(array_chunk($slice, $capacity) as $chunk){ + $id = $tree['next']++; + $box = null; + $payload = []; + foreach($chunk as $item){ + $box = self::boxUnion($box, $item[0]); + $payload[] = $level === 0 ? [$item[1][0], $item[1][1][0], $item[1][1][1], $item[1][1][2], $item[1][1][3]] : $item[1]; + } + $tree['nodes'][$id] = [$level, $box, $payload]; + $next[] = [$box, $id]; + } + } + if(count($next) === 1){ + $tree['root'] = $next[0][1]; + return; + } + $items = $next; + $level++; + } + } + + private static function rtreeTie(array $a, array $b): int { + $ka = is_array($a[1]) ? $a[1][0] : $a[1]; + $kb = is_array($b[1]) ? $b[1][0] : $b[1]; + return $ka <=> $kb; + } + + /** + * Parent and leaf maps (built when the first change needs them) + */ + private function rtreeMaps(array &$tree): void { + if($tree['parent'] !== null){ + return; + } + $parent = []; + $leafOf = []; + foreach($tree['nodes'] as $id => $node){ + if($node[0] === 0){ + foreach($node[2] as $entry){ + $leafOf[$entry[0]] = $id; + } + } else { + foreach($node[2] as $child){ + $parent[$child] = $id; + } + } + } + $tree['parent'] = $parent; + $tree['leafOf'] = $leafOf; + } + + /** + * Insert (or move) a key with its box. The way down is remembered for the box updates and splits, + * so a key never seen before needs no parent map (built only when something is removed). + */ + private function rtreeInsert(array &$tree, int $key, array $box): void { + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } else { + $tree['mk'] = $key; + } + $path = []; + $id = $tree['root']; + while($tree['nodes'][$id][0] > 0){ + $best = null; + $bestGrowth = INF; + $bestArea = INF; + foreach($tree['nodes'][$id][2] as $child){ + $childBox = $tree['nodes'][$child][1]; + if($childBox === null){ + $best = $best ?? $child; + continue; + } + $area = self::boxArea($childBox); + $growth = self::boxArea(self::boxUnion($childBox, $box)) - $area; + if($growth < $bestGrowth || ($growth == $bestGrowth && $area < $bestArea)){ + $best = $child; + $bestGrowth = $growth; + $bestArea = $area; + } + } + if($best === null){ + $tree['nodes'][$id] = [0, null, []]; // an internal node without children becomes a leaf + break; + } + $path[] = $id; + $id = $best; + } + $tree['nodes'][$id][2][] = [$key, $box[0], $box[1], $box[2], $box[3]]; + if($tree['leafOf'] !== null){ + $tree['leafOf'][$key] = $id; + } + $tree['cnt']++; + // Widen the boxes on the way up while they change + for($node = $id, $i = count($path); ; $node = $path[--$i]){ + $old = $tree['nodes'][$node][1]; + $new = self::boxUnion($old, $box); + if($old === $new){ + break; + } + $tree['nodes'][$node][1] = $new; + if($i === 0){ + break; + } + } + if(count($tree['nodes'][$id][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $id, $path); + } + } + + private function rtreeBoxOf(array $tree, int $level, $item): ?array { + return $level === 0 ? [$item[1], $item[2], $item[3], $item[4]] : $tree['nodes'][$item][1]; + } + + /** + * Split an overflowing node (linear seeds, least enlargement), up to the root + * @param array $path The node's ancestors, root first + */ + private function rtreeSplit(array &$tree, int $id, array $path): void { + [$level, , $items] = $tree['nodes'][$id]; + $n = count($items); + $boxes = []; + foreach($items as $i => $item){ + $boxes[$i] = $this->rtreeBoxOf($tree, $level, $item) ?? [0, 0, 0, 0]; + } + // Seeds: the pair farthest apart along the axis with the greatest normalized separation + $best = null; + foreach([0, 1] as $axis){ + $lowHigh = 0; + $highLow = 0; + $min = INF; + $max = -INF; + foreach($boxes as $i => $b){ + if($b[$axis] > $boxes[$lowHigh][$axis]){ + $lowHigh = $i; + } + if($b[$axis + 2] < $boxes[$highLow][$axis + 2]){ + $highLow = $i; + } + $min = min($min, $b[$axis]); + $max = max($max, $b[$axis + 2]); + } + $width = $max - $min; + $separation = $width > 0 ? ($boxes[$lowHigh][$axis] - $boxes[$highLow][$axis + 2]) / $width : 0; + if($best === null || $separation > $best[0]){ + $best = [$separation, $lowHigh, $highLow]; + } + } + [, $seed1, $seed2] = $best; + if($seed1 === $seed2){ + $seed2 = ($seed1 + 1) % $n; + } + $groups = [[$items[$seed1]], [$items[$seed2]]]; + $groupBoxes = [$boxes[$seed1], $boxes[$seed2]]; + $minFill = max(1, (int)($tree['ns'] * 0.4)); + $left = $n - 2; + foreach($items as $i => $item){ + if($i === $seed1 || $i === $seed2){ + continue; + } + if(count($groups[0]) + $left <= $minFill){ + $g = 0; + } elseif(count($groups[1]) + $left <= $minFill){ + $g = 1; + } else { + $grow0 = self::boxArea(self::boxUnion($groupBoxes[0], $boxes[$i])) - self::boxArea($groupBoxes[0]); + $grow1 = self::boxArea(self::boxUnion($groupBoxes[1], $boxes[$i])) - self::boxArea($groupBoxes[1]); + $g = $grow0 < $grow1 || ($grow0 == $grow1 && count($groups[0]) <= count($groups[1])) ? 0 : 1; + } + $groups[$g][] = $item; + $groupBoxes[$g] = self::boxUnion($groupBoxes[$g], $boxes[$i]); + $left--; + } + $newId = $tree['next']++; + $tree['nodes'][$id] = [$level, $groupBoxes[0], $groups[0]]; + $tree['nodes'][$newId] = [$level, $groupBoxes[1], $groups[1]]; + if($tree['parent'] !== null){ + foreach($groups[1] as $item){ + if($level === 0){ + $tree['leafOf'][$item[0]] = $newId; + } else { + $tree['parent'][$item] = $newId; + } + } + } + if(empty($path)){ + $rootId = $tree['next']++; + $tree['nodes'][$rootId] = [$level + 1, self::boxUnion($groupBoxes[0], $groupBoxes[1]), [$id, $newId]]; + $tree['root'] = $rootId; + if($tree['parent'] !== null){ + $tree['parent'][$id] = $rootId; + $tree['parent'][$newId] = $rootId; + } + return; + } + $parentId = array_pop($path); + $tree['nodes'][$parentId][2][] = $newId; + if($tree['parent'] !== null){ + $tree['parent'][$newId] = $parentId; + } + if(count($tree['nodes'][$parentId][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $parentId, $path); + } + } + + /** + * Remove a key; emptied nodes go, boxes shrink up the tree, a root with one child is replaced by it + * @return bool True when the key was indexed + */ + private function rtreeRemove(array &$tree, int $key): bool { + $this->rtreeMaps($tree); + if(!isset($tree['leafOf'][$key])){ + return false; + } + $id = $tree['leafOf'][$key]; + unset($tree['leafOf'][$key]); + $entries = $tree['nodes'][$id][2]; + foreach($entries as $i => $entry){ + if($entry[0] === $key){ + unset($entries[$i]); + break; + } + } + $tree['nodes'][$id][2] = array_values($entries); + $tree['cnt']--; + // Remove emptied nodes bottom-up + while($id !== $tree['root'] && empty($tree['nodes'][$id][2])){ + $parentId = $tree['parent'][$id]; + $children = $tree['nodes'][$parentId][2]; + $at = array_search($id, $children, true); + if($at !== false){ + unset($children[$at]); + } + $tree['nodes'][$parentId][2] = array_values($children); + unset($tree['nodes'][$id], $tree['parent'][$id]); + $id = $parentId; + } + // Recompute boxes up the tree while they change + while(true){ + [$level, $old, $items] = $tree['nodes'][$id]; + $box = null; + foreach($items as $item){ + $itemBox = $this->rtreeBoxOf($tree, $level, $item); + if($itemBox !== null){ + $box = self::boxUnion($box, $itemBox); + } + } + if($box === $old){ + break; + } + $tree['nodes'][$id][1] = $box; + if(!isset($tree['parent'][$id])){ + break; + } + $id = $tree['parent'][$id]; + } + // Collapse the root + while(true){ + [$level, , $items] = $tree['nodes'][$tree['root']]; + if($level === 0){ + break; + } + if(empty($items)){ + $tree['nodes'][$tree['root']] = [0, null, []]; + break; + } + if(count($items) > 1){ + break; + } + $child = $items[0]; + unset($tree['nodes'][$tree['root']]); + $tree['root'] = $child; + unset($tree['parent'][$child]); + } + return true; + } + + /** + * Keys whose boxes overlap one of the (integer) boxes + */ + private function rtreeSearch(array $tree, array $boxes): array { + $found = []; + $stack = [$tree['root']]; + while(!empty($stack)){ + $id = array_pop($stack); + [$level, $nodeBox, $items] = $tree['nodes'][$id]; + if($nodeBox === null){ + continue; + } + $hit = false; + foreach($boxes as $box){ + if(self::boxOverlaps($nodeBox, $box)){ + $hit = true; + break; + } + } + if(!$hit){ + continue; + } + foreach($items as $item){ + if($level > 0){ + $stack[] = $item; + continue; + } + foreach($boxes as $box){ + if($item[1] <= $box[2] && $box[0] <= $item[3] && $item[2] <= $box[3] && $box[1] <= $item[4]){ + $found[$item[0]] = $item[0]; + break; + } + } + } + } + return array_values($found); + } + + /** + * Apply one data line to an R-tree: the key now holds $record (null: deleted) + */ + private function rtreeApply(array &$tree, int $key, ?array $record): void { + $geometry = $record === null ? null : ($record[$tree['field']] ?? null); + $box = is_array($geometry) && $this->validateGeoJSON($geometry)['valid'] ? $this->quantizedMBR($geometry) : null; + if($box === null){ + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } elseif($key > $tree['mk']){ + $tree['mk'] = $key; + } + return; + } + $this->rtreeInsert($tree, $key, $box); + } + + /** + * Build a segment's R-tree from its data (covers the key index's bytes) + */ + private function buildSegmentSpatialIndex(string $dbname, string $field, ?int $shardId, array $options = []): array { + $key = $this->keyIndexWithEpoch($dbname, $shardId); + $nodeSize = isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4 ? (int)$options['nodeSize'] : $this->rtreeNodeSize; + $tree = $this->newRTree($field, $nodeSize); + $entries = []; + foreach($this->readSegmentRecords($dbname, $shardId) as $k => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->validateGeoJSON($geometry)['valid']){ + $box = $this->quantizedMBR($geometry); + if($box !== null){ + $entries[] = [(int)$k, $box]; + } + } + } + $this->rtreeBulkLoad($tree, $entries); + $tree['mk'] = $key === null || empty($key['p']) ? -1 : max(array_keys($key['p'])); + $tree['s'] = $key === null ? 0 : (int)$key['s']; + $tree['e'] = $key === null ? null : $key['e']; + return $tree; + } + + /** + * A segment's R-tree, current (the data lines after the bytes it covers are replayed); null when + * none fits the segment (see ensureSegmentSpatialIndex) + */ + private function readSpatialIndex(string $dbname, string $field, ?int $shardId = null): ?array { + $key = $this->currentKeyIndex($dbname, $shardId); + if($key === null){ + return null; + } + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + self::$cacheTouch[$path] = ++self::$cacheTick; + if(isset($this->spatialIndexCache[$path])){ + $tree = $this->spatialIndexCache[$path]; + if($tree['e'] !== null && $tree['e'] === $key['e'] && $tree['s'] <= (int)$key['s']){ + if($tree['s'] < (int)$key['s']){ + unset($this->spatialIndexCache[$path], $tree['due']); + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + } elseif(isset($tree['due']) && self::$noWrites === 0 && self::$multiLock === 0){ + unset($tree['due'], $this->spatialIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + } + return $this->spatialIndexCache[$path]; + } + unset($this->spatialIndexCache[$path]); + } + $file = $this->readSecondaryFile($path); + if($file === null || !$this->secondaryFits($dbname, $shardId, $file[0], 'rx', $field)){ + return null; + } + $nodes = json_decode($file[1], true); + $header = $file[0]; + if(!is_array($nodes) || !isset($header['root'], $header['next'], $nodes[$header['root']])){ + return null; + } + $tree = $this->newRTree($field, (int)($header['ns'] ?? $this->rtreeNodeSize)); + $tree['nodes'] = []; + foreach($nodes as $id => $node){ + $tree['nodes'][(int)$id] = $node; + } + $tree['root'] = (int)$header['root']; + $tree['next'] = (int)$header['next']; + $tree['mk'] = (int)($header['mk'] ?? -1); + $tree['cnt'] = (int)($header['cnt'] ?? 0); + $tree['s'] = $header['s']; + $tree['ck'] = $header['s']; + $tree['e'] = $header['e']; + $tree['bytes'] = strlen($file[1]); + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + self::$cacheGrew = true; + if($this->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + return $this->spatialIndexCache[$path]; + } + + private function replaySpatialIndex(string $dbname, ?int $shardId, array &$tree, int $to): void { + foreach($this->segmentTail($dbname, $shardId, $tree['s'], $to) as [$k, $record]){ + $this->rtreeApply($tree, (int)$k, $record); + } + $tree['s'] = $to; + } + + /** + * A segment's v3.2 R-tree (only for a segment this version hasn't written yet), searched as it is + */ + private function readLegacySpatialIndex(string $dbname, string $field, ?int $shardId): ?array { + $path = $this->getLegacySpatialIndexPath($dbname, $field, $shardId); + if(array_key_exists($path, $this->spatialIndexCache)){ + return $this->spatialIndexCache[$path]; + } + $index = $this->atomicRead($path, null); + if(!is_array($index) || ($index['v'] ?? 0) < 3 || !isset($index['nodes'], $index['rootId'], $index['leafOf']) + || (isset($index['field']) && $index['field'] !== $field)){ + $index = null; + } else { + $index['legacy'] = true; + } + $this->spatialIndexCache[$path] = $index; + self::$cacheGrew = true; + return $index; + } + + /** + * Write an R-tree checkpoint + */ + private function writeSpatialIndex(string $dbname, string $field, array &$tree, ?int $shardId = null): bool { + if(!empty($tree['legacy']) || $tree['e'] === null){ + return false; + } + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + $crc = $this->segmentCrc($this->segDataPath($dbname, $shardId), (int)$tree['s']); + if($crc === null){ + return false; + } + $nodes = []; + foreach($tree['nodes'] as $id => $node){ + $nodes[(string)$id] = $node; + } + $body = $this->encodeJson($nodes); + if($body === false){ + return false; + } + $rootBox = $tree['nodes'][$tree['root']][1]; + $this->writeSecondaryFile($dbname, $path, ['v' => 4, 'kind' => 'rx', 'field' => $field, 'q' => self::RTREE_SCALE, 'ns' => $tree['ns'], + 's' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'mk' => $tree['mk'], 'cnt' => $tree['cnt'], + 'root' => $tree['root'], 'next' => $tree['next'], 'mbr' => $rootBox], $body . "\n"); + $tree['ck'] = (int)$tree['s']; + $tree['bytes'] = strlen($body) + 1; + $this->spatialIndexCache[$path] = $tree; + self::$cacheGrew = true; + return true; + } + + /** + * Spatial indexes are written as checkpoints when a write session ends: nothing waits here (kept + * public for compatibility) + * @return int 0 + */ + public function flushSpatialIndexes(): int { + return 0; + } + + private function forgetDirtySpatial(string $dbname): void { + } + + /** + * Read the global spatial index: ['le', 'sh' => [shard => ['s','e','h','b' => box or null]]] + */ + private function readGlobalSpatialIndex(string $dbname, string $field): ?array { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + self::$cacheTouch[$path] = ++self::$cacheTick; + if(array_key_exists($path, $this->globalSpatialIndexCache)){ + return $this->globalSpatialIndexCache[$path]; + } + $header = $this->readSecondaryHeader($path); + $global = null; + if(is_array($header) && ($header['kind'] ?? null) === 'grx' && ($header['field'] ?? null) === $field && is_array($header['sh'] ?? null)){ + $global = ['le' => $header['le'] ?? null, 'sh' => []]; + foreach($header['sh'] as $shardId => $entry){ + if(is_array($entry) && isset($entry['s'], $entry['e'], $entry['h']) && array_key_exists('b', $entry)){ + $global['sh'][(int)$shardId] = $entry; + } + } + } + $this->globalSpatialIndexCache[$path] = $global; + self::$cacheGrew = true; + return $global; + } + + private function writeGlobalSpatialIndex(string $dbname, string $field, array $global): void { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $lock = self::$lockRegistry[$this->getLockPath($dbname)] ?? null; + $global['le'] = $lock === null ? null : $this->registryEpoch($this->getLockPath($dbname)); + $sh = []; + foreach($global['sh'] as $shardId => $entry){ + $sh[(string)$shardId] = $entry; + } + $this->writeSecondaryFile($dbname, $path, ['v' => 4, 'kind' => 'grx', 'field' => $field, 'q' => self::RTREE_SCALE, + 'le' => $global['le'], 'sh' => empty($sh) ? new \stdClass() : $sh], ''); + $this->globalSpatialIndexCache[$path] = $global; + unset(self::$lockRegistry[$this->getLockPath($dbname)]['globals'][$path]); + } + + /** + * Record a shard's box (at the bytes its R-tree covers) in the global spatial index + */ + private function updateGlobalSpatialShard(string $dbname, string $field, int $shardId, array $tree): void { + $entry = $this->globalSpatialEntry($dbname, $shardId, $tree); + if($entry === null){ + return; + } + $global = $this->freshGlobalEntries($dbname, $this->readGlobalSpatialIndex($dbname, $field) ?? ['le' => null, 'sh' => []]); + $lockPath = $this->getLockPath($dbname); + if(($global['sh'][$shardId] ?? null) === $entry && $global['le'] === (isset(self::$lockRegistry[$lockPath]) ? $this->registryEpoch($lockPath) : null)){ + return; + } + $global['sh'][$shardId] = $entry; + $this->deferGlobalWrite($dbname, 'grx', $field, $global); + } + + /** + * A shard's entry in the global spatial index: its R-tree's root box at the bytes it covers (null: + * none can be recorded) + */ + private function globalSpatialEntry(string $dbname, int $shardId, array $tree): ?array { + if(!empty($tree['legacy']) || $tree['e'] === null){ + return null; + } + $crc = $this->segmentCrc($this->segDataPath($dbname, $shardId), (int)$tree['s']); + return $crc === null ? null : ['s' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'b' => $tree['nodes'][$tree['root']][1]]; + } + + /** + * A segment's R-tree: the current one, a v3.2 one while the segment isn't converted, or built from + * the data (exclusive lock) + */ + private function ensureSegmentSpatialIndex(string $dbname, string $field, ?int $shardId): array { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if($tree !== null){ + return $tree; + } + if(!$this->segmentIsConverted($dbname, $shardId)){ + $legacy = $this->readLegacySpatialIndex($dbname, $field, $shardId); + if($legacy !== null){ + return $legacy; + } + } + if(self::$noWrites > 0){ + throw new \RuntimeException("noneDB: no spatial index to read in a read-only check"); + } + return $this->withLock($dbname, true, function () use ($dbname, $field, $shardId) { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if ($tree === null) { + $nodeSize = $this->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null; + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, $nodeSize ? ['nodeSize' => $nodeSize] : []); + $this->writeSpatialIndex($dbname, $field, $tree, $shardId); + if ($shardId !== null) { + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); + } + } + return $tree; + }); + } + + /** + * Keys of a segment whose geometry box may overlap the query boxes (float [minLon, minLat, maxLon, + * maxLat], or a list of them); every caller checks the exact geometry + */ + private function rtreeSearchMBR(string $dbname, string $field, array $searchMBR, ?int $shardId = null): array { + $tree = $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); + $boxes = is_array($searchMBR[0] ?? null) ? $searchMBR : [$searchMBR]; + if(!empty($tree['legacy'])){ + $results = []; + foreach($boxes as $box){ + $found = []; + $this->legacyRtreeSearch($tree, (string)$tree['rootId'], $box, $found); + foreach($found as $key){ + $results[$key] = $key; + } + } + return array_values($results); + } + return $this->rtreeSearch($tree, array_map([$this, 'quantizeBox'], $boxes)); + } + + /** + * Search a v3.2 R-tree (float boxes, nodes with entries/children) + */ + private function legacyRtreeSearch(array $index, string $nodeId, array $box, array &$results): void { + $node = $index['nodes'][$nodeId] ?? null; + if($node === null || $node['mbr'] === null || !$this->mbrOverlaps($node['mbr'], $box)){ + return; + } + if($node['isLeaf']){ + foreach($node['entries'] as $entry){ + if($this->mbrOverlaps($entry['mbr'], $box)){ + $results[] = $entry['key']; + } + } + } else { + foreach($node['children'] as $childId){ + $this->legacyRtreeSearch($index, (string)$childId, $box, $results); + } + } + } + + /** + * Rebuild a spatial index for the whole database (all segments + global shard boxes) + */ + private function rebuildSpatialIndexForDb($dbname, $field, $nodeSize = null){ + $manifest = $this->getIndexManifest($dbname); + $nodeSize = $nodeSize ?? $manifest['spatialOptions'][$field]['nodeSize'] ?? null; + $options = $nodeSize ? ['nodeSize' => $nodeSize] : []; + $segments = $this->listSegments($dbname); + if(!in_array(null, $segments, true)){ + $segments[] = null; // leftovers of the non-sharded database + } + foreach($segments as $segment){ + foreach([$this->getSpatialIndexPath($dbname, $field, $segment), $this->getLegacySpatialIndexPath($dbname, $field, $segment)] as $path){ + $this->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + } + foreach([$this->getGlobalSpatialIndexPath($dbname, $field), $this->getLegacyGlobalSpatialIndexPath($dbname, $field)] as $path){ + $this->fsUnlink($path); + unset($this->globalSpatialIndexCache[$path]); + } + $indexed = 0; + $sharded = $this->isSharded($dbname); + $global = ['le' => null, 'sh' => []]; // written once at the end (once per shard was O(shards²)) + foreach ($this->listSegments($dbname) as $segment) { + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $segment, $options); + $this->writeSpatialIndex($dbname, $field, $tree, $segment); + $indexed += $tree['cnt']; + if ($sharded && ($entry = $this->globalSpatialEntry($dbname, $segment, $tree)) !== null) { + $global['sh'][$segment] = $entry; + } + } + if ($sharded) { + ksort($global['sh']); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + return $indexed; + } + + /** + * Delete the spatial index files of one shard + */ + private function removeSpatialShardFiles($dbname, $shardId){ + foreach (array_merge( + $this->listFilesWithPrefix($this->dbBase($dbname) . '_s' . (int)$shardId . '.nonedb.rx.'), + $this->listFilesWithPrefix($this->dbDir . $this->hashDBName($dbname) . '_s' . (int)$shardId . '.nonedb.sidx.') + ) as $file) { + $this->fsUnlink($file); + unset($this->spatialIndexCache[$file]); + } + foreach ($this->getIndexManifest($dbname)['spatial'] as $field) { + $global = $this->readGlobalSpatialIndex($dbname, $field); + if ($global !== null && isset($global['sh'][(int)$shardId])) { + unset($global['sh'][(int)$shardId]); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + } + } + + /** + * Spatial indexes follow the data by themselves (replayed when used): kept for the write paths + */ + private function applySpatialIndexChanges($dbname, $shardId, array $removed, array $added){ + } + + /** + * True when a shard can be skipped for the query boxes: its global entry fits the shard, its box + * misses every box, and the shard's newer lines hold no geometry in them + */ + private function globalSpatialSkips(string $dbname, string $field, int $shardId, ?array $global, array $boxes): bool { + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || !$this->globalEntryFits($dbname, $shardId, $entry, $global['le'])){ + return false; + } + $quantized = array_map([$this, 'quantizeBox'], $boxes); + foreach($quantized as $box){ + if(self::boxOverlaps($entry['b'], $box)){ + return false; + } + } + return !$this->shardTailMayMatch($dbname, $shardId, $entry, function($record) use ($field, $quantized){ + $geometry = $record[$field] ?? null; + if(!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']){ + return false; + } + $box = $this->quantizedMBR($geometry); + foreach($quantized as $q){ + if(self::boxOverlaps($box, $q)){ + return true; + } + } + return false; + }); + } + + /** + * Check if a spatial index exists + * @param string $dbname Database name + * @param string $field Field name + * @param int|null $shardId Shard ID + * @return bool + */ + public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $shardId){ + return $this->hasSpatialIndexUnlocked($dbname, $field, $shardId); + }); + } + + private function hasSpatialIndexUnlocked(string $dbname, string $field, ?int $shardId = null): bool{ + if (!$this->dbExists($dbname) || !in_array($field, $this->getIndexManifest($dbname)['spatial'], true)) { + return false; + } + return $shardId === null || in_array($shardId, $this->listSegments($dbname), true); + } + + // ========================================== + // SPATIAL INDEX - PUBLIC API + // ========================================== + + /** + * Create a spatial index on a GeoJSON field + * @param string $dbname Database name + * @param string $field Field name containing GeoJSON + * @param array $options Options ['nodeSize' => 32] + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + */ + public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field, $options){ + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + }); + } + + private function createSpatialIndexUnlocked(string $dbname, string $field, array $options = []): array{ + if (!$this->spatialIndexEnabled) { + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + if ($field === '' || $field === 'key') { + return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + } + $this->flushPendingBuffers($dbname); + if (!$this->dbExists($dbname)) { + if (!$this->autoCreateDB) { + return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + } + $this->createDBUnlocked($dbname); + } + + $manifest = $this->getIndexManifest($dbname); + if (in_array($field, $manifest['spatial'], true)) { + return ['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field '$field'"]; + } + $nameError = $this->indexNameError($dbname, $field); + if ($nameError !== null) { + return ['success' => false, 'indexed' => 0, 'error' => $nameError]; + } + + // Build first, list it last: an index that fails half-way is never listed (B-10) + $nodeSize = isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4 ? (int)$options['nodeSize'] : null; + $indexed = $this->rebuildSpatialIndexForDb($dbname, $field, $nodeSize); + $manifest = $this->getIndexManifest($dbname); + $manifest['spatial'][] = $field; + if ($nodeSize !== null) { + $manifest['spatialOptions'][$field] = ['nodeSize' => $nodeSize]; + } + $this->saveIndexManifest($dbname, $manifest); + return ['success' => true, 'indexed' => $indexed, 'error' => null]; + } + + /** + * Drop a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => ?string] + */ + public function dropSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropSpatialIndexUnlocked($dbname, $field); + }); + } + + private function dropSpatialIndexUnlocked(string $dbname, string $field): array{ + foreach (array_merge([null], $this->isSharded($dbname) ? $this->listSegments($dbname) : []) as $segment) { + foreach ([$this->getSpatialIndexPath($dbname, $field, $segment), $this->getLegacySpatialIndexPath($dbname, $field, $segment)] as $path) { + $this->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + } + foreach ([$this->getGlobalSpatialIndexPath($dbname, $field), $this->getLegacyGlobalSpatialIndexPath($dbname, $field)] as $globalPath) { + $this->fsUnlink($globalPath); + unset($this->globalSpatialIndexCache[$globalPath]); + } + + if ($this->dbExists($dbname)) { + $manifest = $this->getIndexManifest($dbname); + $manifest['spatial'] = array_values(array_diff($manifest['spatial'], [$field])); + unset($manifest['spatialOptions'][$field]); + $this->saveIndexManifest($dbname, $manifest); + } + return ['success' => true, 'error' => null]; + } + + /** + * Get list of spatial indexes for a database + * @param string $dbname Database name + * @return array List of indexed field names + */ + public function getSpatialIndexes(string $dbname): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname){ + return $this->getSpatialIndexesUnlocked($dbname); + }); + } + + private function getSpatialIndexesUnlocked(string $dbname): array{ + return $this->dbExists($dbname) ? $this->getIndexManifest($dbname)['spatial'] : []; + } + + /** + * Rebuild a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createSpatialIndex + */ + public function rebuildSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildSpatialIndexUnlocked($dbname, $field); + }); + } + + private function rebuildSpatialIndexUnlocked(string $dbname, string $field): array{ + if (!$this->spatialIndexEnabled) { + // Checked before dropping anything: the drop used to succeed and the create to fail (B-15) + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + $options = []; + $nodeSize = $this->dbExists($dbname) ? ($this->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null) : null; + if ($nodeSize) { + $options['nodeSize'] = $nodeSize; + } + $this->dropSpatialIndexUnlocked($dbname, $field); + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + } + + // ========================================== + // SPATIAL QUERIES + // ========================================== + + /** + * Records whose geometry may overlap the search boxes: [key => record]. + * Uses the R-tree (and shard boxes) when the field is indexed; otherwise every record. + * @param array $boxes List of MBRs + */ + private function spatialCandidates(string $dbname, string $field, array $boxes): array { + if (!$this->dbExists($dbname)) { + return []; + } + $indexed = $this->spatialIndexEnabled && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); + $records = []; + $this->indexedCandidates = []; + $sharded = $this->isSharded($dbname); + $global = ($indexed && $sharded) ? $this->readGlobalSpatialIndex($dbname, $field) : null; + + foreach ($this->listSegments($dbname) as $segment) { + if (!$indexed) { + $records += $this->readSegmentRecords($dbname, $segment); + continue; + } + // Shard skip: only with an entry that fits the shard, misses the boxes, and no newer geometry of + // the shard in them. A missing entry (or global file) means unknown: searched (B-9). + if ($global !== null && $this->globalSpatialSkips($dbname, $field, $segment, $global, $boxes)) { + continue; + } + try { + $keys = $this->rtreeSearchMBR($dbname, $field, $boxes, $segment); + if (empty($this->ensureSegmentSpatialIndex($dbname, $field, $segment)['legacy'])) { + foreach ($keys as $key) { + $this->indexedCandidates[$key] = true; + } + } + } catch (\Throwable $e) { + if ($e->getCode() === self::LOCK_FAILED) { + throw $e; + } + $records += $this->readSegmentRecords($dbname, $segment); // index can't be built: scan (B-1) + continue; + } + if (!empty($keys)) { + $records += $this->readSegmentRecords($dbname, $segment, $keys); + } + } + return $records; + } + + /** + * True when an MBR (null: empty) overlaps one of the boxes + */ + private function anyBoxOverlaps(?array $mbr, array $boxes): bool { + if ($mbr === null) { + return false; + } + foreach ($boxes as $box) { + if ($this->mbrOverlaps($mbr, $box)) { + return true; + } + } + return false; + } + + /** + * Search boxes for a circle; splits at the antimeridian and covers all longitudes near the poles + * @return array List of [minLon, minLat, maxLon, maxLat] + */ + private function circleToBBoxes(float $lon, float $lat, float $radiusMeters): array { + $latDeg = $radiusMeters / 111000.0; + $minLat = $lat - $latDeg; + $maxLat = $lat + $latDeg; + if ($minLat <= -90 || $maxLat >= 90) { + return [[-180.0, max(-90.0, $minLat), 180.0, min(90.0, $maxLat)]]; + } + $cos = cos(deg2rad(max(abs($minLat), abs($maxLat)))); + $lonDeg = $cos > 1e-9 ? $radiusMeters / (111000.0 * $cos) : 360.0; + if ($lonDeg >= 180) { + return [[-180.0, $minLat, 180.0, $maxLat]]; + } + $minLon = $lon - $lonDeg; + $maxLon = $lon + $lonDeg; + if ($minLon < -180) { + return [[-180.0, $minLat, $maxLon, $maxLat], [$minLon + 360.0, $minLat, 180.0, $maxLat]]; + } + if ($maxLon > 180) { + return [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon - 360.0, $maxLat]]; + } + return [[$minLon, $minLat, $maxLon, $maxLat]]; + } + + /** + * Distance in meters from a point to a geometry: 0 inside a polygon, otherwise to the nearest + * vertex or edge (edges use a local equirectangular approximation) + */ + private function distanceToGeometry(float $lon, float $lat, array $geometry): float { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + switch ($type) { + case 'Point': + return $this->haversineDistance($lon, $lat, (float)$coords[0], (float)$coords[1]); + case 'MultiPoint': + $best = INF; + foreach ($coords as $p) { + $best = min($best, $this->haversineDistance($lon, $lat, (float)$p[0], (float)$p[1])); + } + return $best; + case 'LineString': + return $this->distanceToPath($lon, $lat, $coords); + case 'MultiLineString': + $best = INF; + foreach ($coords as $line) { + $best = min($best, $this->distanceToPath($lon, $lat, $line)); + } + return $best; + case 'Polygon': + if ($this->pointInPolygon($lon, $lat, $coords)) { + return 0.0; + } + $best = INF; + foreach ($coords as $ring) { + $best = min($best, $this->distanceToPath($lon, $lat, $ring)); + } + return $best; + case 'MultiPolygon': + $best = INF; + foreach ($coords as $polygon) { + $best = min($best, $this->distanceToGeometry($lon, $lat, ['type' => 'Polygon', 'coordinates' => $polygon])); + } + return $best; + case 'GeometryCollection': + $best = INF; + foreach ($geometry['geometries'] ?? [] as $g) { + $best = min($best, $this->distanceToGeometry($lon, $lat, $g)); + } + return $best; + } + return INF; + } + + /** + * Distance from a point to a polyline (meters): to its nearest point, an edge being straight in + * longitude/latitude. Exact up to floating point: every vertex is measured, and every edge whose + * bounding box could hold a nearer point is searched (golden section on the haversine of the point at + * t along the edge, which wraps at the antimeridian by itself); the other edges are skipped by a lower + * bound of the haversine over their box. + */ + private function distanceToPath(float $lon, float $lat, array $points): float { + $n = count($points); + if ($n === 0) { + return INF; + } + $cosLat = cos(deg2rad($lat)); + $xs = []; + $ys = []; + $best = INF; // the haversine "a" of the nearest point found (monotonic in the distance) + foreach ($points as $i => $point) { + $xs[$i] = (float)$point[0]; + $ys[$i] = (float)$point[1]; + $best = min($best, $this->haversineA($lon, $lat, $cosLat, $xs[$i], $ys[$i])); + } + $points = array_keys($xs); + $bounds = []; + for ($e = 0; $e < $n - 1; $e++) { + $i = $points[$e]; + $j = $points[$e + 1]; + $bound = $this->edgeBoxLowerBound($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j]); + if ($bound < $best) { + $bounds[$e] = $bound; + } + } + asort($bounds); + foreach ($bounds as $e => $bound) { + if ($bound >= $best) { + break; // every other edge is at least this far + } + $i = $points[$e]; + $j = $points[$e + 1]; + $best = min($best, $this->edgeMinimumA($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j])); + } + return 6371000.0 * (2 * atan2(sqrt($best), sqrt(1 - $best))); + } + + /** + * The "a" of haversineDistance() (the same operations, so a vertex gives the same distance) + */ + private function haversineA(float $lon, float $lat, float $cosLat, float $x, float $y): float { + $latDelta = deg2rad($y - $lat); + $lonDelta = deg2rad($x - $lon); + return sin($latDelta / 2) * sin($latDelta / 2) + $cosLat * cos(deg2rad($y)) * sin($lonDelta / 2) * sin($lonDelta / 2); + } + + /** + * A lower bound of haversineA() over the bounding box of an edge: a = hav(dlat) + cos(lat) cos(lat2) + * hav(dlon), each term bounded by the box's smallest latitude and longitude difference to the point + * and its smallest cosine + */ + private function edgeBoxLowerBound(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $loY = min($ay, $by); + $hiY = max($ay, $by); + $dLat = $lat < $loY ? $loY - $lat : ($lat > $hiY ? $lat - $hiY : 0.0); + $loX = min($ax, $bx); + $width = max($ax, $bx) - $loX; + $dLon = 0.0; + if ($width < 360) { + $offset = fmod($lon - $loX, 360.0); + if ($offset < 0) { + $offset += 360.0; + } + if ($offset > $width) { + $dLon = min($offset - $width, 360.0 - $offset); + } + } + $cosMin = max(0.0, min(cos(deg2rad($loY)), cos(deg2rad($hiY)))); + $sLat = sin(deg2rad($dLat) / 2); + $sLon = sin(deg2rad($dLon) / 2); + return $sLat * $sLat + $cosLat * $cosMin * $sLon * $sLon; + } + + /** + * The smallest haversineA() of the points of an edge: golden-section search over t (60 steps: t to + * about 1e-13), plus the point a local flat projection picks, whichever is nearer + */ + private function edgeMinimumA(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $dx = $bx - $ax; + $dy = $by - $ay; + $at = function (float $t) use ($lon, $lat, $cosLat, $ax, $ay, $dx, $dy) { + return $this->haversineA($lon, $lat, $cosLat, $ax + $dx * $t, $ay + $dy * $t); + }; + // Local projection around the point, the longitude of the edge's start taken on the point's side + $px = fmod($ax - $lon + 540.0, 360.0) - 180.0; + $qx = ($px + $dx) * $cosLat; + $px *= $cosLat; + $py = $ay - $lat; + $qy = $by - $lat; + $ex = $qx - $px; + $ey = $qy - $py; + $len2 = $ex * $ex + $ey * $ey; + $best = $at($len2 > 0 ? max(0.0, min(1.0, -($px * $ex + $py * $ey) / $len2)) : 0.0); + $ratio = 0.6180339887498949; + $lo = 0.0; + $hi = 1.0; + $c = $hi - $ratio * ($hi - $lo); + $d = $lo + $ratio * ($hi - $lo); + $fc = $at($c); + $fd = $at($d); + for ($step = 0; $step < 60; $step++) { + if ($fc < $fd) { + $hi = $d; + $d = $c; + $fd = $fc; + $c = $hi - $ratio * ($hi - $lo); + $fc = $at($c); + } else { + $lo = $c; + $c = $d; + $fc = $fd; + $d = $lo + $ratio * ($hi - $lo); + $fd = $at($d); + } + } + return min($best, $fc, $fd); + } + + /** + * Find records within a distance from a point (distance to the nearest point of the geometry) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @param array $options ['includeDistance' => false] + * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) + */ + public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $distanceMeters, $options){ + return $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $distanceMeters, $options); + }); + } + + private function withinDistanceUnlocked(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + $includeDistance = $options['includeDistance'] ?? false; + if ($distanceMeters < 0 || is_nan($distanceMeters) || !is_finite($lon) || !is_finite($lat)) { + return []; + } + $distanceMeters = min($distanceMeters, self::EARTH_HALF_CIRCUMFERENCE); // infinite: everything + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $this->circleToBBoxes($lon, $lat, $distanceMeters)) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree (v3.3) has a valid geometry: checked when it was indexed (B-11) + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && $distance <= $distanceMeters) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + } + } + usort($results, function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + $this->clearDistanceCache(); + return $results; + } + + /** + * Find records within a bounding box (geometries whose bounding box overlaps it). + * minLon > maxLon means the box crosses the antimeridian. + * @return array Matching records + */ + public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $minLon, $minLat, $maxLon, $maxLat){ + return $this->withinBBoxUnlocked($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); + }); + } + + private function withinBBoxUnlocked(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + if (is_nan($minLon) || is_nan($minLat) || is_nan($maxLon) || is_nan($maxLat)) { + return []; + } + // Infinite edges reach the edge of the map + $minLon = max(-180.0, min(180.0, $minLon)); + $maxLon = max(-180.0, min(180.0, $maxLon)); + $minLat = max(-90.0, min(90.0, $minLat)); + $maxLat = max(-90.0, min(90.0, $maxLat)); + if ($minLat > $maxLat) { + [$minLat, $maxLat] = [$maxLat, $minLat]; + } + $boxes = $minLon <= $maxLon + ? [[$minLon, $minLat, $maxLon, $maxLat]] + : [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon, $maxLat]]; + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree (v3.3) has a valid geometry: checked when it was indexed (B-11) + } + $recordMBR = $this->calculateMBR($geometry); + if ($recordMBR === null) { + continue; + } + foreach ($boxes as $box) { + if ($this->mbrOverlaps($recordMBR, $box)) { + $results[] = $record; + break; + } + } + } + return $results; + } + + /** + * Find K nearest records to a point + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] + * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + */ + public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $k, $options){ + return $this->nearestUnlocked($dbname, $field, $lon, $lat, $k, $options); + }); + } + + /** + * True when spatial queries on the field use the R-tree (used by the query builder) + */ + private function usesSpatialIndex(string $dbname, string $field): bool{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field){ + return $this->spatialIndexEnabled && $this->dbExists($dbname) + && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); + }); + } + + private function nearestUnlocked(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + $maxDistance = isset($options['maxDistance']) ? (float)$options['maxDistance'] : null; + $includeDistance = $options['includeDistance'] ?? true; + // NaN matches nothing (and NaN radii made the expanding search below loop forever) + if ($k <= 0 || !is_finite($lon) || !is_finite($lat) || ($maxDistance !== null && is_nan($maxDistance)) || !$this->dbExists($dbname)) { + return []; + } + + $indexed = $this->spatialIndexEnabled && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); + // Half the earth's circumference covers everything (also an infinite maxDistance) + $limit = min($maxDistance ?? self::EARTH_HALF_CIRCUMFERENCE, self::EARTH_HALF_CIRCUMFERENCE); + $results = []; + + if ($indexed && $k < $this->countFast($dbname)) { + // Expanding search: nothing outside radius r is closer than r. Results are ordered by the + // rounded distance, so only records rounded below round(r) are certainly in final order + // (a record just outside r can round to the same value and has to win a key tie). + $radius = 1000.0; + while (true) { + $radius = min($radius, $limit); + $found = $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $radius, ['includeDistance' => true]); + if ($radius >= $limit || (count($found) >= $k && $found[$k - 1]['_distance'] < round($radius, 2))) { + $results = $found; + break; + } + $radius *= 4; + } + } else { + foreach ($this->listSegments($dbname) as $segment) { + foreach ($this->readSegmentRecords($dbname, $segment) as $record) { + $geometry = $record[$field] ?? null; + if (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']) { + continue; + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && ($maxDistance === null || $distance <= $maxDistance)) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + } + } + } + usort($results, function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }); + } + + $results = array_slice($results, 0, $k); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + $this->clearDistanceCache(); + return $results; + } + + /** + * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon or MultiPolygon geometry + * @return array Matching records + */ + public function withinPolygon(string $dbname, string $field, array $polygon): array{ + $dbname = $this->normalizeDbName($dbname); + return $this->withReadLock($dbname, function() use ($dbname, $field, $polygon){ + return $this->withinPolygonUnlocked($dbname, $field, $polygon); + }); + } + + private function withinPolygonUnlocked(string $dbname, string $field, array $polygon): array{ + $validation = $this->validateGeoJSON($polygon); + if (!$validation['valid'] || !in_array($validation['type'], ['Polygon', 'MultiPolygon'], true)) { + return []; + } + $polygons = $validation['type'] === 'Polygon' ? [$polygon['coordinates']] : $polygon['coordinates']; + $boxes = []; + foreach ($polygons as $coords) { + $boxes[] = $this->mbrFromPoints($coords[0]); + } + + // One point inside each hole of the query: a record polygon that contains one covers part of the hole + $parts = []; + foreach ($polygons as $coords) { + $holePoints = []; + for ($h = 1; $h < count($coords); $h++) { + $point = $this->pointInsideRing($coords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + $parts[] = ['coords' => $coords, 'holePoints' => $holePoints]; + } + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree (v3.3) has a valid geometry: checked when it was indexed (B-11) + } + if ($this->geometryWithinParts($geometry, $parts)) { + $results[] = $record; + } + } + return $results; + } + + /** + * Within a Polygon or MultiPolygon query (its parts): a single geometry must fit one part; every + * component of a multi-part geometry must fit some part (B-14). A component spanning two adjacent + * parts doesn't fit (the union of the parts isn't computed). + */ + private function geometryWithinParts(array $geometry, array $parts): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + $components = null; + if ($type === 'MultiPoint') { + $components = array_map(function ($p) { return ['type' => 'Point', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'MultiLineString') { + $components = array_map(function ($l) { return ['type' => 'LineString', 'coordinates' => $l]; }, $coords); + } elseif ($type === 'MultiPolygon') { + $components = array_map(function ($p) { return ['type' => 'Polygon', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'GeometryCollection') { + // Members without a position don't count; a collection without any position is never within + $components = array_values(array_filter($geometry['geometries'] ?? [], function ($g) { + return is_array($g) && $this->calculateMBR($g) !== null; + })); + } + if ($components !== null) { + if (empty($components)) { + return false; + } + foreach ($components as $component) { + if (!$this->geometryWithinParts($component, $parts)) { + return false; + } + } + return true; + } + foreach ($parts as $part) { + if ($this->geometryWithinPolygon($geometry, $part['coords'], $part['holePoints'])) { + return true; + } + } + return false; + } + + /** + * A point strictly inside a ring: the middle of the first span a horizontal line between the two + * lowest distinct y values has inside the ring. Null for a degenerate ring. + */ + private function pointInsideRing(array $ring): ?array { + $ys = array_values(array_unique(array_map(function ($p) { return (float)$p[1]; }, $ring))); + sort($ys); + if (count($ys) < 2) { + return null; + } + $y = ($ys[0] + $ys[1]) / 2; + $xs = []; + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + [$x1, $y1] = [(float)$ring[$i][0], (float)$ring[$i][1]]; + [$x2, $y2] = [(float)$ring[$i + 1][0], (float)$ring[$i + 1][1]]; + if (($y1 <= $y && $y2 > $y) || ($y2 <= $y && $y1 > $y)) { + $xs[] = $x1 + ($y - $y1) / ($y2 - $y1) * ($x2 - $x1); + } + } + sort($xs); + return count($xs) >= 2 ? [($xs[0] + $xs[1]) / 2, $y] : null; + } + + /** + * Check if a geometry lies entirely within a polygon. The polygon is closed (its boundary, also a + * hole's boundary, belongs to it, as in withinBBox). Every vertex is inside, no edge crosses a ring of + * the polygon, and every piece of an edge between polygon vertices on it is inside; a record polygon + * must also not contain a point inside one of the polygon's holes ($holePoints). + */ + private function geometryWithinPolygon(array $geometry, array $polygonCoords, ?array $holePoints = null): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + if ($holePoints === null) { + $holePoints = []; + for ($h = 1; $h < count($polygonCoords); $h++) { + $point = $this->pointInsideRing($polygonCoords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + } + + switch ($type) { + case 'Point': + return $this->pointInPolygonClosed((float)$coords[0], (float)$coords[1], $polygonCoords); + case 'MultiPoint': + foreach ($coords as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + return true; + case 'LineString': + return $this->pathWithinPolygon($coords, $polygonCoords); + case 'MultiLineString': + foreach ($coords as $line) { + if (!$this->pathWithinPolygon($line, $polygonCoords)) { + return false; + } + } + return true; + case 'Polygon': + return $this->polygonWithinPolygon($coords, $polygonCoords, $holePoints); + case 'MultiPolygon': + foreach ($coords as $poly) { + if (!$this->polygonWithinPolygon($poly, $polygonCoords, $holePoints)) { + return false; + } + } + return true; + case 'GeometryCollection': + // Members without a position don't count; a collection without any position is never within + $located = false; + foreach ($geometry['geometries'] ?? [] as $g) { + if (is_array($g) && $this->calculateMBR($g) === null) { + continue; + } + if (!$this->geometryWithinPolygon($g, $polygonCoords)) { + return false; + } + $located = true; + } + return $located; + } + return false; + } + + /** + * A polygon (record) within a polygon (query): all its rings within, and no point inside one of the + * query's holes inside it (a record around a hole has its whole boundary inside the query) + */ + private function polygonWithinPolygon(array $rings, array $polygonCoords, array $holePoints): bool { + foreach ($rings as $ring) { + if (!$this->pathWithinPolygon($ring, $polygonCoords)) { + return false; + } + } + foreach ($holePoints as [$x, $y]) { + if ($this->pointInPolygon($x, $y, $rings)) { + return false; + } + } + return true; + } + + /** + * Point inside a polygon or on its boundary (holes excluded, their boundary included) + */ + private function pointInPolygonClosed(float $lon, float $lat, array $polygonCoords): bool { + if (!$this->pointOnRing($lon, $lat, $polygonCoords[0]) && !$this->pointInRing($lon, $lat, $polygonCoords[0])) { + return false; + } + for ($i = 1; $i < count($polygonCoords); $i++) { + if ($this->pointInRing($lon, $lat, $polygonCoords[$i]) && !$this->pointOnRing($lon, $lat, $polygonCoords[$i])) { + return false; + } + } + return true; + } + + /** + * A path within the closed polygon: every vertex inside, no segment properly crossing a ring, and + * every piece of a segment between polygon vertices lying on it inside (a segment can run along a + * boundary or through a hole between two of its vertices) + */ + private function pathWithinPolygon(array $points, array $polygonCoords): bool { + foreach ($points as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + $n = count($points); + for ($i = 0; $i < $n - 1; $i++) { + $a = [(float)$points[$i][0], (float)$points[$i][1]]; + $b = [(float)$points[$i + 1][0], (float)$points[$i + 1][1]]; + $ts = [0.0, 1.0]; + $alongX = abs($b[0] - $a[0]) >= abs($b[1] - $a[1]); + foreach ($polygonCoords as $ring) { + $m = count($ring); + for ($j = 0; $j < $m - 1; $j++) { + if ($this->segmentsCrossProperly($a, $b, $ring[$j], $ring[$j + 1])) { + return false; + } + $v = [(float)$ring[$j][0], (float)$ring[$j][1]]; + if ($this->crossProductDirection($a, $b, $v) == 0 && $this->onSegment($a, $b, $v) && $a != $b) { + $ts[] = $alongX ? ($v[0] - $a[0]) / ($b[0] - $a[0]) : ($v[1] - $a[1]) / ($b[1] - $a[1]); + } + } + } + if (count($ts) > 2) { + sort($ts); + for ($k = 0; $k < count($ts) - 1; $k++) { + if ($ts[$k + 1] - $ts[$k] <= 0) { + continue; + } + $t = ($ts[$k] + $ts[$k + 1]) / 2; + if (!$this->pointInPolygonClosed($a[0] + $t * ($b[0] - $a[0]), $a[1] + $t * ($b[1] - $a[1]), $polygonCoords)) { + return false; + } + } + } elseif (!$this->pointInPolygonClosed(($a[0] + $b[0]) / 2, ($a[1] + $b[1]) / 2, $polygonCoords)) { + return false; // e.g. a chord between two boundary vertices across a concave notch + } + } + return true; + } + + /** + * True when two segments cross at a single interior point (touching endpoints don't count) + */ + private function segmentsCrossProperly(array $p1, array $p2, array $p3, array $p4): bool { + $d1 = $this->crossProductDirection($p3, $p4, $p1); + $d2 = $this->crossProductDirection($p3, $p4, $p2); + $d3 = $this->crossProductDirection($p1, $p2, $p3); + $d4 = $this->crossProductDirection($p1, $p2, $p4); + return (($d1 > 0 && $d2 < 0) || ($d1 < 0 && $d2 > 0)) && (($d3 > 0 && $d4 < 0) || ($d3 < 0 && $d4 > 0)); + } + + // ========================================== + // WRITE BUFFER PUBLIC API + // ========================================== + + /** + * Manually flush buffer for a database + * @param string $dbname + * @return array ['success' => bool, 'flushed' => int, 'error' => string|null] + */ + public function flush($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname){ + return $this->flushUnlocked($dbname); + }); + } + + private function flushUnlocked($dbname){ + $flushed = $this->flushPendingBuffers($dbname); + return ['success' => true, 'flushed' => $flushed, 'error' => null]; + } + + /** + * Flush the write buffers of every database (of this secretKey) in dbDir + * @return array ['databases' => int, 'flushed' => int] + */ + public function flushAllBuffers(){ + $names = []; + if(!empty(self::$lockRegistry)){ + return ['databases' => 0, 'flushed' => 0]; // inside an interrupted operation (see releaseInterruptedSessions) + } + if(is_dir($this->dbDir)){ + foreach(scandir($this->dbDir) ?: [] as $file){ + if(preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+?)(?:_s\\d+)?\\.nonedb\\.buffer(?:\\.flushing(?:\\.\\d+)?)?$/", $file, $m) + && $this->hashDBName($m[2]) === $m[1]){ + $names[$m[2]] = true; + } + } + } + $total = 0; + foreach(array_keys($names) as $dbname){ + $result = $this->flush((string)$dbname); + $total += $result['flushed']; + } + return ['databases' => count($names), 'flushed' => $total]; + } + + /** + * Get buffer information for a database + * @param string $dbname + * @return array + */ + public function getBufferInfo($dbname){ + $dbname = $this->normalizeDbName($dbname); + $fn = function() use ($dbname){ + return $this->getBufferInfoUnlocked($dbname); + }; + return $this->hasLockFile($dbname) || $this->dbFilesExist($dbname) ? $this->withLock($dbname, false, $fn) : $fn(); + } + + private function getBufferInfoUnlocked($dbname){ + $info = [ + 'enabled' => $this->bufferEnabled, + 'sizeLimit' => $this->bufferSizeLimit, + 'countLimit' => $this->bufferCountLimit, + 'flushInterval' => $this->bufferFlushInterval, + 'buffers' => [] + ]; + $describe = function($path){ + clearstatcache(true, $path); + $exists = file_exists($path) && filesize($path) > 0; + return [ + 'exists' => $exists, + 'size' => $exists ? (int)filesize($path) : 0, + 'records' => $exists ? $this->getBufferRecordCount($path) : 0 + ]; + }; + $info['buffers']['main'] = $describe($this->getBufferPath($dbname)); + foreach($this->legacyShardBufferFiles($dbname) as $file){ + if(preg_match('/_s(\d+)\.nonedb\.buffer$/', $file, $m)){ + $info['buffers']['shard_' . $m[1]] = $describe($file); + } + } + return $info; + } + + /** + * Enable or disable write buffering + * @param bool $enable + */ + public function enableBuffering($enable = true){ + $this->bufferEnabled = (bool)$enable; + } + + /** + * Check if buffering is enabled + * @return bool + */ + public function isBufferingEnabled(){ + return $this->bufferEnabled; + } + + /** + * Set buffer size limit (in bytes) + * @param int $bytes + */ + public function setBufferSizeLimit($bytes){ + $this->bufferSizeLimit = max(1024, (int)$bytes); // Minimum 1KB + } + + /** + * Set buffer flush interval (in seconds) + * @param int $seconds 0 to disable time-based flush + */ + public function setBufferFlushInterval($seconds){ + $this->bufferFlushInterval = max(0, (int)$seconds); + } + + /** + * Set buffer count limit + * @param int $count + */ + public function setBufferCountLimit($count){ + $this->bufferCountLimit = max(10, (int)$count); // Minimum 10 records + } + + /** + * Compact a database: drop old record versions and deleted records from the data files. + * Keys are preserved (sharded and non-sharded alike); indexes stay valid. + * @param string $dbname + * @return array ['success' => bool, 'freedSlots' => int, 'totalRecords' => int, 'sharded' => bool] + */ + public function compact($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname){ + $this->refuseInUncommittedWrite($dbname, 'compact'); + return $this->compactUnlocked($dbname); + }); + } + + private function compactUnlocked($dbname){ + $result = array("success" => false, "freedSlots" => 0); + $this->flushPendingBuffers($dbname); + if(!$this->dbExists($dbname)){ + $result['status'] = 'database_not_found'; + return $result; + } + + $sharded = $this->isSharded($dbname); + $freed = 0; + if($sharded){ + $meta = $this->getCachedMeta($dbname, true); + $freed = (int)($meta['deletedCount'] ?? 0); + } + $total = 0; + foreach($this->listSegments($dbname) as $segment){ + $compacted = $this->compactJsonl($dbname, $segment); + if(!$sharded){ + $freed += $compacted['freed']; + } + $total += $compacted['compacted']; + } + if($sharded){ + $this->removeEmptyShards($dbname); + $meta = $this->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + $meta['shards'][$i]['deleted'] = 0; + } + $meta['deletedCount'] = 0; + $meta['totalRecords'] = $total; + $this->writeMeta($dbname, $meta); + } + + $result['success'] = true; + $result['freedSlots'] = $freed; + $result['totalRecords'] = $total; + $result['sharded'] = $sharded; + if($sharded){ + $result['newShardCount'] = count($this->listSegments($dbname)); + } + return $result; + } + + /** + * Remove shards without records (keys never move, so nothing else changes). The shard that holds + * the newest key is kept: new records go there. + */ + private function removeEmptyShards($dbname){ + $meta = $this->getCachedMeta($dbname, true); + if($meta === null || count($meta['shards']) < 2){ + return; + } + $lastShard = intdiv(max(0, (int)$meta['nextKey'] - 1), (int)$meta['shardSize']); + $kept = []; + $removed = []; + foreach($meta['shards'] as $shard){ + $shardId = (int)$shard['id']; + $index = $this->readJsonlIndex($dbname, $shardId); + if($shardId !== $lastShard && ($index === null || empty($index['p']))){ + $removed[] = $shardId; + } else { + $kept[] = $shard; + } + } + if(empty($removed)){ + return; + } + $meta['shards'] = $kept; + // The meta must not lose this change in a power loss after the files are gone (where PHP can sync) + $this->writeMeta($dbname, $meta, true); + + $manifest = $this->getIndexManifest($dbname); + $this->bumpEpoch($dbname); // shard files go away + foreach($removed as $shardId){ + $dataPath = $this->getShardPath($dbname, $shardId); + foreach($manifest['fields'] as $field){ + $this->deleteFieldIndexFile($dbname, $field, $shardId); + } + $this->removeSpatialShardFiles($dbname, $shardId); + $this->fsUnlink($dataPath . '.jidx'); + $this->fsUnlink($dataPath); + $this->markFileNotExists($dataPath); + unset($this->indexCache[$dataPath . '.jidx'], $this->jsonlFormatCache[$dataPath]); + } + $this->syncDirectory($this->dbDir); + foreach($manifest['fields'] as $field){ + $this->rebuildGlobalFieldIndex($dbname, $field); + } + } + /** + * Manually trigger migration to sharded format + * @param string $dbname + * @return array ["success" => bool, "status" => string] + */ + public function migrate($dbname){ + $dbname = $this->normalizeDbName($dbname); + return $this->withLock($dbname, true, function() use ($dbname){ + $this->refuseInUncommittedWrite($dbname, 'migrate'); + return $this->migrateUnlocked($dbname); + }); + } + + private function migrateUnlocked($dbname){ + if($this->isSharded($dbname)){ + return array("success" => true, "status" => "already_sharded"); + } + $this->flushPendingBuffers($dbname); + if($this->isSharded($dbname)){ + return array("success" => true, "status" => "migrated"); + } + if(!$this->cachedFileExists($this->getDataPath($dbname))){ + return array("success" => false, "status" => "database_not_found"); + } + if($this->migrateToSharded($dbname)){ + return array("success" => true, "status" => "migrated"); + } + return array("success" => false, "status" => "migration_failed"); + } + + /** + * Check if sharding is enabled + * @return bool + */ + public function isShardingEnabled(){ + return $this->shardingEnabled; + } + + /** + * Get current shard size setting + * @return int + */ + public function getShardSize(){ + return $this->shardSize; + } + + // ========================================== + // QUERY BUILDER (FLUENT INTERFACE) + // ========================================== + + /** + * Create a new query builder for fluent interface + * @param string $dbname + * @return noneDBQuery + */ + public function query(string $dbname): noneDBQuery { + return new noneDBQuery($this, $this->normalizeDbName($dbname)); + } +} + +/** + * Query Builder for noneDB - Fluent Interface + * + * Allows method chaining for queries: + * $db->query("users")->where(["active" => true])->sort("name")->limit(10)->get(); + */ +class noneDBQuery { + private noneDB $db; + private string $dbname; + /** @var array[] where() groups; all of them must match (AND) */ + private array $whereFilters = []; + /** @var array[] orWhere() groups; a record matches if the where groups match OR any orWhere group does */ + private array $orWhereFilters = []; + private array $whereInFilters = []; + private array $whereNotInFilters = []; + /** @var array[] whereNot() groups */ + private array $whereNotFilters = []; + private array $likeFilters = []; + private array $notLikeFilters = []; + private array $betweenFilters = []; + private array $notBetweenFilters = []; + private array $selectFields = []; + private array $exceptFields = []; + private ?string $groupByField = null; + private array $havingFilters = []; + private array $searchFilters = []; + private array $joinConfigs = []; + /** @var array[] sort keys in priority order: [['field' => ..., 'order' => 'asc'|'desc'], ...] */ + private array $sortKeys = []; + private ?int $limitCount = null; + private int $offsetCount = 0; + + // Spatial query support + private array $spatialFilters = []; + private ?array $distanceConfig = null; + /** @var array|null Statistics of the last scan (explain()) */ + private ?array $lastScan = null; + + /** @var \Closure Calls noneDB internals without Reflection */ + private \Closure $internal; + + private const OPERATORS = ['$gt', '$gte', '$lt', '$lte', '$ne', '$eq', '$in', '$nin', '$exists', '$like', '$regex', '$contains']; + + /** + * @param noneDB $db + * @param string $dbname + */ + public function __construct(noneDB $db, string $dbname) { + $this->db = $db; + $this->dbname = $dbname; + $this->internal = \Closure::bind(function (string $method, array $args) { + return $this->$method(...$args); + }, $db, noneDB::class); + } + + /** + * Debug output without the parent instance's internals + */ + public function __debugInfo() { + return [ + 'dbname' => $this->dbname, + 'where' => $this->whereFilters, + 'orWhere' => $this->orWhereFilters, + 'sort' => $this->sortKeys, + 'limit' => $this->limitCount, + 'offset' => $this->offsetCount, + ]; + } + + // ========================================== + // FILTER HELPERS + // ========================================== + + /** + * Call a private method of the parent noneDB instance + * @param string $methodName + * @param mixed ...$args + * @return mixed + */ + private function callPrivateMethod(string $methodName, ...$args) { + return ($this->internal)($methodName, $args); + } + + /** + * True for a filter value that is an operator spec: a non-empty array whose keys all start with "$" + * @throws \InvalidArgumentException when operators and plain keys are mixed + */ + private function isOperatorSpec($value): bool { + if (!is_array($value) || empty($value)) { + return false; + } + $operators = 0; + foreach (array_keys($value) as $key) { + if (is_string($key) && $key !== '' && $key[0] === '$') { + $operators++; + } + } + if ($operators > 0 && $operators !== count($value)) { + throw new \InvalidArgumentException('noneDB: a filter value cannot mix operators ($gt, $in, ...) with plain keys'); + } + return $operators > 0; + } + + /** + * Compile where()/orWhere() groups once per query run: operators are validated and patterns + * turned into regexes here instead of for every record. + * @return array[] per group a list of [field, operator or null for a plain value, operand] + * @throws \InvalidArgumentException for unknown operators or invalid operands + */ + private function compileGroups(array $groups): array { + $compiled = []; + foreach ($groups as $group) { + $conditions = []; + foreach ($group as $field => $value) { + if ($this->isOperatorSpec($value)) { + foreach ($value as $operator => $operand) { + $conditions[] = [(string)$field, $operator, $this->compileOperand($operator, $operand)]; + } + } elseif ((string)$field === 'key') { + // As find(): key 5, "5" or a list of them (only one plain where() went through find()) + $conditions[] = ['key', '#keys', array_flip($this->callPrivateMethod('normalizeKeyFilter', $value))]; + } else { + $conditions[] = [(string)$field, null, $value]; + } + } + $compiled[] = $conditions; + } + return $compiled; + } + + private function compileOperand(string $operator, $operand) { + switch ($operator) { + case '$in': + case '$nin': + if (!is_array($operand)) { + throw new \InvalidArgumentException("noneDB: {$operator} needs an array"); + } + return $this->membership($operand); + case '$like': + return $this->likeSpec($operand); + case '$regex': + return self::regexOperand($operand); + case '$gt': + case '$gte': + case '$lt': + case '$lte': + case '$ne': + case '$eq': + case '$exists': + case '$contains': + return $operand; + default: + throw new \InvalidArgumentException("noneDB: unknown operator '{$operator}' (supported: " . implode(', ', self::OPERATORS) . ')'); + } + } + + /** + * Check if a record matches one compiled where()/orWhere() group (AND of its conditions). + * Plain values compare with ===; operator conditions apply the operator. + */ + private function matchesGroup(array $record, array $conditions): bool { + foreach ($conditions as [$field, $operator, $operand]) { + if ($operator === null) { + if (!array_key_exists($field, $record) || $record[$field] !== $operand) { + return false; + } + } elseif ($operator === '#keys') { + if (!isset($record['key']) || !is_int($record['key']) || !isset($operand[$record['key']])) { + return false; + } + } elseif (!$this->matchesOperator($record, $field, $operator, $operand)) { + return false; + } + } + return true; + } + + /** + * where() groups AND-ed, OR any orWhere() group. Without where(), only orWhere() groups decide. + * @param array[] $where compiled where() groups + * @param array[] $orWhere compiled orWhere() groups + */ + private function matchesWhereLogic(array $record, array $where, array $orWhere): bool { + if (!empty($where)) { + $all = true; + foreach ($where as $conditions) { + if (!$this->matchesGroup($record, $conditions)) { + $all = false; + break; + } + } + if ($all) { + return true; + } + } + foreach ($orWhere as $conditions) { + if ($this->matchesGroup($record, $conditions)) { + return true; + } + } + return false; + } + + /** + * Compare two values for ordering operators ($gt, between, ...). + * Numbers (and numeric strings) compare numerically, other strings lexically; anything else + * (null, booleans, arrays, number vs. non-numeric string) is not comparable. + * @internal also used by noneDB::between(); not part of the public API + * @return int|null + */ + public static function compareValues($a, $b): ?int { + if ($a === null || $b === null || is_bool($a) || is_bool($b) || is_array($a) || is_array($b) || is_object($a) || is_object($b)) { + return null; + } + // NAN compares as greater than everything with <=> ($gt NAN matched every number) + if ((is_float($a) && is_nan($a)) || (is_float($b) && is_nan($b))) { + return null; + } + $numA = self::numberOf($a); + $numB = self::numberOf($b); + if ($numA !== null && $numB !== null) { + return $numA <=> $numB; + } + if (is_string($a) && is_string($b)) { + return strcmp($a, $b) <=> 0; + } + return null; + } + + /** + * The number of an int, float or numeric string, else null. A numeric string may have leading and + * trailing whitespace on every PHP version (7.4's is_numeric() rejects trailing whitespace; the sort + * order and range indexes use the same rule) + * @internal also used by noneDB's aggregates; not part of the public API + * @return int|float|null + */ + public static function numberOf($value) { + if (is_int($value) || is_float($value)) { + return $value; + } + if (is_string($value)) { + $trimmed = rtrim($value, " \t\n\r\v\f"); + return is_numeric($trimmed) ? $trimmed + 0 : null; + } + return null; + } + + /** + * Compiled like()/$like pattern ("^" start anchor, trailing "$" end anchor, rest literal) + */ + private function likeSpec($pattern): array { + if (!is_scalar($pattern)) { + throw new \InvalidArgumentException('noneDB: like pattern must be a string'); + } + return self::compileLike((string)$pattern); + } + + /** + * Run preg_match, turning compile errors and backtrack-limit failures into exceptions + */ + private static function regexMatch(string $regex, string $subject): bool { + $result = @preg_match($regex, $subject); + if ($result === false) { + $error = preg_last_error(); + if ($error === PREG_BAD_UTF8_ERROR || $error === PREG_BAD_UTF8_OFFSET_ERROR) { + return false; // subject isn't UTF-8: can't match a UTF-8 pattern + } + if ($error === PREG_NO_ERROR) { + throw new \InvalidArgumentException('noneDB: invalid regular expression'); + } + throw new \RuntimeException('noneDB: regular expression failed (' . (function_exists('preg_last_error_msg') ? preg_last_error_msg() : 'error ' . $error) . ')'); + } + return $result === 1; + } + + /** + * Regex for a $regex operand: the pattern body, case-insensitive (delimiter added here) + */ + private static function regexOperand($pattern): string { + if (!is_string($pattern) || $pattern === '') { + throw new \InvalidArgumentException('noneDB: $regex needs a non-empty string pattern'); + } + // Escape the delimiter, but not where "~" is already escaped: skip every "\x" pair first (a "\\" + // before "~" is an escaped backslash, the "~" still needs escaping) + $body = preg_replace('/\\\\.(*SKIP)(*FAIL)|~/s', '\\~', $pattern); + $regex = '~' . $body . '~i' . (preg_match('//u', $pattern) === 1 ? 'u' : ''); + if (@preg_match($regex, '') === false) { + throw new \InvalidArgumentException("noneDB: invalid \$regex pattern '{$pattern}'"); + } + return $regex; + } + + /** + * Apply a single comparison operator (operand compiled by compileOperand(): $like/$regex are regexes) + */ + private function matchesOperator(array $record, string $field, string $operator, $operand): bool { + $hasField = array_key_exists($field, $record); + $fieldValue = $hasField ? $record[$field] : null; + + switch ($operator) { + case '$gt': + $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + return $c !== null && $c > 0; + case '$gte': + $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + return $c !== null && $c >= 0; + case '$lt': + $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + return $c !== null && $c < 0; + case '$lte': + $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + return $c !== null && $c <= 0; + case '$ne': + return !$hasField || $fieldValue !== $operand; + case '$eq': + return $hasField && $fieldValue === $operand; + case '$in': + return $hasField && $this->isMember($fieldValue, $operand); + case '$nin': + return !$hasField || !$this->isMember($fieldValue, $operand); + case '$exists': + return $operand ? $hasField : !$hasField; + case '$like': + if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { + return false; + } + return self::likeMatch($operand, (string)$fieldValue); + case '$regex': + if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { + return false; + } + return self::regexMatch($operand, (string)$fieldValue); + case '$contains': + if (!$hasField) { + return false; + } + if (is_array($fieldValue)) { + return in_array($operand, $fieldValue, true); + } + if (is_string($fieldValue) && is_scalar($operand) && !is_bool($operand)) { + return (string)$operand === '' || strpos($fieldValue, (string)$operand) !== false; + } + return false; + default: + throw new \InvalidArgumentException("noneDB: unknown operator '{$operator}' (supported: " . implode(', ', self::OPERATORS) . ')'); + } + } + + /** + * Check if a record matches all the other filters (whereNot, whereIn, whereNotIn, like, notLike, + * between, notBetween, search) + */ + private function matchesAdvancedFilters(array $record): bool { + foreach ($this->whereNotFilters as $fields) { + foreach ($fields as $conditions) { + if ($this->matchesGroup($record, $conditions)) { + return false; + } + } + } + + foreach ($this->whereInFilters as $filter) { + if (!array_key_exists($filter['field'], $record)) return false; + if (!$this->isMember($record[$filter['field']], $filter['values'])) return false; + } + + foreach ($this->whereNotInFilters as $filter) { + if (array_key_exists($filter['field'], $record) && $this->isMember($record[$filter['field']], $filter['values'])) { + return false; + } + } + + foreach ($this->likeFilters as $like) { + $value = $record[$like['field']] ?? null; + if (!is_scalar($value) || is_bool($value)) return false; + if (!self::likeMatch($like['like'], (string)$value)) return false; + } + + foreach ($this->notLikeFilters as $notLike) { + $value = $record[$notLike['field']] ?? null; + if (is_scalar($value) && !is_bool($value) && self::likeMatch($notLike['like'], (string)$value)) { + return false; + } + } + + foreach ($this->betweenFilters as $between) { + if (!array_key_exists($between['field'], $record)) return false; + $value = $record[$between['field']]; + $lo = self::compareValues($value, $between['min']); + $hi = self::compareValues($value, $between['max']); + if ($lo === null || $hi === null || $lo < 0 || $hi > 0) return false; + } + + foreach ($this->notBetweenFilters as $notBetween) { + if (!array_key_exists($notBetween['field'], $record)) continue; + $value = $record[$notBetween['field']]; + $lo = self::compareValues($value, $notBetween['min']); + $hi = self::compareValues($value, $notBetween['max']); + if ($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0) return false; + } + + foreach ($this->searchFilters as $search) { + $term = (string)$search['term']; + if ($term === '') continue; + $searchFields = empty($search['fields']) ? array_keys($record) : $search['fields']; + $found = false; + foreach ($searchFields as $field) { + if (empty($search['fields']) && ($field === 'key' || $field === '_distance')) continue; // not content + $value = $record[$field] ?? null; + if (!is_scalar($value) || is_bool($value)) continue; + if (self::containsIgnoreCase((string)$value, $term)) { + $found = true; + break; + } + } + if (!$found) return false; + } + + return true; + } + + /** + * Case-insensitive substring test with Unicode case folding when mbstring is available + */ + private static function containsIgnoreCase(string $haystack, string $needle): bool { + if (noneDB::hasMbstring() && preg_match('//u', $haystack) === 1 && preg_match('//u', $needle) === 1) { + return strpos(self::foldCase($haystack), self::foldCase($needle)) !== false; + } + return stripos($haystack, $needle) !== false; + } + + /** + * Case folding for matching (Unicode full folding: "ß" and "ẞ" are "ss", "ſ" is "s", "ς" is "σ"). + * "İ" folds to "i" + combining dot (U+0307); the dot is dropped so "İzmir", "IZMIR" and "izmir" all + * match each other. Language independent: "I" and "ı" don't match. + */ + private static function foldCase(string $s): string { + return str_replace("\u{0307}", '', mb_convert_case($s, MB_CASE_FOLD, 'UTF-8')); + } + + /** + * Compile a like() pattern: a leading "^" anchors the start, a trailing "$" the end, everything else + * is literal and case-insensitive. + * @internal also used by noneDB::like(); not part of the public API + * @return array + */ + public static function compileLike(string $pattern): array { + $start = $pattern !== '' && $pattern[0] === '^'; + $end = strlen($pattern) > ($start ? 1 : 0) && substr($pattern, -1) === '$'; + $body = (string)substr($pattern, $start ? 1 : 0, strlen($pattern) - ($start ? 1 : 0) - ($end ? 1 : 0)); + $utf8 = preg_match('//u', $body) === 1; + return [ + 'start' => $start, 'end' => $end, + // Folded body for UTF-8 subjects (null: no mbstring, or the pattern isn't UTF-8) + 'folded' => $utf8 && noneDB::hasMbstring() ? self::foldCase($body) : null, + // Otherwise a regex (PCRE /iu: simple case folding) + // ("D": the end anchor is the end of the value, not also before a final newline) + 'regex' => '~' . ($start ? '^' : '') . preg_quote($body, '~') . ($end ? '$' : '') . '~D' . ($utf8 ? 'iu' : 'i'), + ]; + } + + /** + * Match a compiled like() pattern + * @internal also used by noneDB::like(); not part of the public API + */ + public static function likeMatch(array $like, string $subject): bool { + if ($like['folded'] === null || preg_match('//u', $subject) !== 1) { + return self::regexMatch($like['regex'], $subject); + } + $t = self::foldCase($subject); + $b = $like['folded']; + if ($like['start'] && $like['end']) { + return $t === $b; + } + if ($like['start']) { + return strncmp($t, $b, strlen($b)) === 0; + } + if ($like['end']) { + return $b === '' || (strlen($t) >= strlen($b) && substr($t, -strlen($b)) === $b); + } + return $b === '' || strpos($t, $b) !== false; + } + + private function hasAdvancedFilters(): bool { + return !empty($this->whereNotFilters) || !empty($this->whereInFilters) || !empty($this->whereNotInFilters) + || !empty($this->likeFilters) || !empty($this->notLikeFilters) || !empty($this->betweenFilters) + || !empty($this->notBetweenFilters) || !empty($this->searchFilters); + } + + // ========================================== + // CHAINABLE METHODS (return $this) + // ========================================== + + /** + * Add where filters. Each call adds a group; all groups must match (AND). + * Values are compared with === or, for arrays like ['$gt' => 10], with operators. + * @param array $filters + * @return self + */ + public function where(array $filters): self { + if (!empty($filters)) { + foreach ($filters as $value) { + $this->isOperatorSpec($value); // validates early + } + $this->whereFilters[] = $filters; + } + return $this; + } + + /** + * Add pattern matching filter + * @param string $field + * @param string $pattern "^" anchors the start, a trailing "$" the end; everything else is literal + * @return self + */ + public function like(string $field, string $pattern): self { + $this->likeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; + return $this; + } + + /** + * Add range filter (min <= value <= max) + * @param string $field + * @param mixed $min + * @param mixed $max + * @return self + */ + public function between(string $field, $min, $max): self { + $this->betweenFilters[] = ['field' => $field, 'min' => $min, 'max' => $max]; + return $this; + } + + /** + * Add a sort key. The first call is the primary key; later calls break ties. + * @param string $field + * @param string $order 'asc' or 'desc' + * @return self + */ + public function sort(string $field, string $order = 'asc'): self { + $this->sortKeys[] = ['field' => $field, 'order' => strtolower($order) === 'desc' ? 'desc' : 'asc']; + return $this; + } + + /** + * Limit results + * @param int $count + * @return self + */ + public function limit(int $count): self { + $this->limitCount = max(0, $count); + return $this; + } + + /** + * Skip first N results (for pagination) + * @param int $count + * @return self + */ + public function offset(int $count): self { + $this->offsetCount = max(0, $count); + return $this; + } + + /** + * Alias for offset() - skip first N results + * @param int $count + * @return self + */ + public function skip(int $count): self { + return $this->offset($count); + } + + /** + * Alias for sort() + * @param string $field + * @param string $order 'asc' or 'desc' + * @return self + */ + public function orderBy(string $field, string $order = 'asc'): self { + return $this->sort($field, $order); + } + + /** + * Add an OR group: a record matches if all where() groups match OR any orWhere() group does + * @param array $filters + * @return self + */ + public function orWhere(array $filters): self { + foreach ($filters as $value) { + $this->isOperatorSpec($value); + } + $this->orWhereFilters[] = $filters; + return $this; + } + + /** + * Filter where field value is in given array (strict comparison) + * @param string $field + * @param array $values + * @return self + */ + public function whereIn(string $field, array $values): self { + $this->whereInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values)), 'raw' => array_values($values)]; + return $this; + } + + /** + * Value list for strict membership tests: longer lists become a hash set of exact identities + * (noneDB::valueKey: same result as in_array(..., true), without scanning the list per record) + * @return array|\stdClass + */ + private function membership(array $values) { + if (count($values) <= 8) { + return $values; + } + $valueKey = $this->callPrivateMethod('valueKeyFunction'); + $set = []; + foreach ($values as $value) { + $set[$valueKey($value)] = true; + } + return (object)['set' => $set, 'key' => $valueKey]; + } + + private function isMember($value, $membership): bool { + if ($membership instanceof \stdClass) { + return isset($membership->set[($membership->key)($value)]); + } + return in_array($value, $membership, true); + } + + /** + * Filter where field value is NOT in given array (strict comparison) + * @param string $field + * @param array $values + * @return self + */ + public function whereNotIn(string $field, array $values): self { + $this->whereNotInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values))]; + return $this; + } + + /** + * Exclude records where a field equals the value (strict comparison) + * @param array $filters + * @return self + */ + public function whereNot(array $filters): self { + if (!empty($filters)) { + // Per field: the field matches when all its conditions match (AND, like in where()); + // a record is excluded when any field matches. Operators are checked now, like where(). + $fields = []; + foreach ($filters as $field => $value) { + $fields[] = $this->compileGroups([[$field => $value]])[0]; + } + $this->whereNotFilters[] = $fields; + } + return $this; + } + + /** + * Filter where field does NOT match pattern + * @param string $field + * @param string $pattern + * @return self + */ + public function notLike(string $field, string $pattern): self { + $this->notLikeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; + return $this; + } + + /** + * Filter where field value is NOT between min and max + * @param string $field + * @param mixed $min + * @param mixed $max + * @return self + */ + public function notBetween(string $field, $min, $max): self { + $this->notBetweenFilters[] = ['field' => $field, 'min' => $min, 'max' => $max]; + return $this; + } + + /** + * Select only specific fields in results ("key" is always included) + * @param array $fields + * @return self + */ + public function select(array $fields): self { + $this->selectFields = array_values($fields); + return $this; + } + + /** + * Exclude specific fields from results + * @param array $fields + * @return self + */ + public function except(array $fields): self { + $this->exceptFields = array_values($fields); + return $this; + } + + /** + * Group results by field. get() then returns groups: ['_group' => value, '_items' => [...], '_count' => n] + * @param string $field + * @return self + */ + public function groupBy(string $field): self { + $this->groupByField = $field; + return $this; + } + + /** + * Filter groups by aggregate condition. Only used together with groupBy(); without it, and for + * unknown aggregates or operators, the condition is ignored (passes) + * @param string $aggregate 'count', 'sum:field', 'avg:field', 'min:field', 'max:field' + * @param string $operator '>', '<', '>=', '<=', '=', '!=' + * @param mixed $value + * @return self + */ + public function having(string $aggregate, string $operator, $value): self { + $this->havingFilters[] = ['aggregate' => $aggregate, 'operator' => $operator, 'value' => $value]; + return $this; + } + + /** + * Case-insensitive full-text search across fields + * @param string $term Search term + * @param array $fields Fields to search in (empty = all fields except "key") + * @return self + */ + public function search(string $term, array $fields = []): self { + $this->searchFilters[] = ['term' => $term, 'fields' => array_values($fields)]; + return $this; + } + + /** + * Join with another database + * @param string $foreignDb Foreign database name + * @param string $localKey Local key field + * @param string $foreignKey Foreign key field + * @param string|null $alias Field that receives the joined record (default: the foreign database name; + * it replaces a field of the same name in the result, never in storage) + * @return self + */ + public function join(string $foreignDb, string $localKey, string $foreignKey, ?string $alias = null): self { + $this->joinConfigs[] = [ + 'foreignDb' => $foreignDb, + 'localKey' => $localKey, + 'foreignKey' => $foreignKey, + 'alias' => $alias ?? $foreignDb + ]; + return $this; + } + + // ========================================== + // SPATIAL QUERY METHODS + // ========================================== + + /** + * Filter records within a distance from a point + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @return self + */ + public function withinDistance(string $field, float $lon, float $lat, float $distanceMeters): self { + $this->spatialFilters[] = ['type' => 'withinDistance', 'field' => $field, 'lon' => $lon, 'lat' => $lat, 'distance' => $distanceMeters]; + return $this; + } + + /** + * Filter records within a bounding box + * @param string $field GeoJSON field name + * @param float $minLon Minimum longitude + * @param float $minLat Minimum latitude + * @param float $maxLon Maximum longitude + * @param float $maxLat Maximum latitude + * @return self + */ + public function withinBBox(string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): self { + $this->spatialFilters[] = ['type' => 'withinBBox', 'field' => $field, 'minLon' => $minLon, 'minLat' => $minLat, 'maxLon' => $maxLon, 'maxLat' => $maxLat]; + return $this; + } + + /** + * The K records nearest to a point among those matching the other filters (adds _distance in meters) + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @return self + */ + public function nearest(string $field, float $lon, float $lat, int $k): self { + $this->spatialFilters[] = ['type' => 'nearest', 'field' => $field, 'lon' => $lon, 'lat' => $lat, 'k' => max(0, $k)]; + return $this; + } + + /** + * Filter records within a polygon + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon geometry + * @return self + */ + public function withinPolygon(string $field, array $polygon): self { + $this->spatialFilters[] = ['type' => 'withinPolygon', 'field' => $field, 'polygon' => $polygon]; + return $this; + } + + /** + * Add _distance to every result: meters from the point to the geometry, measured like + * withinDistance()/nearest() (0 inside a polygon, otherwise to the nearest point of the geometry). + * It is computed before sorting, so sort('_distance') works. + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @return self + */ + public function withDistance(string $field, float $lon, float $lat): self { + $this->distanceConfig = ['field' => $field, 'lon' => $lon, 'lat' => $lat]; + return $this; + } + + // ========================================== + // EXECUTION + // ========================================== + + /** + * Records the spatial filters allow ([key => record]); nearest() returns its candidates + * in distance order with _distance set. Null when there are no spatial filters. + * @param float|null $nearestRadius Only the records within this many meters for a nearest() filter + * (null: all records in distance order) + * @param float|null $nearestBoundary Set when the radius was used: records with a smaller _distance + * are exactly those of the full list (anything outside the radius is at least this far) + */ + private function spatialCandidates(?float $nearestRadius = null, ?float &$nearestBoundary = null): ?array { + $nearestBoundary = null; + if (empty($this->spatialFilters)) { + return null; + } + $result = null; + foreach ($this->spatialFilters as $filter) { + switch ($filter['type']) { + case 'withinDistance': + $rows = $this->db->withinDistance($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $filter['distance'], ['includeDistance' => true]); + break; + case 'withinBBox': + $rows = $this->db->withinBBox($this->dbname, $filter['field'], $filter['minLon'], $filter['minLat'], $filter['maxLon'], $filter['maxLat']); + break; + case 'nearest': + // Records in distance order; the k cut happens after the other filters + if ($filter['k'] <= 0) { + $rows = []; + } elseif ($nearestRadius !== null) { + $rows = $this->db->withinDistance($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $nearestRadius, ['includeDistance' => true]); + $nearestBoundary = round($nearestRadius, 2); + } else { + $all = max(1, $this->db->count($this->dbname)); + $rows = $this->db->nearest($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $all, ['includeDistance' => true]); + } + break; + case 'withinPolygon': + $rows = $this->db->withinPolygon($this->dbname, $filter['field'], $filter['polygon']); + break; + default: + $rows = []; + } + $keyed = []; + foreach ($rows as $row) { + if (isset($row['key'])) { + $keyed[$row['key']] = $row; + } + } + if ($result === null) { + $result = $keyed; + } elseif ($filter['type'] === 'nearest') { + // nearest() orders and cuts by its own distance (B-4): its order and _distance win + $result = array_intersect_key($keyed, $result); + } else { + // Intersect by key, keep the order and fields of the earlier result + $result = array_intersect_key($result, $keyed); + foreach ($result as $key => $row) { + if (isset($keyed[$key]['_distance']) && !isset($row['_distance'])) { + $result[$key]['_distance'] = $keyed[$key]['_distance']; + } + } + } + } + return $result ?? []; + } + + /** + * The plan of the query's filters (see noneDB's Planner): what field indexes, global field indexes and + * key lookups can narrow down. Every record read is still checked against every filter. + * @return array + */ + private function buildPlan(): array { + $where = []; + foreach ($this->whereFilters as $group) { + $where[] = $this->groupPlan($group); + } + $node = count($where) === 1 ? $where[0] : (empty($where) ? null : ['and', $where]); + if (!empty($this->orWhereFilters)) { + // where groups (all) OR any orWhere group; an empty orWhere() group matches everything + $branches = $node === null ? [] : [$node]; + foreach ($this->orWhereFilters as $group) { + $branches[] = $this->groupPlan($group); + } + $node = ['or', $branches]; + } + $and = $node === null ? [] : [$node]; + foreach ($this->whereInFilters as $filter) { + $and[] = $this->inPlan($filter['field'], $filter['raw'] ?? []); + } + foreach ($this->betweenFilters as $between) { + $and[] = ['range', $between['field'], '$gte', $between['min']]; + $and[] = ['range', $between['field'], '$lte', $between['max']]; + } + return self::simplifyPlan(['and', $and]); + } + + /** + * Plan of one where()/orWhere() group (all its conditions) + */ + private function groupPlan(array $group): array { + $and = []; + foreach ($group as $field => $value) { + $field = (string)$field; + if (!$this->isOperatorSpec($value)) { + if ($field === 'key') { + $and[] = ['keys', $this->callPrivateMethod('normalizeKeyFilter', $value)]; + } elseif (!is_array($value)) { + $and[] = ['eq', $field, $value]; + } + continue; + } + foreach ($value as $operator => $operand) { + if ($operator === '$eq' && !is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->callPrivateMethod('normalizeKeyFilter', $operand)] : ['eq', $field, $operand]; + } elseif ($operator === '$in' && is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->callPrivateMethod('normalizeKeyFilter', array_values($operand))] : $this->inPlan($field, $operand); + } elseif (in_array($operator, ['$gt', '$gte', '$lt', '$lte'], true) && $field !== 'key' && !is_array($operand)) { + $and[] = ['range', $field, $operator, $operand]; + } + // $ne, $nin, $exists, $like, $regex, $contains: checked on the records only + } + } + return self::simplifyPlan(['and', $and]); + } + + private function inPlan(string $field, array $values): array { + foreach ($values as $value) { + if (is_array($value) || is_object($value)) { + return ['any']; // indexes hold no arrays + } + } + return ['in', $field, array_values($values)]; + } + + private static function simplifyPlan(array $node): array { + if ($node[0] !== 'and' && $node[0] !== 'or') { + return $node; + } + $children = []; + foreach ($node[1] as $child) { + $child = self::simplifyPlan($child); + if ($child[0] === 'any') { + if ($node[0] === 'or') { + return ['any']; // a branch that matches anything + } + continue; // AND with "anything" is the rest + } + $children[] = $child; + } + if (empty($children)) { + return ['any']; + } + return count($children) === 1 ? $children[0] : [$node[0], $children]; + } + + /** + * True when the filters are one plain where() group with a key filter: find() keeps the order of + * the keys asked for, which a scan in key order wouldn't + */ + private function keyOrderLookup(): bool { + if (count($this->whereFilters) !== 1 || !empty($this->orWhereFilters) || !array_key_exists('key', $this->whereFilters[0])) { + return false; + } + foreach ($this->whereFilters[0] as $value) { + if ($this->isOperatorSpec($value)) { + return false; + } + } + return true; + } + + /** + * Filtered records (no join, grouping or projection) + * @param bool $earlyExit Stop once offset+limit records are collected (only when order can't change) + */ + private function filteredRecords(bool $earlyExit, ?int $target = null, ?int $topK = null): array { + // One shared lock for every read the query makes (count, spatial rounds, records), so they + // all see the same state of the database + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + return $this->callPrivateMethod('withReadLock', $dbname, function () use ($earlyExit, $target, $topK) { + return $this->filteredRecordsLocked($earlyExit, $target, $topK); + }); + } + + /** + * @param int|null $target Matches enough to stop early (exists()); default: offset + limit without sort + */ + private function filteredRecordsLocked(bool $earlyExit, ?int $target = null, ?int $topK = null): array { + $nearest = null; + $nearestFilters = 0; + foreach ($this->spatialFilters as $filter) { + if ($filter['type'] === 'nearest') { + $nearest = $nearest === null ? $filter['k'] : min($nearest, $filter['k']); + $nearestFilters++; + } + } + // A single nearest() as the first spatial filter (its distances are the ones kept) on an indexed + // field doesn't need every record in distance order: search a growing radius until k records match + $nearestRadius = null; + $total = 0; + if ($nearestFilters === 1 && $this->spatialFilters[0]['type'] === 'nearest' && $nearest > 0 + && $this->callPrivateMethod('usesSpatialIndex', $this->dbname, $this->spatialFilters[0]['field'])) { + $nearestRadius = 1000.0; + $total = $this->db->count($this->dbname); + } + + $needWhere = !empty($this->whereFilters) || !empty($this->orWhereFilters); + $where = $needWhere ? $this->compileGroups($this->whereFilters) : []; + $orWhere = $needWhere ? $this->compileGroups($this->orWhereFilters) : []; + $needAdvanced = $this->hasAdvancedFilters(); + if ($target !== null && $earlyExit && $nearest === null) { + // exists(): independent of sort + } else { + $target = ($earlyExit && $nearest === null && empty($this->sortKeys) && $this->limitCount !== null) + ? $this->offsetCount + $this->limitCount + : PHP_INT_MAX; + } + // The usual case, one where() group: no need to go through the AND/OR logic per record + $single = ($needWhere && count($where) === 1 && empty($orWhere)) ? $where[0] : null; + + while (true) { + $boundary = null; + $spatial = $this->spatialCandidates($nearestRadius, $boundary); + if ($spatial !== null) { + $records = array_values($spatial); + $whereApplied = false; + } elseif ($this->keyOrderLookup()) { + $records = $this->db->find($this->dbname, $this->whereFilters[0]); + $records = $records === false ? [] : $records; + $whereApplied = true; + } else { + // Indexes narrow the segments and keys down; the records are read in batches and the scan + // stops once enough match. A sort with a small limit keeps only the best ones (top-k). + return $this->scanFiltered($single, $where, $orWhere, $needWhere, $needAdvanced, $target, $topK); + } + + $filtered = []; + foreach ($records as $record) { + if (!$whereApplied) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) continue; + } + if ($needAdvanced && !$this->matchesAdvancedFilters($record)) continue; + $filtered[] = $record; + if (count($filtered) >= $target) break; + } + + if ($boundary !== null) { + // Candidates closer than the radius are exactly the full list's (at the radius itself + // rounding may leave some out). Enough matches among them: done. + $certain = []; + foreach ($filtered as $record) { + if (($record['_distance'] ?? INF) < $boundary) { + $certain[] = $record; + } + } + if (count($certain) >= $nearest) { + $filtered = $certain; + break; + } + if (count($records) >= $total) { + break; // every record lies within the radius: the list is complete + } + // Matches grow with the area (r²): aim for k from the matches so far, at least doubling + $factor = count($certain) > 0 ? min(16.0, max(2.0, sqrt($nearest / count($certain)) * 1.25)) : 4.0; + // Half the earth's circumference covers everything: then take the complete list + $nearestRadius = $nearestRadius * $factor >= 20040000.0 ? null : $nearestRadius * $factor; + continue; + } + break; + } + + if ($nearest !== null) { + // Stable on every PHP version: equal distances keep the candidate order + $positions = array_keys($filtered); + $distances = array_map(function ($r) { return $r['_distance'] ?? INF; }, $filtered); + array_multisort($distances, SORT_ASC, SORT_NUMERIC, $positions, SORT_ASC, SORT_NUMERIC); + $sorted = []; + foreach (array_slice($positions, 0, $nearest) as $position) { + $sorted[] = $filtered[$position]; + } + $filtered = $sorted; + } + return $filtered; + } + + /** + * filteredRecordsLocked() without spatial filters: the plan's candidates, every filter checked + * @param int $target Stop after this many matches + * @param int|null $topK Keep only the best $topK by the sort keys (records arrive in key order) + */ + private function scanFiltered(?array $single, array $where, array $orWhere, bool $needWhere, bool $needAdvanced, int $target, ?int $topK): array { + $filtered = []; + $prune = $topK === null ? PHP_INT_MAX : max(4 * $topK, 1024); + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + $this->lastScan = $this->callPrivateMethod('scanRecords', $dbname, $this->buildPlan(), + function (array $records) use (&$filtered, $single, $where, $orWhere, $needWhere, $needAdvanced, $target, $topK, $prune) { + foreach ($records as $record) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) { + continue; + } + if ($needAdvanced && !$this->matchesAdvancedFilters($record)) continue; + $filtered[] = $record; + if ($topK !== null) { + if (count($filtered) > $prune) { + $filtered = array_slice($this->applySort($filtered), 0, $topK); + } + } elseif (count($filtered) >= $target) { + return false; + } + } + return true; + }, $target === PHP_INT_MAX); + return $filtered; + } + + /** + * Add _distance for withDistance() + */ + private function applyDistanceCalculation(array $records): array { + if ($this->distanceConfig === null) { + return $records; + } + $field = $this->distanceConfig['field']; + foreach ($records as &$record) { + $geometry = $record[$field] ?? null; + if (!is_array($geometry) || !$this->db->validateGeoJSON($geometry)['valid']) { + continue; + } + $distance = $this->callPrivateMethod('distanceToGeometry', + (float)$this->distanceConfig['lon'], (float)$this->distanceConfig['lat'], $geometry); + if (is_finite($distance)) { // a geometry without any position has no distance + $record['_distance'] = round($distance, 4); + } + } + unset($record); + return $records; + } + + private function applyJoins(array $records): array { + foreach ($this->joinConfigs as $join) { + if ($join['foreignKey'] === 'key') { + // Joining on record keys: read only the keys asked for (T2-1) + $keys = []; + foreach ($records as $record) { + $lk = self::joinKey($record[$join['localKey']] ?? null); + if (is_int($lk) && $lk >= 0) { + $keys[$lk] = $lk; + } + } + $foreignData = empty($keys) ? [] : $this->db->find($join['foreignDb'], ['key' => array_values($keys)]); + } else { + // Only the foreign records whose value can join: the local raw values (a whole number also as + // its float and canonical string, which join it too), through the foreign field index when + // there is one. The join identity (joinKey) is only compared, never used as a filter value. + $values = []; + foreach ($records as $record) { + $value = $record[$join['localKey']] ?? null; + $lk = self::joinKey($value); + if ($lk === null) { + continue; + } + if (is_int($lk)) { + $values['i' . $lk] = $lk; + $values['d' . $lk] = (float)$lk; + $values['s' . $lk] = (string)$lk; + } else { + $values[$lk] = $value; + } + } + $foreignData = empty($values) ? [] : $this->db->query($join['foreignDb'])->whereIn($join['foreignKey'], array_values($values))->get(); + } + if ($foreignData === false) { + $foreignData = []; + } + $foreignIndexed = []; + foreach ($foreignData as $fRecord) { + $fk = self::joinKey($fRecord[$join['foreignKey']] ?? null); + if ($fk !== null) { + $foreignIndexed[$fk] = $fRecord; // the last record with a key wins + } + } + foreach ($records as &$record) { + $lk = self::joinKey($record[$join['localKey']] ?? null); + $record[$join['alias']] = $lk !== null ? ($foreignIndexed[$lk] ?? null) : null; + } + unset($record); + } + return $records; + } + + /** + * Join identity of a value (only to compare, never to query): whole numbers in the int64 range, + * 5.0 and the canonical string "5" are the same as 5 (record keys join with user_id values); + * other floats by their bits (a float used as an array key was truncated: 1.5 joined 1); other + * strings as themselves ("007" is not 7, "1.5" is not 1.5); booleans only booleans (true joined 1 + * and "1"); null and arrays never join. + * @return int|string|null + */ + private static function joinKey($v) { + if (is_int($v)) { + return $v; + } + if (is_float($v)) { + if (is_finite($v) && floor($v) === $v && $v >= -9.2233720368547758E18 && $v < 9.2233720368547758E18) { + return (int)$v; + } + return 'd' . bin2hex(pack('E', $v + 0.0)); + } + if (is_string($v)) { + if (preg_match('/\A(0|-?[1-9][0-9]*)\z/', $v) === 1 && (string)(int)$v === $v) { + return (int)$v; + } + return 's' . $v; + } + if (is_bool($v)) { + return $v ? 'b1' : 'b0'; + } + return null; + } + + private function applyGroupBy(array $records): array { + $groups = []; + // Exact identity (as distinct()): json_encode() followed serialize_precision and split -0.0 from 0.0 + $valueKey = $this->callPrivateMethod('valueKeyFunction'); + foreach ($records as $record) { + $value = $record[$this->groupByField] ?? null; + $groupKey = $valueKey(is_float($value) ? $value + 0.0 : $value); + if (!isset($groups[$groupKey])) { + $groups[$groupKey] = ['_group' => $value, '_items' => [], '_count' => 0]; + } + $groups[$groupKey]['_items'][] = $record; + $groups[$groupKey]['_count']++; + } + + foreach ($this->havingFilters as $having) { + if (!in_array($having['operator'], ['>', '<', '>=', '<=', '=', '!='], true)) { + continue; // unknown operator: the condition is ignored (for every group) + } + $groups = array_filter($groups, function ($group) use ($having) { + if ($having['aggregate'] === 'count') { + $aggValue = $group['_count']; + } elseif (!preg_match('/^(sum|avg|min|max):.+$/', $having['aggregate'])) { + return true; // unknown aggregate: ignored + } else { + [$aggType, $aggField] = explode(':', $having['aggregate'], 2); + $values = []; + foreach ($group['_items'] as $item) { + $number = self::numberOf($item[$aggField] ?? null); + if ($number !== null) { + $values[] = $number; + } + } + switch ($aggType) { + case 'sum': $aggValue = array_sum($values); break; + case 'avg': $aggValue = count($values) > 0 ? array_sum($values) / count($values) : 0; break; + case 'min': $aggValue = count($values) > 0 ? min($values) : null; break; + default: $aggValue = count($values) > 0 ? max($values) : null; break; + } + } + if ($aggValue === null) { + return false; + } + switch ($having['operator']) { + case '>': return $aggValue > $having['value']; + case '<': return $aggValue < $having['value']; + case '>=': return $aggValue >= $having['value']; + case '<=': return $aggValue <= $having['value']; + case '=': return $aggValue == $having['value']; + case '!=': return $aggValue != $having['value']; + default: return true; // unknown operator: ignored + } + }); + } + return array_values($groups); + } + + private function applySort(array $records): array { + if (empty($this->sortKeys) || count($records) < 2) { + return $records; + } + // Every sort field becomes two columns, kind and value (noneDB::sortKey), sorted by + // array_multisort with the same comparison as <=>. The original position is the last column, + // so records that compare equal keep their order (a stable sort on every PHP version). + $records = array_values($records); + $sortKey = $this->callPrivateMethod('sortKeyFunction'); + $columns = []; + foreach ($this->sortKeys as $sk) { + $kinds = []; + $values = []; + foreach ($records as $record) { + [$kinds[], $values[]] = $sortKey(is_array($record) && array_key_exists($sk['field'], $record) ? $record[$sk['field']] : null); + } + $order = $sk['order'] === 'desc' ? SORT_DESC : SORT_ASC; + array_push($columns, $kinds, $order, SORT_REGULAR, $values, $order, SORT_REGULAR); + } + $columns[] = array_keys($records); + array_multisort(...$columns); + $sorted = []; + foreach ($columns[count($columns) - 1] as $position) { + $sorted[] = $records[$position]; + } + return $sorted; + } + + private function applyProjection(array $records): array { + if (empty($this->selectFields) && empty($this->exceptFields)) { + return $records; + } + if ($this->groupByField !== null) { + // Group rows: _group, _count and _items in select()/except() apply to the group rows; other + // fields apply to the items of each group (C-13: select(['name']) used to empty every group) + $groupFields = ['_group', '_count', '_items']; + $select = $this->selectFields; + $except = $this->exceptFields; + $this->selectFields = array_values(array_diff($select, $groupFields)); + $this->exceptFields = array_values(array_diff($except, $groupFields)); + $groupBy = $this->groupByField; + $this->groupByField = null; + try { + foreach ($records as &$group) { + if (!empty($this->selectFields) || !empty($this->exceptFields)) { + $group['_items'] = $this->applyProjection($group['_items']); + } + $keep = array_values(array_intersect($select, $groupFields)); + if (!empty($keep)) { + if (!empty($this->selectFields)) { + $keep[] = '_items'; + } + $group = array_intersect_key($group, array_flip($keep)); + } + foreach (array_intersect($except, $groupFields) as $field) { + unset($group[$field]); + } + } + unset($group); + } finally { + $this->groupByField = $groupBy; + $this->selectFields = $select; + $this->exceptFields = $except; + } + return $records; + } + return array_map(function ($record) { + if (!empty($this->selectFields)) { + $filtered = []; + foreach ($this->selectFields as $field) { + if (array_key_exists($field, $record)) { + $filtered[$field] = $record[$field]; + } + } + if (isset($record['key']) && !array_key_exists('key', $filtered)) { + $filtered['key'] = $record['key']; + } + if ($this->distanceConfig !== null && isset($record['_distance']) && !array_key_exists('_distance', $filtered)) { + $filtered['_distance'] = $record['_distance']; + } + return $filtered; + } + foreach ($this->exceptFields as $field) { + unset($record[$field]); + } + return $record; + }, $records); + } + + /** + * How the query reads its records (v3.3.0): per segment whether it is skipped (a global field index or + * the keys rule it out), read through an index (candidates) or scanned; how it sorts (top-k, full) and + * whether it stops early. Every record read is checked against every filter ("residual": always). + * Nothing is built or written; an index a query would build first counts as a scan. With $analyze the + * query runs too and "actual" reports what it read and returned. + * @return array + */ + public function explain(bool $analyze = false): array { + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + $plan = $this->buildPlan(); + $info = $this->callPrivateMethod('explainPlan', $dbname, $plan); + $segments = $info['segments']; + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $topK = $this->topKSize($lateJoins); + if (!empty($this->spatialFilters)) { + $access = 'spatial'; + } elseif ($segments['total'] === 0 || $segments['skipped'] === $segments['total']) { + $access = 'none'; + } elseif ($segments['scanned'] === 0) { + $access = 'index'; + } elseif ($segments['indexed'] === 0) { + $access = 'scan'; + } else { + $access = 'mixed'; + } + $joins = []; + foreach ($this->joinConfigs as $join) { + $joins[] = ['db' => $join['foreignDb'], 'on' => $join['localKey'] . ' = ' . $join['foreignKey'], + 'strategy' => $join['foreignKey'] === 'key' ? 'key-lookup' : 'in-lookup', 'when' => $lateJoins ? 'after-slice' : 'before-sort']; + } + $result = [ + 'explain' => 1, + 'db' => $this->dbname, + 'sharded' => $info['sharded'], + 'access' => $access, + 'segments' => $segments, + 'plan' => $info['plan'], + 'planTruncated' => $info['planTruncated'] ?? false, + 'estimatedRead' => $info['estimatedRead'], + 'filter' => $this->keyOrderLookup() ? ['keys-in-filter-order'] : $plan, + 'residual' => true, + 'spatial' => empty($this->spatialFilters) ? null : array_column($this->spatialFilters, 'type'), + 'sort' => empty($this->sortKeys) ? 'none' : ($topK !== null ? 'top-k' : 'full'), + 'earlyExit' => empty($this->sortKeys) && $this->groupByField === null && $this->limitCount !== null + && (empty($this->joinConfigs) || $lateJoins) && empty($this->spatialFilters), + 'joins' => $joins, + 'groupBy' => $this->groupByField, + ]; + if ($analyze) { + $this->lastScan = null; + $start = microtime(true); + $rows = $this->get(); + $result['actual'] = [ + 'recordsRead' => $this->lastScan['recordsRead'] ?? null, + 'segmentsRead' => $this->lastScan === null ? null : $this->lastScan['segments'] - $this->lastScan['skipped'], + 'returned' => count($rows), + 'ms' => round((microtime(true) - $start) * 1000, 3), + ]; + } + return $result; + } + + /** + * Execute query and get all results + * @return array + */ + public function get(): array { + return $this->results(true); + } + + /** + * get() with or without select()/except() (aggregates read the fields themselves) + */ + private function results(bool $project): array { + if (!empty($this->joinConfigs)) { + // Every database of the query read in one consistent state (shared locks held together) + $databases = array_merge([$this->dbname], array_column($this->joinConfigs, 'foreignDb')); + return $this->callPrivateMethod('withSnapshot', $databases, function () use ($project) { + return $this->resultsLocked($project); + }); + } + return $this->resultsLocked($project); + } + + private function resultsLocked(bool $project): array { + // Joins only add fields: without groupBy() and without sorting by a join alias they run on the + // records left after sort, offset and limit (T2-2), which also allows stopping early + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $canEarlyExit = (empty($this->joinConfigs) || $lateJoins) && $this->groupByField === null && empty($this->sortKeys); + $results = $this->filteredRecords($canEarlyExit, null, $this->topKSize($lateJoins)); + $results = $this->applyDistanceCalculation($results); + if (!$lateJoins) { + $results = $this->applyJoins($results); + } + if ($this->groupByField !== null) { + $results = $this->applyGroupBy($results); // having() only applies to groups + } + $results = $this->applySort($results); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $results = array_slice($results, $this->offsetCount, $this->limitCount); + } + if ($lateJoins) { + $results = $this->applyJoins($results); + } + return $project ? $this->applyProjection($results) : $results; + } + + /** + * Size of the top-k a sorted query with a small limit needs (offset + limit), or null: no limit, a + * large one, grouping, joins before the sort, or a sort by withDistance()'s _distance + */ + private function topKSize(bool $lateJoins): ?int { + if (empty($this->sortKeys) || $this->limitCount === null || $this->groupByField !== null + || (!empty($this->joinConfigs) && !$lateJoins) || $this->distanceConfig !== null) { + return null; + } + $k = $this->offsetCount + $this->limitCount; + return $k <= 1000 ? max(1, $k) : null; + } + + private function sortsByJoinAlias(): bool { + $aliases = array_column($this->joinConfigs, 'alias'); + foreach ($this->sortKeys as $sk) { + if (in_array($sk['field'], $aliases, true)) { + return true; + } + } + return false; + } + + /** + * Keys of the records a write (update/delete/removeFields) applies to: + * filters, sort, offset and limit apply; select/except/join don't matter. + * @return int[] + */ + private function targetKeys(): array { + if ($this->groupByField !== null) { + throw new \LogicException('noneDB: update(), delete() and removeFields() cannot be combined with groupBy()'); + } + $records = $this->applySort($this->applyDistanceCalculation($this->filteredRecords(empty($this->sortKeys)))); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $records = array_slice($records, $this->offsetCount, $this->limitCount); + } + $keys = []; + foreach ($records as $record) { + if (isset($record['key']) && is_int($record['key'])) { + $keys[] = $record['key']; + } + } + return $keys; + } + + /** + * Run a write under the database's exclusive lock, so selecting and writing are one atomic step + */ + private function withWriteLock(callable $fn) { + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + return $this->callPrivateMethod('withLock', $dbname, true, $fn); + } + + /** + * Get first matching record + * @return array|null + */ + public function first(): ?array { + $originalLimit = $this->limitCount; + $this->limitCount = $originalLimit === null ? 1 : min(1, $originalLimit); // limit(0): nothing + try { + $results = $this->get(); + } finally { + $this->limitCount = $originalLimit; + } + return $results[0] ?? null; + } + + /** + * Get last matching record + * @return array|null + */ + public function last(): ?array { + $results = $this->get(); + return count($results) > 0 ? $results[count($results) - 1] : null; + } + + /** + * Count matching records (groups, with groupBy()) + * @return int + */ + public function count(): int { + if ($this->isUnfiltered()) { + return $this->db->count($this->dbname, 0); + } + if ($this->groupByField !== null) { + return count($this->get()); + } + if (empty($this->spatialFilters) && empty($this->joinConfigs) && $this->offsetCount === 0 && $this->limitCount === null + && !$this->keyOrderLookup()) { + return $this->countMatches(); // record by record, none kept in memory + } + $results = $this->filteredRecords(true); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $results = array_slice($results, $this->offsetCount, $this->limitCount); + } + return count($results); + } + + private function countMatches(): int { + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + return $this->callPrivateMethod('withReadLock', $dbname, function () use ($dbname) { + $needWhere = !empty($this->whereFilters) || !empty($this->orWhereFilters); + $where = $needWhere ? $this->compileGroups($this->whereFilters) : []; + $orWhere = $needWhere ? $this->compileGroups($this->orWhereFilters) : []; + $single = ($needWhere && count($where) === 1 && empty($orWhere)) ? $where[0] : null; + $needAdvanced = $this->hasAdvancedFilters(); + $count = 0; + $this->callPrivateMethod('scanRecords', $dbname, $this->buildPlan(), + function (array $records) use (&$count, $single, $where, $orWhere, $needWhere, $needAdvanced) { + foreach ($records as $record) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) { + continue; + } + if (!$needAdvanced || $this->matchesAdvancedFilters($record)) { + $count++; + } + } + return true; + }, true); + return $count; + }); + } + + /** + * True when count() can use the stored counters + * @return bool + */ + private function isUnfiltered(): bool { + return empty($this->whereFilters) + && empty($this->orWhereFilters) + && !$this->hasAdvancedFilters() + && empty($this->spatialFilters) + && $this->groupByField === null + && empty($this->havingFilters) + && $this->limitCount === null + && $this->offsetCount === 0; + } + + /** + * Check if any records match + * @return bool + */ + public function exists(): bool { + if ($this->limitCount === 0) { + return false; + } + if ($this->groupByField !== null || !empty($this->havingFilters)) { + return $this->first() !== null; // having() may drop every group: no early exit + } + // Sort, joins and projection don't change whether a record exists: stop at offset + 1 matches (T2-3) + return count($this->filteredRecords(true, $this->offsetCount + 1)) > $this->offsetCount; + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Numeric values of a field in the results (booleans and non-numeric values are ignored) + */ + private function numericValues(string $field): array { + $values = []; + foreach ($this->results(false) as $record) { + $number = self::numberOf($record[$field] ?? null); + if ($number !== null) { + $values[] = $number; + } + } + return $values; + } + + /** + * Sum of field values + * @param string $field + * @return float + */ + public function sum(string $field): float { + return (float)array_sum($this->numericValues($field)); + } + + /** + * Average of field values + * @param string $field + * @return float + */ + public function avg(string $field): float { + $values = $this->numericValues($field); + return count($values) > 0 ? array_sum($values) / count($values) : 0.0; + } + + /** + * Minimum value of field (same ordering as sort()) + * @param string $field + * @return mixed|null + */ + public function min(string $field) { + $min = null; + $found = false; + foreach ($this->results(false) as $record) { + if (!isset($record[$field])) continue; + if (!$found || $this->callPrivateMethod('compareSortValues', $record[$field], $min) < 0) { + $min = $record[$field]; + $found = true; + } + } + return $min; + } + + /** + * Maximum value of field (same ordering as sort()) + * @param string $field + * @return mixed|null + */ + public function max(string $field) { + $max = null; + $found = false; + foreach ($this->results(false) as $record) { + if (!isset($record[$field])) continue; + if (!$found || $this->callPrivateMethod('compareSortValues', $record[$field], $max) > 0) { + $max = $record[$field]; + $found = true; + } + } + return $max; + } + + /** + * Get unique values of field + * @param string $field + * @return array + */ + public function distinct(string $field): array { + return $this->callPrivateMethod('uniqueValues', $this->results(false), $field); + } + + // ========================================== + // WRITE METHODS + // ========================================== + + /** + * Update matching records (sort/offset/limit select which ones) + * @param array $set Fields to update + * @return array ["n" => count] + */ + public function update(array $set): array { + return $this->withWriteLock(function () use ($set) { + $keys = $this->targetKeys(); + if (empty($keys)) { + return ['n' => 0]; + } + return $this->db->update($this->dbname, [['key' => $keys], ['set' => $set]]); + }); + } + + /** + * Delete matching records (sort/offset/limit select which ones) + * @return array ["n" => count] + */ + public function delete(): array { + return $this->withWriteLock(function () { + $keys = $this->targetKeys(); + if (empty($keys)) { + return ['n' => 0]; + } + return $this->db->delete($this->dbname, ['key' => $keys]); + }); + } + + /** + * Remove fields from the stored matching records (permanent) + * @param array $fields Fields to remove ("key" can't be removed) + * @return array ["n" => count of updated records, "fields_removed" => array of removed fields] + */ + public function removeFields(array $fields): array { + if (empty($fields)) { + return ['n' => 0, 'fields_removed' => [], 'error' => 'No fields specified']; + } + $fieldsToRemove = array_values(array_filter($fields, function ($f) { return $f !== 'key'; })); + if (empty($fieldsToRemove)) { + return ['n' => 0, 'fields_removed' => [], 'error' => 'Cannot remove reserved field: key']; + } + + return $this->withWriteLock(function () use ($fieldsToRemove) { + $keys = $this->targetKeys(); + if (empty($keys)) { + return ['n' => 0, 'fields_removed' => []]; // nothing matched: nothing removed + } + // Fields are removed from the stored records, never from a projected/joined result + $stored = $this->db->find($this->dbname, ['key' => $keys]); + $replacements = []; + $removed = []; + foreach ($stored as $record) { + $changed = false; + foreach ($fieldsToRemove as $field) { + if (array_key_exists($field, $record)) { + unset($record[$field]); + $removed[$field] = true; + $changed = true; + } + } + if ($changed) { + $key = $record['key']; + unset($record['key']); + $replacements[$key] = $record; + } + } + $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + return [ + 'n' => $this->callPrivateMethod('replaceRecordsByKey', $dbname, $replacements), + 'fields_removed' => array_values(array_filter($fieldsToRemove, function ($f) use ($removed) { return isset($removed[$f]); })) + ]; + }); + } +} diff --git a/composer.json b/composer.json index 1dee6df..f613219 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,7 @@ } ], "require": { - "php": ">=7.4" + "php": ">=8.1" }, "require-dev": { "phpunit/phpunit": "^9.6", @@ -25,8 +25,13 @@ "noneDB\\Tests\\": "tests/" } }, + "bin": ["noneDB-migrate.php"], "scripts": { - "test": "phpunit", + "build": "@php tools/build.php", + "test": [ + "@php tools/build.php --check", + "phpunit" + ], "test-verbose": "phpunit --testdox" }, "config": { diff --git a/docs/API.md b/docs/API.md index a551d03..a0c9503 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,574 +1,495 @@ # noneDB API Reference -Complete reference for all noneDB methods and operations. +Every public method of noneDB 4.0.0. The query builder (`$db->query()`) is in [QUERY.md](QUERY.md), the spatial +queries in [SPATIAL.md](SPATIAL.md), the settings in [CONFIGURATION.md](CONFIGURATION.md). -**Version:** 3.2.0 +**Database names** may contain `A-Z a-z 0-9`, space, `'` and `-`. Anything else throws +`InvalidArgumentException`, a line feed at the end too; a name is never changed silently. `"0"` is a valid name. +`checkDB()` answers `false` for `''`, `null` and `false` without throwing. -**Database names** may contain `A-Z a-z 0-9`, space, `'` and `-`; every method throws -`InvalidArgumentException` for anything else (names are never changed silently). `"0"` is valid. +**Locking.** An operation locks its database for its whole duration: shared for reads, exclusive for writes. +A read of a database that does not exist takes no database lock (only the store's, shared) and creates nothing. Joins lock every database of the +query at once. Waiting longer than `lockTimeout` seconds throws `RuntimeException` with code +`noneDB::LOCK_FAILED`. -**Locking:** every method locks its database for its whole duration (shared for reads, exclusive for -writes). Waiting longer than `lockTimeout` seconds throws `RuntimeException`. +**A write that throws leaves no record of itself behind.** When `insert()`, `update()`, `delete()` or a query +builder write throws, the lines it appended are cut back, together with writes nested in it on the same +database (from an error handler, for example). The details: +- Records flushed from the write buffer before the call's own writes stay written: they were acknowledged. +- In a sharded database the keys it reserved stay used and the shards it created stay listed, empty: the next + insert gets a later key. +- Work a write triggers after it committed (compaction, the switch to shards, a buffer flush) is not part of + it. If that work fails, the write still succeeds; the failure is logged (`deferred-failed`) and the work runs + again later. +- Operations that replace or remove files (`dropDB()`, `compact()`, `convertToSharded()`, `restore()`), and + `backup()`, refuse to run inside a write of the same database from the write's first step, before its first append + too (a value's `jsonSerialize()` runs there): `LogicException`, no file touched. Call them after the write. +- The outcomes of a write are in [DURABILITY.md](DURABILITY.md) §2: committed, rolled back, or unknown + (`noneDB::OUTCOME_UNKNOWN`). ---- - -## Table of Contents - -1. [Database Operations](#database-operations) -2. [CRUD Operations](#crud-operations) -3. [Query Operations](#query-operations) -4. [Aggregation](#aggregation) -5. [Sharding](#sharding) -6. [Field Indexing](#field-indexing) -7. [Spatial Indexing](#spatial-indexing) -8. [Utility Methods](#utility-methods) - ---- - -## Database Operations +**Options are the documented ones.** A method that takes an options array (`createSpatialIndex()`, +`withinDistance()`, `nearest()`, `restore()`, `migrateKey()`, `registerKey()`, `repairStore()`) accepts exactly the +options its section names, of the types it names. Anything else is an `InvalidArgumentException` before the method +does anything, also from the static key maintenance methods: the message names the option, suggests the one it was +probably meant to be, and lists them. A misspelled `abort` can't let a rotation go on. -### createDB() - -Create a new database. +The examples below run in order against one store (CI runs them). ```php -$result = $db->createDB("users"); -// Returns: true (created) or false (already exists) +$db = new noneDB(); // reads .nonedb (CONFIGURATION.md) ``` -### checkDB() +--- -Check if database exists. Creates it if `autoCreateDB` is true. +## Contents + +1. [Databases](#databases) +2. [Records](#records) +3. [Aggregation and helpers](#aggregation-and-helpers) +4. [Sharding and compaction](#sharding-and-compaction) +5. [Field indexes](#field-indexes) +6. [Spatial indexes](#spatial-indexes) +7. [Write buffer](#write-buffer) +8. [Durability, integrity and metrics](#durability-integrity-and-metrics) +9. [Backup and restore](#backup-and-restore) +10. [Key maintenance](#key-maintenance) +11. [Static helpers](#static-helpers) +12. [Errors](#errors) -```php -$exists = $db->checkDB("users"); -// Returns: true or false (false for null / "") -``` +--- -### dropDB() +## Databases -Delete a database and all of its files (data, shards, indexes, buffers, metadata). +### createDB() / checkDB() ```php -$result = $db->dropDB("users"); -// ["success" => true, "files" => 7] or ["success" => false, "files" => 0, "error" => "Database does not exist"] +$db->createDB("users"); // true: created +$db->createDB("users"); // false: it exists +$db->checkDB("users"); // true; creates a missing database when autoCreateDB is on, else false ``` -### migrateKey() (static) - -Re-key existing databases (all files of every database created with `$oldKey` are renamed). Run it -while nothing else uses the databases, then put the new key in your config. +### getDBs() ```php -$result = noneDB::migrateKey('/var/data/db', $oldKey, $newKey); -// ["success" => true, "databases" => ["users", "orders"], "files" => 14, "error" => null] +$db->getDBs(false); // ["users"]: names only, sorted +$db->getDBs(true); // [["name" => "users", "createdTime" => 1793291271, "size" => "185 B"], ...] +$db->getDBs("users"); // ["name" => "users", "createdTime" => 1793291271, "size" => "185 B"]; false when missing ``` -### getDBs() +Only the databases of this instance's `secretKey` are listed. `size` is the size of the data files, with a +comma as decimal separator (`"2,5 KB"`). A database whose directory holds shard files its shard list doesn't +name lists them as `orphanFiles` (set aside as `*.orphan.{YmdHis}`, never deleted automatically). -List databases. +### dropDB() ```php -// Names only -$names = $db->getDBs(false); -// ["users", "posts", "comments"] - -// With metadata -$dbs = $db->getDBs(true); -// [ -// ["name" => "users", "createdTime" => 1703123456, "size" => "2,5 KB"], -// ... -// ] - -// Single database info (false if it doesn't exist; never creates it) -$info = $db->getDBs("users"); -// ["name" => "users", "createdTime" => 1703123456, "size" => "2,5 KB"] +$db->createDB("old"); +$db->dropDB("old"); // ["success" => true, "files" => 5] +$db->dropDB("old"); // ["success" => false, "files" => 0, "error" => "Database does not exist"] ``` -Only databases of the current `secretKey` are listed; sharded databases appear once, with the total -size of their shards. Sizes use a comma as decimal separator (`"2,5 KB"`). +Removes every file of the database except its lock file (STORAGE_FORMAT.md §8.2). --- -## CRUD Operations +## Records ### insert() -Insert one or more records. - ```php -// Single record -$result = $db->insert("users", ["name" => "John", "email" => "john@test.com"]); -// ["n" => 1] - -// Multiple records -$result = $db->insert("users", [ - ["name" => "John"], - ["name" => "Jane"] -]); -// ["n" => 2] - -// With nested data -$result = $db->insert("users", [ - "name" => "John", - "address" => ["city" => "Istanbul", "country" => "Turkey"] -]); +$db->insert("users", ["name" => "John", "email" => "john@example.com"]); // ["n" => 1] +$db->insert("users", [["name" => "Jane", "age" => 31], ["name" => "Ali", "age" => 27]]); // ["n" => 2] +$db->insert("users", ["name" => "Ayşe", "address" => ["city" => "Istanbul"]]); ``` -**Errors:** +Invalid input returns an `error` and writes nothing: ```php -// Reserved field -$db->insert("users", ["key" => "value"]); -// ["n" => 0, "error" => "You cannot set key name to key"] - -// Invalid data -$db->insert("users", "not an array"); -// ["n" => 0, "error" => "insert data must be array"] - -// Empty data, or a mix of a list and named fields -$db->insert("users", []); // ["n" => 0, "error" => "insert data must not be empty"] -$db->insert("users", [0 => ["a" => 1], "name" => "x"]); // ["n" => 0, "error" => "insert data must be either one record or a list of records"] - -// Values JSON can't represent (invalid UTF-8, NAN, INF): nothing is written -$db->insert("users", ["v" => NAN]); // ["n" => 0, "error" => "insert data cannot be encoded as JSON ..."] +$db->insert("users", ["key" => 5]); // ["n" => 0, "error" => "You cannot set key name to key"] +$db->insert("users", "not an array"); // ["n" => 0, "error" => "insert data must be array"] +$db->insert("users", []); // ["n" => 0, "error" => "insert data must not be empty"] +$db->insert("users", [0 => ["a" => 1], "name" => "x"]); // "insert data must be either one record or a list of records" +$db->insert("users", ["v" => NAN]); // "insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)" ``` -Keys are assigned in order and are **never reused or changed** (not by deletes, not by `compact()`). -Floats keep their type (`2.0` is read back as `2.0`). +Keys are assigned in order and **never reused or changed**: not by a delete, not by `compact()`, not by a +`restore()` of an older backup. Floats keep their type (`2.0` reads back as `2.0`). ### find() -Find records matching filter. - ```php -// All records -$all = $db->find("users", 0); -$all = $db->find("users", []); - -// By field -$results = $db->find("users", ["name" => "John"]); - -// By multiple fields (AND) -$results = $db->find("users", ["name" => "John", "active" => true]); - -// By key -$results = $db->find("users", ["key" => 0]); -$results = $db->find("users", ["key" => [0, 1, 2]]); +$db->find("users", 0); // every record +$db->find("users", ["name" => "John"]); // by field +$db->find("users", ["name" => "Jane", "age" => 31]); // several fields: all must match +$db->find("users", ["key" => 0]); // by key +$db->find("users", ["key" => [0, 1, 2]]); // by keys, in this order ``` -**Matching rules** (shared by `find()`, `update()` and `delete()`): - -- Every filter field must exist in the record and be **identical** (`===`): `"25"` does not match `25`, - `null` matches only a field that exists with the value `null`, arrays must be equal as a whole. -- `key` accepts an integer, a digit string (`"5"`) or a list of them; anything else matches nothing. - It is combined with the other filters (AND). -- Any integer (normally `0`), `null` and `[]` select every record. Other non-array filters return `false`. +**Matching** (shared by `find()`, `update()` and `delete()`): +- every filter field must exist in the record and be **identical** (`===`): `"25"` does not match `25`; + `null` matches only a field that exists with the value `null`; arrays must be equal as a whole; +- `key` takes an integer, a digit string or a list of them, and is combined with the other fields (AND); +- `[]` selects every record. `find()` (and `count()`, `sum()`, `first()`, ...) also take an integer (normally + `0`) or `null` for every record. Any other non-array filter matches nothing: `find()` returns `false`, + `count()`/`sum()`/`avg()` 0, `min()`/`max()`/`first()`/`last()` `null`, `exists()`/`between()` `false`. -**Returns:** Array of records with `key` field added (in key order, or in the order of a `key` list). -Returns `[]` for a database that doesn't exist (`false` if `autoCreateDB` is off); reading never -creates a database. +Records come back with their `key`, in key order (or in the order of a `key` list). A database that does not +exist gives `[]` (`false` with `autoCreateDB` off); a read never creates one. ### update() -Update matching records. - ```php -// By field -$result = $db->update("users", [ - ["name" => "John"], // Filter - ["set" => ["active" => true]] // Updates -]); -// ["n" => 1] - -// By key -$result = $db->update("users", [ - ["key" => [0, 1, 2]], - ["set" => ["status" => "inactive"]] -]); - -// All records -$result = $db->update("users", [ - [], - ["set" => ["updated_at" => time()]] -]); +$db->update("users", [["name" => "John"], ["set" => ["active" => true]]]); // ["n" => 1] +$db->update("users", [["key" => [0, 1]], ["set" => ["status" => "checked"]]]); +$db->update("users", [[], ["set" => ["updated_at" => time()]]]); // every record ``` -The first element must be a filter array and `set` a non-`key` array of fields; otherwise -`["n" => 0, "error" => "Please check your update parameters"]`. An empty `set` changes nothing and -returns the number of matching records. +The first element is a filter, `set` an array of fields other than `key`; anything else returns +`["n" => 0, "error" => "Please check your update parameters"]`. A `set` that can't be encoded as JSON returns the +error `update data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)`. An empty `set` changes +nothing and returns the number of matching records. ### delete() -Delete matching records. - ```php -// By field -$result = $db->delete("users", ["name" => "John"]); -// ["n" => 1] - -// By key -$result = $db->delete("users", ["key" => [0, 2]]); - -// All records (an empty filter matches everything) -$result = $db->delete("users", []); +$db->delete("users", ["name" => "Ali"]); // ["n" => 1] +$db->delete("users", ["key" => [99]]); // ["n" => 0] +$db->delete("users", 0); // ["n" => 0, "error" => "Please check your delete parameters"] ``` ---- +`delete("users", [])` deletes every record: an empty filter matches everything. -## Query Operations +--- -### Query Builder +## Aggregation and helpers ```php -$query = $db->query("users"); +$db->insert("orders", [["amount" => 120, "status" => "paid", "city" => "Izmir"], + ["amount" => 80, "status" => "open", "city" => "Ankara"], + ["amount" => 45, "status" => "paid", "city" => "Izmir"]]); +$db->count("orders", 0); // 3 +$db->count("orders", ["status" => "paid"]); // 2 +$db->sum("orders", "amount"); // 245 +$db->sum("orders", "amount", ["status" => "paid"]); // 165 +$db->avg("orders", "amount"); // 81.666... +$db->min("orders", "amount"); // 45 +$db->max("orders", "amount"); // 120 +$db->distinct("orders", "city"); // ["Izmir", "Ankara"] +$db->first("orders"); // the record with the lowest key +$db->last("orders", ["status" => "paid"]); // the paid record with the highest key +$db->exists("orders", ["city" => "Ankara"]); // true +$db->between("orders", "amount", 50, 150); // records with 50 <= amount <= 150 +$db->between("orders", "amount", 50, 150, ["status" => "paid"]); +$db->like("users", "email", "example.com$"); // ends with ``` -### Filter Methods - -| Method | Description | -|--------|-------------| -| `where($filters)` | AND filter with operator support; several calls are all applied (AND) | -| `orWhere($filters)` | OR group: matches if all `where()` groups match **or** any `orWhere()` group does | -| `whereIn($field, $values)` | Value in array | -| `whereNotIn($field, $values)` | Value not in array | -| `whereNot($filters)` | Excludes records where **any** of the given fields equals its value (`===`); several calls all apply | -| `like($field, $pattern)` | Pattern match | -| `notLike($field, $pattern)` | Pattern not match | -| `between($field, $min, $max)` | Range (inclusive) | -| `notBetween($field, $min, $max)` | Not inside the range: also returns records without the field or with a value that can't be compared (e.g. `null`, `true`, text vs numbers) | -| `search($term, $fields)` | Full-text search | +- `min()`/`max()` use the total order of `sort()`: missing/`null` < booleans < numbers < numeric strings (by + number) < other strings (byte order) < arrays. `max([50, "10"])` is `"10"`. +- `distinct()` keeps the order of first appearance and tells types apart (`1`, `1.0` and `"1"` are three + values); `null` and missing fields are left out. +- `like()`: only a leading `^` and a trailing `$` are special; case-insensitive with Unicode case folding + (QUERY.md). Strings and numbers are matched; booleans, `null` and arrays never are. -### Comparison Operators +Helpers on arrays of records: ```php -// Several operators on one field go into one array (a PHP array can't repeat a key) -$db->query("users")->where([ - 'age' => ['$gte' => 18, '$lt' => 65], // Range - 'role' => ['$ne' => 'guest'], // Not equal - 'status' => ['$in' => ['active', 'pending']], // In array (strict) - 'email' => ['$exists' => true], // Field exists - 'name' => ['$like' => '^John'], // Pattern (only ^ and $ are special) - 'tags' => ['$contains' => 'featured'] // Array element / substring -])->get(); - -// ...or one where() per condition: all of them apply -$db->query("users") - ->where(['age' => ['$gt' => 18]]) - ->where(['age' => ['$lte' => 65]]) - ->where(['email' => ['$regex' => '@gmail\\.com$']]) // regular expression, case-insensitive - ->get(); +$records = $db->find("orders", 0); +$db->sort($records, "amount", "desc"); // stable, the total order above; false for a non-array +$db->limit($records, 2); // the first 2; false for invalid arguments ``` -- `$gt`, `$gte`, `$lt`, `$lte` (and `between()`) compare numbers and numeric strings numerically and - other strings lexically. `null`, booleans, arrays, and a number against a non-numeric string never - match. -- `$eq`, `$ne`, `$in`, `$nin` use strict comparison (`===`). -- An unknown operator (e.g. a typo like `$gtee`), `$in`/`$nin` without an array or an invalid `$regex` - throws `InvalidArgumentException`, also when there are no records to check. - -### Modifiers - -| Method | Description | -|--------|-------------| -| `select($fields)` | Include only specified fields | -| `except($fields)` | Exclude specified fields | -| `sort($field, $order)` | Sort results; further calls add tie-breaking keys | -| `limit($count)` | Limit results | -| `offset($count)` | Skip records | -| `join($db, $localKey, $foreignKey, $alias = null)` | Attach the matching foreign record under `$alias` (default: the foreign database name; it replaces a field of that name in the result only). With duplicate foreign keys the last record wins | -| `groupBy($field)` | Group results (values keep their type: `1`, `"1"` and `1.5` are different groups) | -| `having($aggregate, $op, $value)` | Filter groups: `count`, `sum:f`, `avg:f`, `min:f`, `max:f`. Ignored without `groupBy()` or for unknown aggregates/operators | - -### Terminal Methods - -| Method | Returns | -|--------|---------| -| `get()` | All matching records | -| `first()` | First record or null | -| `last()` | Last record or null | -| `count()` | Number of matches | -| `exists()` | Boolean | -| `sum($field)` | Sum of field | -| `avg($field)` | Average of field | -| `min($field)` | Minimum value | -| `max($field)` | Maximum value | -| `distinct($field)` | Unique values | -| `update($set)` | Update matches (sort/offset/limit choose which ones) | -| `delete()` | Delete matches (sort/offset/limit choose which ones) | -| `removeFields($fields)` | Remove fields from the **stored** matching records | - --- -## Aggregation +## Sharding and compaction -### count() +A database switches to the sharded layout once it has assigned `shardSize` keys (`shardingEnabled`, on by default): +shard N holds the keys `[N × shardSize, (N + 1) × shardSize)`. The shard size is fixed when the database is +created. ```php -$total = $db->count("users", 0); -$active = $db->count("users", ["active" => true]); -``` +$db->getShardInfo("orders"); +// ["sharded" => false, "shards" => 0, "totalRecords" => 3, "shardSize" => 10000, "nextKey" => 3] +// sharded: ["sharded" => true, "shards" => 2, "totalRecords" => 5, "deletedCount" => 0, "shardSize" => 4, "nextKey" => 5] +// false when the database doesn't exist -### sum() / avg() +$db->compact("orders"); +// ["success" => true, "freedSlots" => 0, "totalRecords" => 3, "sharded" => false] +// sharded databases add "newShardCount"; a missing one: ["success" => false, "freedSlots" => 0, "status" => "database_not_found"] -```php -$totalSalary = $db->sum("users", "salary"); -$avgAge = $db->avg("users", "age"); -$filteredSum = $db->sum("orders", "amount", ["status" => "paid"]); +$db->convertToSharded("orders"); // ["success" => true, "status" => "converted"] (or "already_sharded") +$db->isShardingEnabled(); // true +$db->getShardSize(); // 10000: the setting for databases created from now on ``` -### min() / max() - -```php -$minPrice = $db->min("products", "price"); -$maxScore = $db->max("scores", "points"); -``` - -### distinct() - -```php -$cities = $db->distinct("users", "city"); -// ["Istanbul", "Ankara", "Izmir"] -``` +- `nextKey` is the key the next insert gets. +- `compact()` rewrites data files without deleted and superseded lines. Keys don't change; shards that became + empty are removed, except the one of the newest key. Usually not needed: compaction runs by itself + (`autoCompactThreshold`). +- `convertToSharded()` switches a database to the sharded layout now (a generation switch: STORAGE_FORMAT.md + §5.2). A database that doesn't exist: `["success" => false, "status" => "database_not_found"]`. --- -## Sharding - -### getShardInfo() +## Field indexes ```php -$info = $db->getShardInfo("users"); -// [ -// "sharded" => true, -// "shards" => 5, -// "totalRecords" => 45000, -// "deletedCount" => 150, -// "shardSize" => 10000, -// "nextKey" => 45150 -// ] +$db->createFieldIndex("users", "email"); // ["success" => true, "indexed" => 1, "error" => null] (+ "shards" => N) +$db->hasFieldIndex("users", "email"); // true +$db->getFieldIndexes("users"); // ["fields" => ["email"], "sharded" => false] +$db->getKeysFromFieldIndex("users", "email", "john@example.com"); // [0]; null without an index +$db->rebuildFieldIndex("users", "email"); // as createFieldIndex() +$db->dropFieldIndex("users", "email"); // ["success" => true, "error" => null]; without one: "Index does not exist" ``` -### compact() - -Rewrites data files without deleted and superseded records. Keys never change; shards that became -empty are removed. Usually not needed: compaction also runs automatically. +- `indexed` counts the records whose value can be indexed (a scalar or `null`). An index keeps values by type, + as `find()` compares them with `===`: `1`, `1.0` and `"1"` are three values. +- A field name is any non-empty string except `key`; names that differ in letter case get their own indexes. +- An index always gives the results of a full scan. A write appends to the data file only; an index replays + the lines after the bytes it covers when it is used. +- `getKeysFromFieldIndex()` takes a shard number as a fourth argument (sharded databases). -```php -$result = $db->compact("users"); -// ["success" => true, "freedSlots" => 150, "totalRecords" => 44850, "sharded" => true, "newShardCount" => 5] -``` +--- -### migrate() +## Spatial indexes ```php -$result = $db->migrate("users"); -// ["success" => true, "status" => "migrated"] +$db->insert("places", [["name" => "Galata", "loc" => ["type" => "Point", "coordinates" => [28.9741, 41.0256]]], + ["name" => "Kadikoy", "loc" => ["type" => "Point", "coordinates" => [29.0275, 40.9906]]]]); +$db->createSpatialIndex("places", "loc"); // ["success" => true, "indexed" => 2, "error" => null] +$db->createSpatialIndex("places", "loc"); // ["success" => false, "indexed" => 0, "error" => "Spatial index already exists for field 'loc'"] +$db->hasSpatialIndex("places", "loc"); // true (a shard number as third argument: that shard) +$db->getSpatialIndexes("places"); // ["loc"] +$db->rebuildSpatialIndex("places", "loc"); // dropped and made again from the data +$db->withinDistance("places", "loc", 28.97, 41.02, 2000); // within 2 km, nearest first +$db->withinBBox("places", "loc", 28.9, 40.9, 29.1, 41.1); +$db->nearest("places", "loc", 28.97, 41.02, 1); // [Galata with "_distance" => 711.37] (meters) +$db->withinPolygon("places", "loc", ["type" => "Polygon", "coordinates" => [[[28.9, 41.0], [29.0, 41.0], [29.0, 41.1], [28.9, 41.1], [28.9, 41.0]]]]); +$db->validateGeoJSON(["type" => "Point", "coordinates" => [200, 0]]); // ["valid" => false, "error" => "...", "type" => null] +$db->haversineDistance(28.97, 41.00, 28.98, 41.00); // 839.2 (meters) +$db->dropSpatialIndex("places", "loc"); // ["success" => true, "error" => null], also without an index ``` -### Configuration - -```php -$db->isShardingEnabled(); // true/false -$db->getShardSize(); // 10000 -``` +Records whose field is not valid GeoJSON are not indexed (`indexed` counts the others). +`createSpatialIndex($db, $field, ["nodeSize" => 16])` sets the R-tree node size (an integer of at least 4, default +32). `withinDistance()` takes `includeDistance` (bool, default false), `nearest()` takes `maxDistance` (meters, a +number or `null`) and `includeDistance` (bool, default true). +Options, rules and the query builder integration: [SPATIAL.md](SPATIAL.md). --- -## Field Indexing - -### createFieldIndex() - -```php -$result = $db->createFieldIndex("users", "email"); -// ["success" => true, "indexed" => 150, "values" => 120, "error" => null] (+ "shards" => N when sharded) -// Calling it again for an indexed field rebuilds the index -``` - -### hasFieldIndex() - -```php -$exists = $db->hasFieldIndex("users", "email"); -// true/false -``` +## Write buffer -### getFieldIndexes() / rebuildFieldIndex() / getKeysFromFieldIndex() +Off by default. With the buffer on, inserts go to a buffer file first and reach the data files in batches; +buffered records are flushed before the next read or write of the database, so they are always visible. ```php -$db->getFieldIndexes("users"); // ["fields" => ["email", "e-mail"], "sharded" => false] -$db->rebuildFieldIndex("users", "email"); // same result as createFieldIndex() -$db->getKeysFromFieldIndex("users", "status", "active"); // [0, 4, 7] or null without an index +$db->enableBuffering(true); +$db->isBufferingEnabled(); // true +$db->setBufferSizeLimit(1048576); // bytes (at least 1 KB) +$db->setBufferCountLimit(10000); // records (at least 10) +$db->setBufferFlushInterval(30); // seconds, 0: no flush by time +$db->insert("events", ["type" => "login"]); +$db->getBufferInfo("events"); // ["enabled" => true, "sizeLimit" => ..., "countLimit" => ..., "flushInterval" => 30, + // "buffers" => ["main" => ["exists" => true, "size" => 17, "records" => 1]]] +$db->flush("events"); // ["success" => true, "flushed" => 1, "error" => null] +$db->flushAllBuffers(); // ["databases" => 0, "flushed" => 0] +$db->enableBuffering(false); ``` -Field names may contain any characters. Indexes are maintained by every write and give exactly the -results of a full scan. - -### dropFieldIndex() - -```php -$result = $db->dropFieldIndex("users", "email"); -// ["success" => true, ...] or ["success" => false, "error" => "Index does not exist"] -``` +Buffered records get their keys when they are flushed. A buffered insert is durable once it returns, like a +direct one (DURABILITY.md). --- -## Spatial Indexing +## Durability, integrity and metrics -### createSpatialIndex() +### getDurability() ```php -$result = $db->createSpatialIndex("locations", "coords"); -// ["success" => true, "indexed" => 150, "error" => null] -// An existing index: ["success" => false, "indexed" => 0, "error" => "Spatial index already exists for field 'coords'"] - -// Records with invalid/missing GeoJSON are skipped (not indexed) -// Always check 'indexed' count to verify +$db->getDurability(); +// ["mode" => "performance", +// "platform" => ["os" => "Linux", "durableSupported" => true, "notes" => []], +// "lastOperation" => ["op" => "flush", "db" => "events", "outcome" => "committed", "required" => [], "completed" => [], "reason" => null], +// "lastSyncError" => null, +// "syncs" => ["file" => 7, "dir" => 7, "failed" => 0]] ``` -### hasSpatialIndex() +The contract, the outcomes and every field: [DURABILITY.md](DURABILITY.md) §6. -```php -$exists = $db->hasSpatialIndex("locations", "coords"); -// true/false -``` - -### getSpatialIndexes() +### checkIntegrity() ```php -$indexes = $db->getSpatialIndexes("locations"); -// ["coords", "boundary"] +$db->checkIntegrity("users"); +// ["ok" => true, "database" => "users", "exists" => true, "segments" => 1, "lines" => 3, "records" => 3, +// "badLines" => [], "tornTail" => [], "indexMismatches" => [], "bufferedRecords" => 0, +// "pendingRecovery" => false, "quarantineFiles" => [], "recoveryLog" => [...]] ``` -### dropSpatialIndex() +A full check of the database against its data, read-only: every line is decoded; the key index, the field and +spatial indexes of each segment and the shard list are compared with the data. Nothing is created, changed, cut or flushed. +- `badLines`: `[segment, offset, length]` of lines that don't decode; +- `tornTail`: an unterminated last line (a crashed append), with its segment, offset and length; +- `indexMismatches`: entries that disagree with the data, and generations other than the active one; +- `pendingRecovery`: the state record says recovery is due (the next read or write runs it); +- `quarantineFiles`, `recoveryLog`: the quarantine files and the newest 20 events. -```php -$result = $db->dropSpatialIndex("locations", "coords"); -// ["success" => true, "error" => null], also when there was no such index (unlike dropFieldIndex()) -``` +A damaged database and an interrupted generation switch throw `FORMAT_DAMAGED` here too; the next read or write +repairs what it can prove. `rebuildIndex()` repairs index mismatches. -### rebuildSpatialIndex() +### getRecoveryLog() ```php -$result = $db->rebuildSpatialIndex("locations", "coords"); -// Drops and recreates the spatial index from existing data +$db->getRecoveryLog("users", 100); // the newest 100 events, oldest first: [["t" => ..., "action" => ..., ...], ...] ``` -### withinDistance() +| Action | What happened | +|---|---| +| `cut-tail`, `drop-line`, `drop-buffer` | Bytes removed from a data or buffer file, kept in a verified quarantine file (`quarantine`) | +| `salvaged`, `skip-line` | A damaged line found while an index was rebuilt: the record at its end was recovered, or the line skipped | +| `meta-rebuilt` | `shards.json` rebuilt from the shard files | +| `index-dropped` | An index that couldn't be rebuilt was dropped (queries scan; it is made again later) | +| `deferred-failed` | Work after a committed write failed; it runs again | +| `format-repaired` | FORMAT rewritten from the proof of CURRENT and the generation's metadata | +| `generation-recovered`, `generation-cleanup-failed` | An interrupted generation switch finished, or its obsolete generation couldn't be removed | +| `restore`, `foreign-restore` | A `restore()`, of the database's own backup or of another one with `force` | -```php -$nearby = $db->withinDistance("locations", "coords", 28.97, 41.00, 5000); -// Records within 5000 meters (5km) -``` +Events never hold record values or key material. Reads never write the log: what a read found is listed +with `"persisted" => false` until a write session of the same process logs it. -### withinBBox() +### getMetrics() ```php -$inArea = $db->withinBBox("locations", "coords", 28.97, 41.00, 29.00, 41.03); +$db->getMetrics(); // ["process" => [...]] +$db->getMetrics("users"); // ["process" => [...], "database" => [...]] ``` -### nearest() - -```php -$closest = $db->nearest("locations", "coords", 28.97, 41.00, 5); -// 5 nearest records -``` +Counters of the process (index cache hits, misses and evictions, index pages read, checkpoints, compactions, +lock waits and timeouts, recovery events, durability, the last `checkIntegrity()`) and a database's metadata +(identity, counters, sizes by class, state). Nothing is scanned or written. Every member: +[OPERATIONS.md](OPERATIONS.md) §1. -### withinPolygon() +### rebuildIndex() / getIndexInfo() ```php -$polygon = ['type' => 'Polygon', 'coordinates' => [...]]; -$inside = $db->withinPolygon("locations", "coords", $polygon); +$db->rebuildIndex("users"); // ["success" => true, "totalRecords" => 3, "time" => "0.3ms"] +$db->getIndexInfo("users"); // ["exists" => true, "version" => 1, "created" => ..., "updated" => ..., "totalRecords" => 3, + // "sharded" => false, "path" => ".../segments/main.kix"]; null when missing ``` -### validateGeoJSON() - -```php -$result = $db->validateGeoJSON($geometry); -// ["valid" => true, "error" => null, "type" => "Point"] -// ["valid" => false, "error" => "Longitude must be between -180 and 180", "type" => null] -``` +`rebuildIndex()` makes the key indexes again from the data files, and the field and spatial indexes of every +segment and the global ones. Normal operation never needs it; it is a repair tool. --- -## Utility Methods - -### sort() +## Backup and restore ```php -$sorted = $db->sort($results, "name", "asc"); -$sorted = $db->sort($results, "created_at", "desc"); -``` +$backup = __DIR__ . '/backups/users-' . date('Ymd-His'); // outside dbDir +$db->backup("users", $backup); +// ["database" => "users", "dir" => "/var/www/app/backups/users-20261029-142233", "databaseId" => "...", "generation" => 1, +// "nextKey" => 4, "records" => 3, "bufferedRecords" => 0, "files" => 6, "bytes" => 1688] -Stable, total order: missing/`null` < booleans < numbers < strings < arrays (reversed for `desc`). -Returns `[]` for an empty array and `false` for a non-array. +$db->insert("users", ["name" => "after the backup"]); +$db->restore($backup, "users"); +// ["database" => "users", "generation" => 2, "nextKey" => 5, "records" => 3, "bufferedRecords" => 0, +// "target" => "present", "foreign" => false] +``` -### limit() +- `backup($db, $dir)` copies one database as it is at one moment into `$dir`: a directory outside `dbDir` that + doesn't exist or is empty (0700, files 0600). `manifest.json` is written last; `store.json` is the store + record, usable by `repairStore()`. Writers of the database wait while it runs, other databases don't; key + maintenance fails with `MAINTENANCE_BUSY`. The buffer is copied, not flushed: nothing of the database is + written. +- `restore($dir, $db, ['force' => false])` makes the backup the content of the database with one commit: after + a crash it opens as it was or as restored. New records get keys above every key the database handed out + (`nextKey` 5 above, not 4). +- A backup of another database is `RESTORE_REFUSED` unless `['force' => true]`, which loads it under this + database's identity. An incomplete or damaged backup is always refused; nothing changes. +- `target`: `present`, `absent` (the database is created), or `damaged` (`restore()` is the way out of + `FORMAT_DAMAGED`; the damaged generations are kept for inspection). -```php -$limited = $db->limit($results, 10); // [] for an empty array, false for invalid arguments -``` +Procedures, the directory layout and the rules: [BACKUP_RESTORE.md](BACKUP_RESTORE.md). -### first() / last() +--- -```php -$first = $db->first("users"); -$last = $db->last("users", ["active" => true]); -``` +## Key maintenance -### exists() +Static methods for the store in a directory; each returns `success`, `error` and `code` instead of throwing, +and `storeManifest`: the complete store record (keep it; `repairStore()` restores it). + ```php -$exists = $db->exists("users", ["email" => "test@test.com"]); -// true/false -``` +noneDB::registerKey($dbDir, $registeredKey, $newKey); +// ["success" => true, "storeManifest" => [...], "error" => null, "code" => null] -### like() +noneDB::migrateKey($dbDir, $oldKey, $newKey); // ['abort' => true] undoes an interrupted one +// ["success" => true, "databases" => ["users", "orders"], "storeManifest" => [...], "error" => null, "code" => null] -```php -$results = $db->like("users", "email", "gmail"); // Contains -$results = $db->like("users", "name", "^John"); // Starts with -$results = $db->like("users", "name", "son$"); // Ends with +noneDB::repairStore($dbDir, [$key1, $key2], ['manifest' => '/backups/users/store.json']); +// ["success" => true, "mode" => "manifest", "complete" => true, "storeManifest" => [...], "error" => null, "code" => null] ``` -Case-insensitive. Only a leading `^` and a trailing `$` are special; every other character (`.`, `/`, -`*`, ...) matches itself. +- `registerKey()` lets a second key open the store; `$registeredKey` must be registered already. +- `migrateKey()` rotates every database of `$oldKey` to `$newKey` (their directories are renamed; the contents + don't change). Stop the application processes first: a database in use is `MAINTENANCE_BUSY` and nothing is + written. An interrupted rotation is finished by running it again. +- `repairStore()` is only for a store whose two record copies are both invalid. With a manifest (a backup's + `store.json`, a `storeManifest`) the record comes back complete; without one only + `['confirmManualRebuild' => true]` rebuilds it from the given keys (`complete` false). Every database + directory must belong to one of the keys. +- `['durability' => 'durable']` syncs every step as durable mode does. +- The options: `migrateKey()` `abort` (bool) and `durability`; `registerKey()` `durability`; `repairStore()` + `manifest` (a file path or a store record array), `confirmManualRebuild` (bool) and `durability`. -### between() +Order of the steps and what happens after a crash: STORAGE_FORMAT.md §9; procedures: OPERATIONS.md. -```php -$results = $db->between("products", "price", 100, 500); -$results = $db->between("products", "price", 100, 500, ["active" => true]); -``` +--- -### Static Cache +## Static helpers ```php -noneDB::clearStaticCache(); -noneDB::disableStaticCache(); +noneDB::configExists(); // a .nonedb file was found +noneDB::getConfigTemplate(); // path of the template file, or null +noneDB::isDevMode(); // false +noneDB::clearStaticCache(); // drop the process's caches (tests; long-running workers after maintenance) +noneDB::disableStaticCache(); // each instance keeps its own caches noneDB::enableStaticCache(); ``` -### Buffering and indexes - + ```php -$db->enableBuffering(true); // off by default -$db->setBufferSizeLimit(1048576); // bytes (min 1KB) -$db->setBufferCountLimit(10000); // records (min 10) -$db->setBufferFlushInterval(30); // seconds, 0 = no time-based flush -$db->flush("users"); // ["success" => true, "flushed" => 12, "error" => null] -$db->flushAllBuffers(); // ["databases" => 2, "flushed" => 40] -$db->getBufferInfo("users"); // ["enabled" => true, ..., "buffers" => ["main" => ["exists" => true, "size" => 812, "records" => 12]]] - -$db->rebuildIndex("users"); // rebuild the key indexes from the data files (recovery tool) -$db->getIndexInfo("users"); // ["exists" => true, "totalRecords" => 120, "sharded" => false, ...] +noneDB::setDevMode(true); // no config file needed (development only) +noneDB::clearConfigCache(); // forget the loaded .nonedb (tests) ``` -### Configuration +--- -```php -noneDB::configExists(); // Check config file -noneDB::getConfigTemplate(); // Get template path -noneDB::clearConfigCache(); // Clear cached config -noneDB::setDevMode(true); // Enable dev mode -noneDB::isDevMode(); // Check dev mode +## Errors + +Invalid input to a CRUD method returns an array with an `error` member (above). Wrong names, settings and +query operators throw `InvalidArgumentException`. States throw `RuntimeException` with a code: + +| Code | Constant | Meaning | +|---|---|---| +| 3301 | `OUTCOME_UNKNOWN` | A failure after a commit point, or an undo that failed: read the state again; the next access recovers | +| 3302 | `LOCK_FAILED` | A lock was not taken within `lockTimeout`, or was lost | +| 3304 | `UNSUPPORTED_FORMAT` | `dbDir` holds files that are not noneDB 4 objects: migrate them (MIGRATION.md) or move them | +| 3305 | `FORMAT_DAMAGED` | A store record, FORMAT, CURRENT or generation is damaged and can't be repaired with proof; the data files are kept | +| 3306 | `UNSUPPORTED_DURABILITY` | Durable mode on a platform that can't provide it (only Linux can) | +| 3307 | `DURABILITY_FAILED` | A sync durable mode requires failed before the commit point: rolled back | +| 3308 | `KEY_NOT_REGISTERED` | The configured `secretKey` is not registered for the store | +| 3309 | `MAINTENANCE_IN_PROGRESS` | Key maintenance or a migration is running or was interrupted (run it again) | +| 3310 | `RESTORE_REFUSED` | `restore()` refused the backup; nothing changed | +| 3311 | `MAINTENANCE_BUSY` | Maintenance found a database in use; nothing changed | + +`LogicException`: a file of a database written without its exclusive lock (a bug), `update()`/`delete()`/ +`removeFields()` together with `groupBy()`, a replacing operation inside an uncommitted write, serializing an +instance. Other I/O failures are plain `RuntimeException`s. + +```php +try { + $db->restore('/nonexistent/backup', "users"); +} catch (RuntimeException $e) { + $e->getCode() === noneDB::RESTORE_REFUSED; // true +} ``` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..e0d1211 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,200 @@ +# noneDB 4 architecture + +How the engine is built from its source and where each responsibility lives. The storage format is in +[STORAGE_FORMAT.md](STORAGE_FORMAT.md), the durability contract in [DURABILITY.md](DURABILITY.md). + +## One file to include, modular source + +Applications include one file: + + +```php +require 'noneDB.php'; +``` + +`noneDB.php` is generated by `tools/build.php` from `src/`. Never edit it directly. + +- **Build steps:** + 1. The header `src/header.php`. + 2. Every internal class file of `src/manifest.php` (`files`), as a bracketed namespace block (`namespace noneDB\Internal\... { ... }`). + 3. The public classes (`classes`) in `namespace { ... }`, each assembled from its modules. +- **Checks on the result:** + - It declares exactly the expected classes and no trait. + - Braces are balanced. + - It passes `php -l`. + - `php tools/build.php --check` fails when `noneDB.php` is not the build of `src/`; CI runs it. +- **File rules:** + - A class file `src/Internal//.php` holds `namespace noneDB\Internal\;` and one class ``. + - A module `src//.php` holds part of a public class's body in a trait. The trait is never loaded; it only helps editors. + - LF line ends only. + +Each internal file imports the PHP functions and constants it uses (`use function`, `use const`). The +compiler then resolves them at compile time: `strlen()`, `count()`, `is_array()`, … compile to their +special opcodes, as in global code, with no namespace fallback at run time. + +## Public API and engine + +| Class | Role | +|---|---| +| `noneDB` (global) | The public API. It holds one integer, the key of its kernel in `Kernel::$instances`, and delegates every method to a service of that kernel. A dump of it (`var_export`, `json_encode`) shows nothing internal; `__debugInfo()` shows the settings without the key. | +| `noneDBQuery` (global) | The fluent query builder: it compiles its filters, sorts and projections and calls the kernel's Query services. | +| `noneDB\Internal\Core\Kernel` | One instance's engine: its values (settings, per-instance caches) and its services. The composition root. | +| `noneDB\Internal\Cache\Runtime` | Process-wide state shared by every instance: the static caches, the lock registry and lock handles, the counters of `getMetrics()`, the test hooks. One object (`Runtime::get()`). | + +### Services + +A service is a class with one responsibility. It has the kernel in its constructor. + +- **Creation:** the kernel creates a service on first use (`Kernel::__get`). A simple operation builds only the services it needs. +- **Binding values:** when created, a service binds the kernel's and the runtime's values it uses **by reference** (`bind()`). It then reads and writes them as `$this->name`, with the same semantics as one shared object. +- **Handles to other services:** these are bound on first use (the service's `__get`). +- **When the `noneDB` instance is destroyed:** + - `Kernel::release()` drops all handles between the services and the kernel. + - Nothing is left for PHP's cycle collector; the memory is freed at once. +- **Cloning:** a cloned instance gets a copy of the kernel with its own values. Only the caches linked to the runtime stay shared; the services of the copy are new. + +## Module map + +| Class (`noneDB\Internal\…`) | Responsibility | +|---|---| +| `Config\Configuration` | Settings of an instance: the `.nonedb` file, the config array, dev mode, the secret key | +| `Cache\Runtime` | Process-wide state shared by every instance | +| `Cache\CacheManager` | Ownership of the caches: the static cache switch, memory limit and eviction (when the last lock is released, and between the segments of any operation over many segments), invalidation by lock state | +| **Storage engine** | | +| `Storage\StoreRecord` | The store record (`STORE`, `STORE.2`, `STORE.LOCK`) and its rules (STORAGE_FORMAT §3) | +| `Storage\Layout` | Database names, directories, `FORMAT`, `CURRENT` and generations (§4, §5, §8) | +| `Storage\Segments` | Segments of a database: their files, existence, the key ranges of the shards | +| `Storage\Checkpoints` | Key-index checkpoints: load, replay, write, repair (§6.4) | +| `Storage\KeyIndex` | The key index of a segment, in memory and in its `.kix` file: the body (12 bytes per slot) plus a delta, lookups, encoding and decoding (static helpers, §6.4) | +| `Storage\RecordIO` | Reading and appending data lines of a segment | +| `Storage\Writer` | Inserts, updates and deletes of records in a segment | +| `Storage\Buffer` | The write buffer: buffered inserts and their flush | +| **Sharding** | | +| `Sharding\ShardMeta` | `shards.json`: the shard list and counters (§5.1) | +| `Sharding\ShardRouter` | Writes to the shards of a sharded database, new shards, orphan shard files | +| `Sharding\LayoutSwitch` | The switch of a database to the sharded layout, a generation switch (§5.2): one shard at a time, each written with its key-index checkpoint and verified | +| `Sharding\ShardInfo` | `getShardInfo()` | +| **Index engine** | | +| `Index\SecondaryFiles` | Secondary index files: names, headers, checkpoints, data tails (§6.5) | +| `Index\FieldIndex` | Field indexes and global field indexes: building, point lookups through pages, loading whole, the shards a value may be in | +| `Index\PagedFile` | The paged `.fx` encoding: JSON data pages, tabled directory pages, directory levels, lookups inside a page (static helpers, §6.5.1) | +| `Index\ShardFilters` | The `.gfx` shard filters: probes of a bucket id, a shard's filter, a shard's column of a page replaced, sealed rows, the shards a set of rows holds (static helpers, §6.5.2) | +| `Index\FieldIndexApi` | The public field index operations | +| `Index\Spatial\GeoJson` | GeoJSON validation (exact orientation tests) | +| `Index\Spatial\Geometry` | Geometry operations: bounding boxes, distances, containment | +| `Index\Spatial\RTree` | R-trees, the global spatial index and their files | +| `Index\Spatial\SpatialIndexApi` | The public spatial index operations | +| **Query engine** | | +| `Query\FindFilter` | Filters of `find()`, `update()`, `delete()`: normalization and matching | +| `Query\Values` | Value semantics: typed identity (`valueKey`), the total sort order (`sortKey`), comparison for ordering operators, numbers of values, `like()` patterns, Unicode case folding | +| `Query\Planner` | Query plans: index candidates, shard skipping, batched scans, `explain()` | +| `Query\QueryMethods`, `Query\Aggregation`, `Query\Utility` | `distinct/sort/count/like`, `sum/avg/min/max`, `first/last/exists/between` | +| `Query\SpatialQueries` | `withinDistance()`, `withinBBox()`, `nearest()`, `withinPolygon()` | +| **Concurrency** | | +| `Concurrency\LockManager` | Database locks, lock sessions, the state record, epochs (§4.2) | +| `Concurrency\Snapshot` | Shared locks of several databases at once: one consistent view for joins | +| **Recovery and durability** | | +| `Recovery\RollbackScope` | The rollback scope of a write session: marks, rollback, independent jobs | +| `Recovery\RecoveryLog` | The recovery log of a database (§7.1) | +| `Recovery\Recovery` | Crash recovery: journal recovery, quarantine of removed bytes, stale temp files, fault points | +| `Durability\FileIO` | Writes to database files: atomic replacement, appends with undo, new directories, the write helpers | +| `Durability\Sync` | `fsync` of files and directories, the syncs durable mode requires and their failures (`DURABILITY_FAILED` / `OUTCOME_UNKNOWN`), the outcome of write operations and the durability report | +| **Operations** | | +| `Core\Crud` | `find()`, `insert()`, `update()`, `delete()` | +| `Core\DatabaseOps` | Databases: create, drop, list, info, key index rebuild | +| `Diagnostics\Integrity` | `checkIntegrity()`: a read-only check of a database against its data | +| `Diagnostics\Metrics` | `getMetrics()`: the counters of the process and the metadata of a database (OPERATIONS.md §1) | +| `Maintenance\Maintenance` | Buffer settings, `flush()`, `compact()`, `convertToSharded()` | +| `Maintenance\Compaction` | Compaction of segments (in-generation data file replacement) | +| `Maintenance\KeyMaintenance` | `migrateKey()`, `registerKey()`, `repairStore()` (§9) | +| `Maintenance\BackupRestore` | `backup()` and `restore()`: the backup directory, its manifest, the restored generation (§10) | + +## Memory + +Operations stream: a read over many segments works on one batch of one segment at a time and calls +`CacheManager::evictBetweenSegments()` before the next, so its memory does not grow with the database. + +What a process keeps between operations has to pay for itself first. All of it lives in the cache: it counts in the +process's memory, goes least recently used first, and is loaded again when needed (`cacheMemoryLimit`, CONFIGURATION.md). + +| Kept | From when | Saves | +|---|---|---| +| Key index arrays (`.kix`) | STORAGE_FORMAT.md §6.4 | Decoding the 12-byte slots per lookup | +| Field index pages, or the whole `.fx` | STORAGE_FORMAT.md §6.5.1 | Reading pages per lookup | +| Global index body, exact map | STORAGE_FORMAT.md §6.5.2 | Probing the file per lookup | +| Data bytes of a segment | See below | An open, a seek and a read per record | + +**Data bytes (`RecordIO::heldData()`).** A record read from a file costs about what reading 128 KiB at once does, +whatever the size of the record. So a data file in repeated use is read whole and held, and its records become +`substr()` calls. The bytes are held when all of these are true: + +- The database has more than 16 segments. With fewer, the key index arrays already answer cheaply. That mark is set + when a key index is loaded from its file: a segment whose index the process built itself can hold its bytes in a + small database too. +- The segment had at least 2 operations, and operations × 128 KiB reach the bytes its key index covers. +- The process holds the shared lock: those bytes are committed, so no rollback can take them back. A writer uses bytes + that are held already and holds no new ones. +- The process has a limit, and stays under half of it with twice the file's size added. A process with no limit + holds none, because nothing would bound them. + +Held bytes stay valid while the key index keeps its epoch: the bytes it covers can't change within one. Lines newer than +those bytes come from the file. `checkIntegrity()` always reads the disk. Held bytes never change an answer; they +only decide where the bytes come from. + +**One lookup per segment.** `FindFilter::segmentMatches()` looks up the key index once per segment and hands it to the +field index and to the read. **Whole segments:** a read of every record of a segment without garbage or tombstones splits the +lines at once (`RecordIO::liveLines()`) and checks each key and offset against the key index; anything unexpected falls +back to the read by position. + +## Rules, checked by `tests/Unit/ArchitectureTest.php` + +1. **The Query layer knows no files and no format.** + - Applies to `src/Internal/Query/` and the builder. + - No file-system calls and no storage names (`.jsonl`, `.kix`, `STORE`, `FORMAT`, `segments/`, …). + - It works on records and plans and calls the storage services. +2. **Dependencies point inwards.** The engine never uses the query builder; the builder calls the engine's Query services. +3. **The migration tool is separate.** `noneDB.php` declares and references none of its symbols. +4. **No trace of earlier versions** in the engine: + - no old file names or encodings, + - no version-dated wording, + - no PHP-version branches. +5. **One class per file**, in the namespace of its directory. +6. **Tests reach the engine only through `noneDB\Tests\Support\Testing`.** No other internal class is referenced and no Reflection is used. +7. **`noneDB.php` holds only the engine.** No test interface, no fault point, test hook or test counter (`ArchitectureTest::theEngineHoldsNoTestCode`); those are test seams of the test build (below). + +`tests/Unit/SourceWriteGuardTest.php` adds the write rule. A function writes a database file only through the write helpers of `Durability\FileIO`: `writeFileAtomic`, `appendToFile`, `fsUnlink`, `fsRename`, `fsTouch`, `fsTruncate`, `fsMkdir`, `fsRmdir`. Each gives what it creates the configured permissions (`setPermissions`). The guard lists the exceptions: lock files, quarantine, rollback, store-level files, backup directories. + +## Test build and test interface + +`noneDB.php` contains only what the engine needs to run. Code that exists for tests is a **test seam** in `src/`, marked for `tools/build.php`: +- **`// @test`:** a line ending with it (a fault point, a test hook, a counter tests read) is left out of `noneDB.php`. +- **`// @test-begin` … `// @test-end`:** the lines between them are left out too (the fault method). +- **`// @prod`:** a line ending with it is the production form of the `@test` line next to it, and is left out of the test build. + +**The test build** (`nonedb_build($root, true)`) keeps the seams. `tests/bootstrap.php` writes it to `tests/build/noneDB.php` (ignored by git) whenever `src/` changed, and the tests and their child processes load it. Both builds come from the same files; they differ only in the marked lines. + +**`noneDB\Tests\Support\Testing`** (`tests/Support/Testing.php`) is how tests reach internals. It reaches the engine's classes with closures bound to them, so it also works with `noneDB.php` itself: the smoke scripts load it next to the production build. The hooks and counters exist only in the test build. + +| Method | What it gives access to | +|---|---| +| `setFaultHook()` | The fault points | +| `get()` / `set()` | Process-wide values: hooks such as `syncFn`, `flushFn`, `writeFn`, `renameFn`, `mbstringFn`, `osFamily`; counters such as `directoryListings`, `indexStats` | +| `getOf()` / `setOf()` | Values of one instance (settings, caches) | +| `call()` / `callStatic()` | Internal methods of any visibility | +| `engine()` | An object standing for the whole engine of an instance, to bind a test closure to | +| `service()` | The service owning a method | +| `liveInstances()` | The number of live instances | + +## Test suites + +| Suite | Contents | +|---|---| +| `Unit` | Components alone, the build, the architecture rules, the write guard, exhaustive geometry checks | +| `Integration` | The public API and every documented rule (specification tests), on real files | +| `Crash` | Interruptions at every step (exit and SIGKILL in child processes), torn writes, power-loss artefacts, rollback | +| `Concurrency` | Several processes on one database: locks, the state record, cache validation | +| `Differential` | Results against independent models: fuzzing, index results = scan results, geometry | +| `Migration` | The migration tool against databases the old releases wrote themselves (`tests/Migration/fixtures/`, made by `tools/make-migration-fixtures.php`, not in git) | +| `tests/Performance` | The performance gate (A/B against a baseline), the large-database harness, the benchmarks (scripts, not PHPUnit) | +| `tests/Smoke` | Standalone checks (`NONEDB_PATH` selects the library) | +| `tests/Docs` | `doctest.php`: the examples of the documentation, run against `noneDB.php` | diff --git a/docs/BACKUP_RESTORE.md b/docs/BACKUP_RESTORE.md new file mode 100644 index 0000000..4c4d796 --- /dev/null +++ b/docs/BACKUP_RESTORE.md @@ -0,0 +1,159 @@ +# noneDB Backup and Restore + +How to back up a database while it is in use, how to check a backup, and how to bring it back. The rules are +normative in [STORAGE_FORMAT.md](STORAGE_FORMAT.md) §10; the methods are in [API.md](API.md#backup-and-restore). +The examples run in order (CI runs them). + +```php +$db = new noneDB(); +$db->insert("users", [["name" => "John", "plan" => "pro"], ["name" => "Jane", "plan" => "free"]]); +$db->createFieldIndex("users", "plan"); +``` + +--- + +## 1. What a backup is + +`backup($db, $dir)` copies **one database, as it was at one moment**, into a directory of its own: + +- **One moment.** The backup holds the store's lock and the database's lock (both shared) until the copy is + complete: data, write buffer, counters, indexes and the store record all belong to the same state. Writers of + this database wait meanwhile (up to their `lockTimeout`); readers and other databases don't. +- **Nothing of the database is written.** The write buffer is copied, not flushed; no checkpoint is written. +- **Verified.** Every copy is read back and compared before `manifest.json` is written, last. A backup without a + manifest (interrupted, failed) is refused by `restore()`. +- **Protected.** The directory is `0700`, its files `0600`. It must lie outside `dbDir` and must not exist or be + empty. +- **Complete for a restore.** It holds `FORMAT`, `CURRENT`, the active generation (data, buffer, shard list, + indexes), the recovery log and `store.json`, the store record. Indexes come along, so a restored database + answers at full speed at once. + +```php +$dir = __DIR__ . '/backups/users-' . date('Ymd-His'); +$result = $db->backup("users", $dir); +$result['records']; // 2 +$result['files']; // the files copied (manifest.json not counted) +``` + +The directory follows the layout of a database directory: + +```text +users-20261029-142233/ + manifest.json written last: identity, counters, every file's size and SHA-256 + store.json the STORE record: keep it for repairStore() + FORMAT, CURRENT + generations/1/... metadata, segments, indexes, buffer + recovery/... the database's recovery log and quarantine files (for inspection) +``` + +A backup takes about the time of copying the database's files twice (the copy and its read-back) and as much +space as they take. A large database keeps its writers waiting that long: plan backups of large databases when +writes are rare, or give the writers a `lockTimeout` that covers them. + +## 2. Backing up several databases + +Each `backup()` is one database at one moment. Databases backed up one after the other belong to different +moments; that is usually what is wanted. When two databases must agree (orders and their items), back them up +while their writers are stopped. + +```php +$db->insert("orders", ["user" => 0, "total" => 49.9]); +$stamp = date('Ymd-His'); +foreach ($db->getDBs(false) as $name) { + $db->backup($name, __DIR__ . "/backups/all-$stamp/$name"); +} +``` + +Keep backups on another disk or another machine, and keep several generations of them. A backup directory is +plain files: `rsync`, `tar` or object storage can carry it. Its checksums detect damage, not a deliberate +change: protect it as you protect `dbDir`. + +## 3. Checking a backup + +`restore()` checks a backup completely before it changes anything (its manifest, every SHA-256, the identity +records). A restore into a scratch store is the full test of a backup: it is also what a restore drill should +do from time to time. + +```php +$scratch = new noneDB(['secretKey' => getenv('NONEDB_SECRET'), 'dbDir' => __DIR__ . '/restore-check/']); +$check = $scratch->restore($dir, "users"); // "absent": created there with the backup's identity +$check['records']; // 2, as the manifest says +$scratch->checkIntegrity("users")['ok']; // true +``` + +## 4. Restoring + +```php +$db->insert("users", ["name" => "written after the backup"]); +$result = $db->restore($dir, "users"); +$result['target']; // "present" +$result['nextKey']; // 3: keys handed out after the backup are not handed out again +$db->count("users"); // 2 +``` + +- **One commit.** The backup is copied into a new generation, verified (checksums, counts, the shard list, the + next key), and published by one replacement of `CURRENT` (STORAGE_FORMAT.md §5.2). After a crash at any step + the database opens either as it was or as restored, never a mix. The old generation is removed only after the + new one is published. +- **Keys never go back.** Records written after the backup are gone, but their keys stay used: new records get + keys above every key the database handed out (a top tombstone keeps the highest one even if the counters are + lost). +- **Buffered records** of the backup come back and are flushed by the next read. +- **Writers wait.** A restore is a write: it waits for the database's lock like any write. +- **Recorded.** The database's recovery log gets a `restore` event (generation, the backup's `databaseId`, + `takenAt`, `records`). + +### Into a database that doesn't exist + +```php +$db->dropDB("users"); +$db->restore($dir, "users"); // target "absent": the database is created with the backup's identity +``` + +A backup is restored into the name it was taken from without `force`. Under another name it is a copy: with +`['force' => true]` it gets a new identity (a new `databaseId`). + +### A backup of another database + +The backup's `databaseId` must be the database's: restoring the wrong backup over a live database is refused +(`RESTORE_REFUSED`, nothing changed). To load another database's backup on purpose (a copy of production into +staging), pass `['force' => true]`: the database keeps its own identity and the event is logged as +`foreign-restore`. + +A sharded backup can't be loaded into a database of another shard size (its shard files wouldn't fit the key +ranges): `RESTORE_REFUSED`, with `force` too. Restore it into a database of its own. + +### Over a damaged database + +A database that fails with `FORMAT_DAMAGED` (its `CURRENT` or `FORMAT` can't be read and can't be repaired with +proof) can still be restored: `restore()` is the way out. The result's `target` is `"damaged"`. The damaged +generations are **kept** in the database directory as unreferenced generations: nothing proves which of them was +the committed one, so noneDB removes none. `checkIntegrity()` reports them; once you have looked at them (or +copied them away), remove them as OPERATIONS.md §5 describes. The next key is above every key found in their +data. + +## 5. When the store record is lost + +Every backup carries `store.json`, the store record of that moment (the keys registered for the store). When +both copies of the store record are damaged, `repairStore()` restores it from there: + + +```php +noneDB::repairStore('/var/lib/app/nonedb', [$key], ['manifest' => '/backups/users-20261029-142233/store.json']); +// ["success" => true, "mode" => "manifest", "complete" => true, ...] +``` + +Keys registered after that backup and without a database of their own are outside it: keep the newest +`storeManifest` a key operation returns (OPERATIONS.md §4). + +## 6. Errors + +| Error | Meaning | Changed | +|---|---|---| +| `InvalidArgumentException` (backup) | The directory is inside `dbDir`, or exists and is not empty | nothing | +| `InvalidArgumentException` (restore) | An option other than `force`, or a `force` that is not `true` or `false` | nothing | +| `RuntimeException` (backup) | The database doesn't exist; a copy failed or didn't read back (the backup is removed) | nothing | +| `MAINTENANCE_IN_PROGRESS`, `KEY_NOT_REGISTERED`, `LOCK_FAILED` | As for any operation | nothing | +| `RESTORE_REFUSED` | An incomplete or damaged backup; another database's without `force`; another shard size; a restored generation that doesn't verify | nothing (a generation that didn't verify is removed) | +| `DURABILITY_FAILED` | A sync failed before the commit point (restored data files are synced in performance mode too) | nothing: rolled back | +| `OUTCOME_UNKNOWN` | A failure after the commit point | read the state again: the next access recovers | diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md deleted file mode 100644 index 3e0c73d..0000000 --- a/docs/BENCHMARKS.md +++ /dev/null @@ -1,350 +0,0 @@ -# noneDB Performance Benchmarks - -Performance measurements of noneDB 3.2.0, compared with 3.1.0 and SleekDB. - -**Version:** 3.2.0 -**Test environment:** PHP 8.2.0 (CLI), macOS 15.8, Apple M2, 16 GB RAM -**Measured:** 2026-10-24. The 3.1.0 numbers come from the same scripts, run on the same machine in the same session. - -All 3.2.0 numbers include the database lock every operation takes (shared for reads, exclusive for -writes) and crash-safe writes. 3.1.0 took no lock for reads and overwrote index files in place. - ---- - -## Table of Contents - -1. [Test Data Structure](#test-data-structure) -2. [What Changed in 3.2](#what-changed-in-32) -3. [Single Operations (5,000 records)](#single-operations-5000-records) -4. [Key Lookup](#key-lookup) -5. [3.2 vs 3.1 at Scale](#32-vs-31-at-scale) -6. [Write Operations](#write-operations) -7. [Read Operations](#read-operations) -8. [Query & Aggregation](#query--aggregation) -9. [Method Chaining](#method-chaining) -10. [Comparison Operators](#comparison-operators) -11. [Spatial Query Performance](#spatial-query-performance) -12. [Storage & Memory](#storage--memory) -13. [SleekDB Comparison](#sleekdb-comparison) -14. [Running Benchmarks](#running-benchmarks) - ---- - -## Test Data Structure - -The performance and SleekDB benchmarks use 7 fields per record: - -```php -[ - "name" => "User123", - "email" => "user123@test.com", - "age" => 25, - "salary" => 8500, - "city" => "Istanbul", - "department" => "IT", - "active" => true -] -``` - ---- - -## What Changed in 3.2 - -| Change | Effect | -|--------|--------| -| **Database lock per operation** | Correct results with several processes; costs ~0.01-0.02 ms per call | -| **Lock file state record** | Generation counter, buffer flag, index journal and the one-time old-version check are read with the lock: no per-operation `stat`/`glob` | -| **Kept lock handles** | No `fopen`/`fclose` per operation (pid- and inode-checked) | -| **Atomic index writes** (temp file + rename) | A crash never leaves a half-written index; a single-record write costs ~0.15 ms more | -| **Batch inserts/updates** | Records encoded and appended in one write, each index written once: insert 10x, filtered update 4x faster | -| **Per-shard key lookup** | A cold lookup loads only the target shard's index: 5 ms instead of 156 ms at 500K | -| **Compiled query filters** | `where()`/`orWhere()` compiled once per query; `array_multisort` on precomputed sort keys | -| **Adaptive `query()->nearest()`** | With other filters, searches a growing radius instead of sorting the whole database | - -### Earlier optimizations - -| Optimization | Since | -|--------------|-------| -| JSONL storage with byte-offset index (O(1) key lookup) | 3.0 | -| Static cache shared across instances | 3.0 | -| Batch file read (64 KB buffered) | 3.0 | -| O(1) count from index metadata | 3.0 | -| R-tree spatial index: parent pointer map, linear split, dirty-flag batch writes, node size 32 | 3.1 | -| Distance and centroid memoization | 3.1 | - ---- - -## Single Operations (5,000 records) - -Milliseconds per call, best of 5 runs (`tests/operation_benchmark.php`). - -| Operation | 3.1.0 | 3.2.0 | -|-----------|-------|-------| -| insert (5,000 records, one call) | 175.4 | **17.2** | -| insert (one record) | **0.169** | 0.321 | -| find (all records) | 16.2 | **10.4** | -| find (by key) | **0.026** | 0.047 | -| find (filter) | 17.4 | **12.0** | -| count | **0.002** | 0.018 | -| query (where range + sort + limit) | 25.5 | **19.9** | -| update (by key) | **0.830** | 0.977 | -| update (filter, 1,250 records) | 102.6 | **25.0** | -| delete (by key) | **0.933** | 1.1 | -| find (filter on indexed field) | 4.0 | **3.2** | -| insert (one record, indexed field) | **1.2** | 1.5 | -| new instance + find (by key) | **0.029** | 0.056 | - -Bulk work is much faster. The smallest operations pay a fixed cost: the lock (all operations) and the -atomic index replacement (single writes). Those are what make 3.2 safe with concurrent processes and -crashes. - ---- - -## Key Lookup - -`find($db, ['key' => $k])`. Cold = nothing cached (first lookup of a new process); warm = average of -200 lookups spread over all shards, after each shard's index was loaded once. - -| Records | Cold 3.1.0 | Cold 3.2.0 | Warm 3.1.0 | Warm 3.2.0 | -|---------|------------|------------|------------|------------| -| 100 | 3 ms | **0.21 ms** | **0.030 ms** | 0.045 ms | -| 1,000 | 3 ms | **0.45 ms** | **0.030 ms** | 0.046 ms | -| 10,000 | 9 ms | **3 ms** | **0.030 ms** | 0.054 ms | -| 50,000 | 22 ms | **5 ms** | **0.042 ms** | 0.057 ms | -| 100,000 | 37 ms | **4 ms** | 0.060 ms | **0.057 ms** | -| 500,000 | 156 ms | **5 ms** | 0.206 ms | **0.066 ms** | - -A cold lookup in 3.2 reads one shard's index, so it stays ~5 ms at any size. - ---- - -## 3.2 vs 3.1 at Scale - -Same script (`tests/performance_benchmark.php`), same machine. - -| Operation | 100K 3.1.0 | 100K 3.2.0 | 500K 3.1.0 | 500K 3.2.0 | -|-----------|------------|------------|------------|------------| -| insert() | 3.6 s | **1.3 s** | 18.8 s | **7.1 s** | -| update() (city filter) | 2.1 s | **455 ms** | 19 s | **2.4 s** | -| delete() (department filter) | 2.4 s | **831 ms** | 20.8 s | **4.4 s** | -| find(all) | 598 ms | **376 ms** | 2.7 s | **1.8 s** | -| find(key), first after insert | 490 ms | **<1 ms** | 2.5 s | **<1 ms** | -| find(filter) | 514 ms | **336 ms** | 2.7 s | **1.8 s** | -| distinct() | 586 ms | **361 ms** | 3 s | **2.1 s** | -| sort() + limit | 1.3 s | **452 ms** | 7.1 s | **2.7 s** | -| search() | **736 ms** | 849 ms | **4.2 s** | 4.6 s | -| select() | 864 ms | **596 ms** | 4.6 s | **3.6 s** | -| complex chain | 676 ms | **484 ms** | 3.7 s | **2.7 s** | - -`search()` is slightly slower because it now matches Unicode text case-insensitively (`İzmir` = `izmir`). - ---- - -## Write Operations - -| Operation | 100 | 1K | 10K | 50K | 100K | 500K | -|-----------|-----|-----|------|------|-------|-------| -| insert() | 6 ms | 7 ms | 121 ms | 596 ms | 1.3 s | 7.1 s | -| update() | <1 ms | 4 ms | 40 ms | 216 ms | 455 ms | 2.4 s | -| delete() | 2 ms | 8 ms | 78 ms | 426 ms | 831 ms | 4.4 s | - -> `update()` changes the ~20% of records in one city, `delete()` the ~20% in one department. - ---- - -## Read Operations - -| Operation | 100 | 1K | 10K | 50K | 100K | 500K | -|-----------|-----|-----|------|------|-------|-------| -| find(all) | <1 ms | 3 ms | 28 ms | 169 ms | 376 ms | 1.8 s | -| find(key) | <1 ms | <1 ms | <1 ms | <1 ms | <1 ms | <1 ms | -| find(filter) | <1 ms | 3 ms | 32 ms | 163 ms | 336 ms | 1.8 s | - ---- - -## Query & Aggregation - -| Operation | 100 | 1K | 10K | 50K | 100K | 500K | -|-----------|-----|-----|------|------|-------|-------| -| count() | **<1 ms** | **<1 ms** | **<1 ms** | **<1 ms** | **<1 ms** | **<1 ms** | -| distinct() | <1 ms | 3 ms | 34 ms | 170 ms | 361 ms | 2.1 s | -| sum() | <1 ms | 3 ms | 31 ms | 167 ms | 354 ms | 2.1 s | -| like() | <1 ms | 4 ms | 38 ms | 209 ms | 432 ms | 2.5 s | -| between() | <1 ms | 4 ms | 43 ms | 229 ms | 502 ms | 2.7 s | -| sort() | <1 ms | 4 ms | 38 ms | 211 ms | 452 ms | 2.7 s | -| first() | <1 ms | 3 ms | 31 ms | 168 ms | 364 ms | 1.8 s | -| exists() | <1 ms | 3 ms | 32 ms | 173 ms | 370 ms | 1.9 s | - ---- - -## Method Chaining - -| Operation | 100 | 1K | 10K | 50K | 100K | 500K | -|-----------|-----|-----|------|------|-------|-------| -| whereIn() | <1 ms | 3 ms | 35 ms | 227 ms | 419 ms | 2.5 s | -| orWhere() | <1 ms | 4 ms | 39 ms | 220 ms | 460 ms | 2.5 s | -| search() | <1 ms | 7 ms | 76 ms | 398 ms | 849 ms | 4.6 s | -| groupBy() | <1 ms | 3 ms | 35 ms | 209 ms | 434 ms | 2.4 s | -| select() | <1 ms | 4 ms | 38 ms | 277 ms | 596 ms | 3.6 s | -| complex chain | <1 ms | 4 ms | 39 ms | 235 ms | 484 ms | 2.7 s | - ---- - -## Comparison Operators - -`tests/operator_benchmark.php`, best of 3 runs (records with 10 fields, including a GeoJSON point). - -| Operator | 100 | 500 | 1K | 5K | -|----------|-----|-----|-----|-----| -| `$gt` | 0.68 ms | 2.9 ms | 5.8 ms | 29 ms | -| `$gte + $lte` (range) | 0.72 ms | 3.5 ms | 7.1 ms | 36 ms | -| `$in` (2 values) | 0.6 ms | 2.8 ms | 5.6 ms | 30 ms | -| `$nin` (2 values) | 0.6 ms | 2.8 ms | 5.7 ms | 29 ms | -| `$ne` | 0.59 ms | 2.8 ms | 5.5 ms | 30 ms | -| `$like` | 0.64 ms | 3 ms | 6 ms | 32 ms | -| `$exists` | 0.61 ms | 2.8 ms | 5.7 ms | 30 ms | -| Complex (4 operators) | 0.8 ms | 3.7 ms | 7.5 ms | 40 ms | - ---- - -## Spatial Query Performance - -| Operation | 100 | 500 | 1K | 5K | -|-----------|-----|-----|-----|-----| -| createSpatialIndex | 2.6 ms | 23 ms | 61 ms | 434 ms | -| withinDistance (10 km) | 1.1 ms | 5.2 ms | 10.5 ms | 64 ms | -| withinBBox | 0.31 ms | 1 ms | 2.1 ms | 10 ms | -| nearest(10) | 0.33 ms | 1.4 ms | 2.6 ms | 3.1 ms | - -> The 10 km circle covers most of the test area, so `withinDistance` returns most records; its cost is -> dominated by the matches, not by the R-tree. - -### Spatial + Operator Combination - -| Query Type | 100 | 500 | 1K | 5K | -|------------|-----|-----|-----|-----| -| withinDistance + where (simple) | 1.1 ms | 5.5 ms | 11 ms | 64 ms | -| + where (`$gte`) | 1.1 ms | 5.7 ms | 11 ms | 66 ms | -| + where (`$in`) | 1.1 ms | 5.6 ms | 12.5 ms | 74 ms | -| + range (`$gte` + `$lte`) | 1.2 ms | 6 ms | 12.4 ms | 70 ms | -| + complex (4 operators) | 1.2 ms | 5.8 ms | 11.5 ms | 66 ms | -| + sort + limit | 1.6 ms | 8 ms | 15 ms | 88 ms | -| nearest(50) + operators + limit | 1.6 ms | 5.5 ms | 3.9 ms | 7.8 ms | - -> Since 3.2 `nearest(k)` returns the k nearest records **that match the other filters** (3.1 cut to k -> first and then filtered, which could return fewer or wrong records). With a spatial index this uses a -> growing search radius, so it stays fast as the database grows. - ---- - -## Storage & Memory - -| Records | Data Size | Peak Memory | -|---------|-----------|-------------| -| 100 | 12 KB | 4 MB | -| 1,000 | 127 KB | 6 MB | -| 10,000 | 1.3 MB | 34 MB | -| 50,000 | 6.5 MB | 124 MB | -| 100,000 | 13 MB | 238 MB | -| 500,000 | 66 MB | ~1.2 GB | - -> Peak memory of the benchmark process, including the generated test data and full result sets -> (`find(all)` of 500K records); a single key lookup needs only one shard's index. - ---- - -## SleekDB Comparison - -`tests/sleekdb_comparison.php` with SleekDB 2.15. - -### Why Choose noneDB? - -| Strength | Performance | -|----------|-------------| -| **Bulk Insert** | **12-23x faster** than SleekDB | -| **Find All** | **10-75x faster** | -| **Filter Queries** | **11-53x faster** | -| **Update Operations** | **9-48x faster** | -| **Delete Operations** | **3.5-21x faster** | -| **Count Operations** | **7-450x faster** (O(1) index lookup) | -| **Complex Queries** | **7-18x faster** | -| **Large Datasets** | Auto-sharding, 500K+ records | -| **Concurrency** | Database-level locks across processes, crash recovery | -| **Spatial Queries** | R-tree indexing (SleekDB: none) | - -### When to Consider SleekDB? - -| Scenario | SleekDB Advantage | -|----------|-------------------| -| **First key lookup of a process** | ≤2 ms vs up to 15 ms at 100K (noneDB loads one shard index first; later lookups take ~0.05 ms) | -| **Very low memory** | Lower RAM usage | - -> SleekDB stores each record as a separate file, so a single-record lookup is one file read while bulk -> operations have to open every file. - -### Architectural Differences - -| Feature | SleekDB | noneDB | -|---------|---------|--------| -| **Storage** | One JSON file per record | JSONL + byte-offset index | -| **ID Access** | Direct file read (O(1)) | Index lookup + seek | -| **Bulk Read** | Traverse all files | Single file read per shard | -| **Sharding** | None | Automatic (10K+) | -| **Cache** | Per-query | Static cross-instance, validated per write | -| **Indexing** | None | Byte-offset (.jidx), field indexes | -| **Spatial Index** | None | R-tree | - -### Detailed Benchmark Results - -| Operation | 100 noneDB | 100 SleekDB | 1K noneDB | 1K SleekDB | 10K noneDB | 10K SleekDB | 50K noneDB | 50K SleekDB | 100K noneDB | 100K SleekDB | -|-----------|-----|-----|-----|-----|-----|-----|-----|-----|-----|-----| -| Bulk Insert | 2 ms | 21 ms | 7 ms | 175 ms | 120 ms | 1.97 s | 714 ms | 11.94 s | 1.25 s | 24.35 s | -| Find All | <1 ms | 5 ms | 3 ms | 37 ms | 32 ms | 419 ms | 181 ms | 2.28 s | 372 ms | 27.98 s | -| Find by Key (cold) | <1 ms | <1 ms | <1 ms | <1 ms | 3 ms | <1 ms | 4 ms | <1 ms | 15 ms | 2 ms | -| Find with Filter | <1 ms | 4 ms | 3 ms | 47 ms | 32 ms | 411 ms | 174 ms | 2.41 s | 697 ms | 36.73 s | -| Count | <1 ms | <1 ms | <1 ms | <1 ms | <1 ms | 4 ms | <1 ms | 17 ms | <1 ms | 34 ms | -| Update | <1 ms | 7 ms | 5 ms | 73 ms | 41 ms | 974 ms | 229 ms | 4.89 s | 444 ms | 21.47 s | -| Delete | 2 ms | 6 ms | 9 ms | 49 ms | 81 ms | 745 ms | 434 ms | 3.65 s | 937 ms | 19.72 s | -| Complex Query | <1 ms | 5 ms | 4 ms | 39 ms | 47 ms | 464 ms | 173 ms | 2.33 s | 372 ms | 6.52 s | - -### Performance Ratio (noneDB vs SleekDB) - -| Operation | 100 | 1K | 10K | 50K | 100K | -|-----------|------|------|------|------|------| -| Bulk Insert | **11.6x** | **23.4x** | **16.4x** | **16.7x** | **19.5x** | -| Find All | **10x** | **10.6x** | **13.2x** | **12.6x** | **75.2x** | -| Find by Key (cold) | 0.27x | 0.09x | 0.01x | 0.01x | 0.12x | -| Find with Filter | **11.1x** | **13.4x** | **12.9x** | **13.9x** | **52.7x** | -| Count | **7.2x** | **10.7x** | **64.5x** | **283.4x** | **447.9x** | -| Update | **9.3x** | **14.9x** | **23.6x** | **21.3x** | **48.3x** | -| Delete | **3.5x** | **5.4x** | **9.2x** | **8.4x** | **21x** | -| Complex Query | **6.8x** | **11.1x** | **9.9x** | **13.5x** | **17.5x** | - -> Find by Key (cold) is the first lookup after the data was written and the caches were cleared. Warm -> lookups take ~0.05 ms (see [Key Lookup](#key-lookup)). - ---- - -## Running Benchmarks - -Every script uses its own temporary directory and key; none of them touches your databases. - -```bash -# Performance benchmark (default sizes 100 ... 500K; pass your own as a comma-separated list) -php tests/performance_benchmark.php -php tests/performance_benchmark.php 100,1000,10000 - -# Single-operation latency; optionally compare another version of noneDB.php -php tests/operation_benchmark.php -php tests/operation_benchmark.php /path/to/other/noneDB.php 5 # best of 5 - -# Comparison operators + spatial combinations -php tests/operator_benchmark.php - -# Spatial benchmark -php tests/spatial_benchmark.php - -# SleekDB comparison (requires the dev dependency rakibtg/sleekdb) -php tests/sleekdb_comparison.php -php tests/sleekdb_comparison.php 100,1000 -``` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 7900842..d5f26dc 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -1,537 +1,399 @@ -# noneDB Configuration Reference +# noneDB Configuration -Complete configuration options and security best practices. - -**Version:** 3.2.0 +Every setting of noneDB 4.0.0, the files it keeps, and how to size its cache. --- -## Table of Contents +## Contents -1. [Configuration Methods](#configuration-methods) -2. [Configuration Options](#configuration-options) -3. [Security Best Practices](#security-best-practices) -4. [Development Mode](#development-mode) -5. [Sharding Configuration](#sharding-configuration) -6. [Performance Tuning](#performance-tuning) +1. [Where the settings come from](#where-the-settings-come-from) +2. [Settings](#settings) +3. [The secret key](#the-secret-key) +4. [Files and permissions](#files-and-permissions) +5. [Development mode](#development-mode) +6. [Sharding](#sharding) +7. [Durability](#durability) +8. [Indexes and the cache](#indexes-and-the-cache) +9. [Tuning](#tuning) +10. [Example configurations](#example-configurations) +11. [Troubleshooting](#troubleshooting) --- -## Configuration Methods - -### 1. Config File (Recommended) +## Where the settings come from -Create a `.nonedb` file in your project root: +### A `.nonedb` file ```json { - "secretKey": "YOUR_SECURE_RANDOM_STRING_HERE", + "secretKey": "a random secret of your own", "dbDir": "./db/", - "autoCreateDB": true, - "shardingEnabled": true, - "shardSize": 10000, - "autoMigrate": true, - "autoCompactThreshold": 0.3, - "lockTimeout": 5, - "lockRetryDelay": 10000 + "autoCreateDB": false } ``` -**Setup:** - -```bash -cp .nonedb.example .nonedb -# Edit .nonedb with your settings -``` - -**Usage:** - ```php -// Automatically reads .nonedb from project root -$db = new noneDB(); +$db = new noneDB(); // finds and reads .nonedb ``` -### 2. Programmatic Configuration - -```php -$db = new noneDB([ - 'secretKey' => getenv('NONEDB_SECRET'), // your own random key (example placeholders are rejected) - 'dbDir' => '/path/to/db/', - 'autoCreateDB' => true, - 'shardingEnabled' => true, - 'shardSize' => 10000 -]); -``` +The file is looked for, in this order, and the first one found serves the whole process: +1. the directory of the running script (`$_SERVER['SCRIPT_FILENAME']`); +2. the directory of `noneDB.php`; +3. the Composer project root (when noneDB is installed under `vendor/`); +4. the working directory. -### 3. Mixed Configuration +Keep it **outside the web root**: it holds the secret key. A template is `.nonedb.example` +(`noneDB::getConfigTemplate()` gives its path). -A config array is merged over the `.nonedb` file: values in the array win, everything else comes from the -file. When the array contains both `secretKey` and `dbDir`, the file is optional. +### An array ```php -// secretKey (and the other settings) from .nonedb, dbDir overridden $db = new noneDB([ - 'dbDir' => '/custom/path/' + 'secretKey' => getenv('NONEDB_SECRET'), + 'dbDir' => __DIR__ . '/data/', + 'shardSize' => 50000, ]); ``` -### Where `.nonedb` is looked for - -1. The directory of the running script (`$_SERVER['SCRIPT_FILENAME']`) -2. The directory of `noneDB.php` -3. The Composer project root (when noneDB is installed under `vendor/`) -4. The current working directory +An array is merged over the `.nonedb` file: its values win, the rest comes from the file. An array with both +`secretKey` and `dbDir` needs no file. -The first file found is used for the whole process. Keep it **outside the web root**: it contains the -secret key. +### `dbDir` -### Relative `dbDir` - -- In `.nonedb`, a relative `dbDir` (`./db/`, `../data/` or `db/`) is relative to **the `.nonedb` file's - directory**. Web pages (`public/index.php`) and CLI or cron scripts (`bin/job.php`) therefore use the - same databases. -- In a config array, a relative `dbDir` is relative to the entry script's directory. Absolute paths are - recommended. -- Before 3.2, `./db/` in the file was relative to the running script and `db/` to the working directory, - which split data between directories. If such an old directory exists and the new one doesn't, it is - still used and an `E_USER_DEPRECATED` notice asks you to move it. +- A relative `dbDir` in `.nonedb` is relative to the **`.nonedb` file's directory**, so web pages and CLI or + cron scripts find the same databases. In an array it is relative to the entry script's directory; absolute + paths are clearer. +- `dbDir` is used by its **canonical path**: symbolic links and `.`/`..` are resolved when the instance is + configured (for a directory that doesn't exist yet, its nearest existing parent). Two spellings of one + directory are one store, with one lock per file in a process. +- noneDB creates it when it is missing, with an `.htaccess` that denies web access and an empty `index.html`. --- -## Configuration Options - -### Core Settings - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `secretKey` | string | *required* | Secret used to derive the database file names. Must be a non-empty string and not an example placeholder, otherwise the constructor throws `InvalidArgumentException`. Use 32+ random characters. | -| `dbDir` | string | *required* (dev mode: `db/` in the project root) | Directory for database files. Created automatically (with an `.htaccess` that denies web access and an empty `index.html`). See *Relative `dbDir`* above. | -| `autoCreateDB` | bool | `true` | Let writes (`insert`, `createFieldIndex`, ...) create missing databases. Reads never create anything. Set `false` in production. | - -### Sharding Settings - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `shardingEnabled` | bool | `true` | Enable auto-sharding for large datasets. | -| `shardSize` | int | `10000` | Records per shard. Recommended: 10,000-100,000. | -| `autoMigrate` | bool | `true` | Auto-migrate to sharded format when threshold reached. | - -### Compaction Settings - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `autoCompactThreshold` | float | `0.3` | Compact a data file when deleted/superseded records exceed this share of its records. Must be > 0. | - -### Lock Settings - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `lockTimeout` | int | `5` | Seconds to wait for a database lock before throwing `RuntimeException`. `0` waits forever. | -| `lockRetryDelay` | int | `10000` | Microseconds between lock attempts (10ms default). Must be > 0. | - -### Field Indexing (v3.0+) - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `fieldIndexEnabled` | bool | `true` | Use field indexes in queries. Indexes are always kept up to date, so turning this back on is safe. | -| `spatialIndexEnabled` | bool | `true` | Allow creating and using spatial indexes. Existing ones are always kept up to date. | - -### Write Buffer - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `bufferEnabled` | bool | `false` | Collect inserts in a buffer file and add them in batches. Off by default since 3.2: direct inserts are fast and immediately visible. Buffered records are flushed before every read and write. | +## Settings + +Values are checked when the instance is created; an invalid value throws `InvalidArgumentException`. So does a +key that is not one of the settings below (the message names it, suggests the setting it was probably meant to be, +and lists them), in the array and in a `.nonedb` file, and a setting given as `null`. Boolean settings accept `true`/`false`, `1`/`0` and `"1"`/`"0"`/ +`"true"`/`"false"` (any case), nothing else. Numeric settings take numbers or numeric strings; booleans, +arrays, `"10abc"` and NAN are refused. + +| Setting | Type | Default | Meaning | +|---|---|---|---| +| `secretKey` | string | required | The key of your databases ([below](#the-secret-key)). A non-empty string that is not a published placeholder. | +| `dbDir` | string | required (dev mode: `db/` in the project root) | The store directory. | +| `autoCreateDB` | bool | `true` | Writes (`insert()`, `createFieldIndex()`, ...) create a missing database. Reads never create anything; with `false`, `find()` on a missing database returns `false` instead of `[]`. `false` is the safer choice in production. | +| `shardingEnabled` | bool | `true` | A database switches to the sharded layout once it has assigned `shardSize` keys. With `false` it stays in the single layout until `convertToSharded()`. | +| `shardSize` | int ≥ 1 | `10000` | Keys per shard of databases **created from now on**; each database keeps the size it was created with. | +| `autoCompactThreshold` | float > 0 | `0.3` | Compact a segment when its deleted and superseded lines exceed this share of its lines. | +| `lockTimeout` | float ≥ 0 | `5` | Seconds to wait for a lock before `LOCK_FAILED`; fractions work, `0` waits forever. | +| `lockRetryDelay` | int ≥ 1 | `10000` | Microseconds between lock attempts. | +| `bufferEnabled` | bool | `false` | Collect inserts in a buffer file and add them in batches (API.md "Write buffer"). | +| `fieldIndexEnabled` | bool | `true` | Use field indexes in queries. Indexes are kept up to date either way. | +| `spatialIndexEnabled` | bool | `true` | Allow creating and using spatial indexes. | +| `durability` | string | `'performance'` | `'performance'` or `'durable'` (Linux only; elsewhere `UNSUPPORTED_DURABILITY` at construction). [DURABILITY.md](DURABILITY.md). | +| `cacheMemoryLimit` | int ≥ 0 \| `'auto'` | `'auto'` | Process memory (bytes) above which cached indexes are dropped; `'auto'`: half of `memory_limit` (none when it is `-1`); `0`: no limit ([below](#the-cache-cachememorylimit)). | +| `filePermissions` | int \| octal string | `0640` | Permissions of every file noneDB creates in `dbDir`, whatever the umask. The owner needs read and write; the world may not write (a group may). | +| `directoryPermissions` | int \| octal string | `0750` | The same for directories; the owner needs read, write and search. | + +`filePermissions` and `directoryPermissions` take an integer (`0640` in PHP) or an octal string (`"0640"`, +`"640"`, the form a JSON file needs). They apply to files created from now on; existing files keep theirs. + +The key maintenance methods (`migrateKey()`, `registerKey()`, `repairStore()`) take `['durability' => ...]` as +an option. --- -## Security Best Practices - -### 1. What the Secret Key Does (and doesn't) - -- Database files are named `{hash}-{name}.nonedb`, where `hash` = PBKDF2(name, secretKey). Without the - key the file names can't be guessed, e.g. by someone probing a web-accessible directory. -- The key does **not** encrypt anything: records are stored as plain JSON lines. Protect the directory - (outside the web root, file permissions) and the `.nonedb` file. -- A wrong key simply doesn't see the databases of another key. -- Anyone who can list the directory sees `{hash}-{name}` and can test guesses for the key offline - (PBKDF2-SHA256, 1000 rounds). A short or guessable key can be found this way: use 32+ random characters. -- To change the key of existing databases (for example away from the old example placeholder): - -```php -noneDB::migrateKey('/var/data/nonedb', $oldKey, $newKey); // renames every file of every database -``` - -### 2. Secret Key Handling - -```php -// BAD: Hardcoded secret -$db = new noneDB(['secretKey' => 'mysecret']); - -// GOOD: Environment variable (if it isn't set, getenv() returns false and the constructor throws) -$db = new noneDB(['secretKey' => getenv('NONEDB_SECRET')]); +## The secret key -// GOOD: Config file (not in git) -// Use .nonedb file and add to .gitignore -``` - -**Generate secure key:** +- A database lives in `{dbDir}/{locator}.ndb4/`, where the locator is an HMAC of the database name with the key + ([STORAGE_FORMAT.md](STORAGE_FORMAT.md) §1). Without the key the directory names can't be guessed or linked + to the names. +- The key does **not** encrypt anything: records are plain JSON lines. Protect `dbDir` and `.nonedb` with file + permissions, and use disk encryption where the data needs it. +- The store records which keys may open it (the `STORE` file holds an HMAC of each key, never the key). The + first key that writes to an empty directory creates the store; another key gets `KEY_NOT_REGISTERED` until + `noneDB::registerKey()` adds it. A wrong key never sees an empty database where data exists. +- Generate a key with `php -r "echo bin2hex(random_bytes(32));"` (32 random bytes). Keep it out of version + control: in `.nonedb` (ignored by git) or in the environment: ```php -// PHP 7+ -$secretKey = bin2hex(random_bytes(32)); - -// Or use command line -// openssl rand -hex 32 +$db = new noneDB(['secretKey' => getenv('NONEDB_SECRET'), 'dbDir' => __DIR__ . '/data/']); ``` -### 3. Protect Config File +To change the key of existing databases, rotate it: `noneDB::migrateKey($dbDir, $oldKey, $newKey)` renames +every database directory of the old key ([OPERATIONS.md](OPERATIONS.md) §4). -Add to `.gitignore`: +--- -```gitignore -.nonedb -db/ -*.nonedb -*.nonedb.jidx +## Files and permissions + +```text +{dbDir}/ + STORE, STORE.2, STORE.LOCK the store record (two copies) and its lock + .htaccess, index.html deny web access + {locator}.ndb4/ one directory per database + FORMAT, CURRENT, LOCK identity, active generation, lock and state record + generations/{N}/ the data, the indexes, the write buffer + recovery/ recovery log and quarantine files + .nonedb-maintenance/ maintenance journal and the store's event log ``` -### 4. Protect Database Directory - -noneDB writes this `.htaccess` (and an empty `index.html`) when it creates `dbDir`. For an existing -directory, create `db/.htaccess`: - -```apache -# Deny all access -Deny from all -``` +The complete layout is [STORAGE_FORMAT.md](STORAGE_FORMAT.md). Rules for operators: +- Don't edit, move or delete anything in a store while processes use it. A `LOCK` file in particular must stay: + the locks and the state record live there. +- Anything noneDB doesn't know in `dbDir` is a *foreign entry*: databases that exist keep working, but no + database is created or answered as missing while one is there (`UNSUPPORTED_FORMAT`). Keep other files + elsewhere. +- A `{locator}.ndb4` that is a symbolic link is not a database: noneDB follows no link into a store. +- Files are `0640`, directories `0750` (`filePermissions`, `directoryPermissions`), whatever the umask. Backups + are `0600`/`0700`. -Or for Apache 2.4+: +For a web server that serves the directory anyway: ```apache Require all denied +Deny from all ``` -**Nginx:** - ```nginx -location ~ \.nonedb { - deny all; -} -``` - -### 5. Disable Auto-Create in Production - -```json -{ - "autoCreateDB": false -} +location /db/ { deny all; } ``` -This prevents accidental database creation from typos or injection. Database names are validated -(`A-Z a-z 0-9`, space, `'`, `-`); anything else throws `InvalidArgumentException`. - -### 6. Database Directory Location +`.gitignore` (the template is in the repository): -```php -// BAD: Inside web root -$db = new noneDB(['dbDir' => './public/db/']); - -// GOOD: Outside web root -$db = new noneDB(['dbDir' => '/var/data/nonedb/']); - -// GOOD: Protected directory -$db = new noneDB(['dbDir' => './storage/db/']); // with .htaccess +```gitignore +.nonedb +db/ +!.nonedb.example ``` --- -## Development Mode - -For development without config file: +## Development mode -### Enable Dev Mode +Without a `.nonedb` file, and when the array doesn't give both `secretKey` and `dbDir`, development mode fills in +what is missing: + ```php -// Option 1: Static method (recommended) -noneDB::setDevMode(true); -$db = new noneDB(); - -// Option 2: Environment variable -putenv('NONEDB_DEV_MODE=1'); -$db = new noneDB(); - -// Option 3: PHP constant -define('NONEDB_DEV_MODE', true); +noneDB::setDevMode(true); // or putenv('NONEDB_DEV_MODE=1'), or define('NONEDB_DEV_MODE', true) $db = new noneDB(); +noneDB::isDevMode(); // true ``` -### Dev Mode Defaults - -When dev mode is enabled without config: - -| Option | Dev Mode Default | -|--------|------------------| -| secretKey | `"nonedb_dev_mode_key_" . md5()` — predictable by design | -| dbDir | `db/` in the project root (next to `vendor/` when installed with Composer, otherwise next to `noneDB.php`) | -| autoCreateDB | `true` | -| shardingEnabled | `true` | +| Setting | In development mode | +|---|---| +| `secretKey` | `"nonedb_dev_mode_key_" . md5()`: predictable by design | +| `dbDir` | `db/` in the project root (next to `vendor/` with Composer, else next to `noneDB.php`) | -### Check Dev Mode Status - -```php -if (noneDB::isDevMode()) { - echo "Running in development mode"; -} -``` - -### Warning - -Dev mode should **never** be used in production: its key is predictable. Outside the CLI, noneDB raises -an `E_USER_NOTICE` once per process while dev mode is active. +Never use it in production: its key is predictable. Outside the CLI noneDB raises an `E_USER_NOTICE` once per +process while it is active. --- -## Sharding Configuration - -### When to Enable Sharding - -| Dataset Size | Recommendation | -|--------------|----------------| -| < 10K records | Disable sharding | -| 10K - 100K | Enable with default shard size (10K) | -| 100K - 500K | Enable with larger shard size (50K-100K) | -| > 500K | Enable with optimized shard size | - -### Shard Size Tuning +## Sharding -```php -// Small shards = more files, faster individual shard access -$db = new noneDB(['shardSize' => 5000]); - -// Large shards = fewer files, better bulk operations -$db = new noneDB(['shardSize' => 50000]); -``` - -**Trade-offs:** - -| Shard Size | Pros | Cons | -|------------|------|------| -| Small (5K) | Fast shard access | More files to manage | -| Medium (10K) | Balanced | Default recommendation | -| Large (50K+) | Fewer files | Slower shard operations | - -### Manual Migration Control +A database starts in the single layout (one data file). Once it has assigned `shardSize` keys it switches to +the sharded layout: shard N holds the keys `[N × shardSize, (N + 1) × shardSize)`. The switch is a generation +switch: the new layout is built and verified completely before it is published (STORAGE_FORMAT.md §5.2). ```php -// Disable auto-migration -$db = new noneDB([ - 'autoMigrate' => false -]); - -// Manually migrate when ready -$result = $db->migrate("users"); +$db = new noneDB(['shardSize' => 50000, 'shardingEnabled' => false]); +$db->insert("big", ["v" => 1]); +$db->convertToSharded("big"); // switch now: ["success" => true, "status" => "converted"] +$db->getShardInfo("big"); // ["sharded" => true, "shards" => 1, ..., "shardSize" => 50000, "nextKey" => 1] ``` -### Check Sharding Status +| Shard size | Effect | +|---|---| +| 10,000 (default) | More files; a cold lookup loads less (one shard's key index) | +| 100,000 | Fewer files; faster queries that read every shard | -```php -$info = $db->getShardInfo("users"); -// { -// "sharded": true, -// "shards": 5, -// "totalRecords": 45000, -// "shardSize": 10000 -// } -``` +Measured on 10 million records ([PERFORMANCE.md](PERFORMANCE.md)): a key lookup in a new process takes about +1.5 ms with either size; an indexed query of 1000 results 120 ms with 100 shards and 310 ms with 1000 shards in +a new process, 12 and 26 ms once its indexes are cached. --- -## Performance Tuning - -### Compaction Threshold +## Durability ```php -// Aggressive compaction (compact when 10% deleted) -$db = new noneDB(['autoCompactThreshold' => 0.1]); - -// Conservative compaction (compact when 50% deleted) -$db = new noneDB(['autoCompactThreshold' => 0.5]); +$db = new noneDB(['durability' => 'performance']); // the default ``` -**Recommendations:** +| Mode | A write that returned survives | +|---|---| +| `performance` | the process dying (SIGKILL, fatal error), as long as the operating system keeps running; a power loss is best effort | +| `durable` (Linux) | a power loss too: every sync the write needs is done before it returns, or it fails (`DURABILITY_FAILED`, rolled back; `OUTCOME_UNKNOWN` after the commit point) | -| Use Case | Threshold | -|----------|-----------| -| High delete rate | 0.2 (20%) | -| Normal operations | 0.3 (30%) - default | -| Disk space priority | 0.5 (50%) | -| Minimal disk writes | 0.7 (70%) | +Durable mode on macOS or Windows is refused at construction (`UNSUPPORTED_DURABILITY`): macOS's `fsync()` +doesn't flush the drive cache and PHP can't call `F_FULLFSYNC`; Windows can't sync a directory. Network file +systems are supported in neither mode. Every write waits for the disk in durable mode: it is slower. -### Lock Settings +The contract (commit points, sync order, the three outcomes, `getDurability()`): [DURABILITY.md](DURABILITY.md). -For high-concurrency environments: +--- -```php -// Increase timeout for slower systems -$db = new noneDB([ - 'lockTimeout' => 10, // 10 seconds - 'lockRetryDelay' => 5000 // 5ms between retries -]); -``` +## Indexes and the cache -Each database has a lock file `{hash}-{name}.nonedb.lock`. Besides the lock it holds a small state -record (change counter, pending write buffer, index journal) that every operation reads with the lock, -so cache validation and crash recovery need no extra file access. Lock files stay when a database is -dropped and their handles stay open between operations (one per database, at most 64 per process). -Don't delete them while processes use the databases; deleting them while nothing runs is harmless (the -one-time check for pre-3.2 leftovers then runs again). +### Index files -### Static Cache +Every index is a **checkpoint** of the data: it covers the data file up to an offset, and loading it replays +the complete lines after that. Inserts, updates and deletes only append to the data file. -```php -// Disable for memory-constrained environments -noneDB::disableStaticCache(); +| File (in `generations/{N}/`) | Contents | +|---|---| +| `segments/{seg}.kix` | key index: where each key's current line is | +| `indexes/{seg}.{token}.fx` | field index of a segment: the keys of each value, by type (`1`, `1.0` and `"1"` are three values) | +| `indexes/{token}.gfx` | global field index (sharded): a filter per shard of its values | +| `spatial/{seg}.{token}.rx` | R-tree of a segment's geometries | +| `spatial/{token}.grx` | global spatial index (sharded): the bounding box of each shard | +| `indexes.json` | the indexed fields | -// Re-enable -noneDB::enableStaticCache(); +- `{seg}` is `main` (single layout) or `s{N}` (shard N). `{token}` is the field name when it matches + `[a-z0-9_]{1,40}`, else a lower-case prefix and 12 hex digits of its SHA-256: any field name can be indexed, + and `city` and `City` have files of their own on every file system. +- A key-index checkpoint is written at the end of a write session once 64 lines plus 1/16 of the records (or + 1 MiB) follow it. A field or spatial index checkpoint is written by a process that uses the index once the + data after it reaches 64 KiB or a quarter of the index file, whichever is larger. A process that only writes + never loads field or spatial indexes. +- A checkpoint is trusted only for the data file it was made from (the offset, a checksum of the bytes before + it, an epoch that changes when compaction or a generation switch replaces the file). One that doesn't fit is + rebuilt from the data. Index files are never synced: they can always be rebuilt. +- **Sharded lookups:** a lookup on an indexed field reads the shards the global index says may hold the value, + and the shards whose newer lines may. A write that adds a value doesn't rewrite the global index. -// Clear cache (useful between test runs) -noneDB::clearStaticCache(); -``` +### The cache (`cacheMemoryLimit`) -### Field Index Configuration +Indexes, metadata and the bytes of data files in repeated use are cached per process and shared by all its +instances. `cacheMemoryLimit` is the memory of the process above which they are dropped, least recently used first: ```php -// Create indexes for frequently filtered fields -$db->createFieldIndex("users", "email"); -$db->createFieldIndex("users", "status"); - -// Check if index exists -$db->hasFieldIndex("users", "email"); - -// Drop unused index -$db->dropFieldIndex("users", "old_field"); +new noneDB(['cacheMemoryLimit' => 32 * 1024 * 1024]); // 32 MiB +new noneDB(['cacheMemoryLimit' => 0]); // no limit +new noneDB(['cacheMemoryLimit' => 'auto']); // the default: half of memory_limit ``` ---- +- **When:** when the last lock of the process is released and something new was cached; and **between the + segments** of an operation over many segments, once the process uses more than the limit (entries go until + it is under three quarters of it; the next drop waits until it has grown by another quarter). The cache of + one operation over a large database stays within the limit: the indexes of the segments it finished go first. + A dropped index is loaded again when it is needed. +- **Near `memory_limit`.** `memory_limit` counts what PHP holds for the process, which a large read leaves above what + the process uses. Once the process's peak has come within an eighth of `memory_limit`, the drop between segments + measures what PHP holds too, and what the process has freed goes back to the system. +- Only what is cached is dropped; what else the process holds (an answer, the caller's data) stays. A limit + below what the process needs without the cache leaves nothing cached, and every lookup loads again + (`getMetrics()`: `indexCache.evictions` and `misses` grow with every operation). +- The setting is process-wide: the last instance created with it in its configuration sets it. +- **Data bytes.** In a database of more than 16 segments, a data file in repeated use is held whole while the process + stays under half of the limit, and its records no longer cost an open and a read each. Repeated indexed queries over + many shards gain from a limit that leaves room for the shards they use; with `memory_limit = 512M` (a limit of 256 MiB, + so 128 MiB of bytes at most) on a 10M database they gain little. A process with no limit holds no data bytes, because + nothing would bound them: `cacheMemoryLimit` 0, or `'auto'` with `memory_limit = -1`. +- **It is not a memory cap.** One operation loads what it needs and can take the process above the limit while it + runs; entries go between segments and when the lock is released. PHP's `memory_limit` stays the hard limit: size + it for your largest operation, and the cache below it. +- After `noneDB::disableStaticCache()` each new instance keeps its own cache, which no limit applies to. + +**Sizes** (PHP 8.2): +- **Key index:** about 12 bytes per record (its checkpoint body). A segment in constant use is decoded to about + 80 bytes per record for faster lookups: in a database of at most 16 segments at once, in a larger one once + its lookups pay for it. A process's first operation never decodes, so a one-shot script stays light. +- **Field index:** a file of at most 64 KiB is loaded whole at its first use. A larger one is read by equality + lookups along their path only, about 4 KiB per page, and the pages are kept; it is loaded whole once lookups + pay for it. Ranges and sorting by the field load it whole (20–80 bytes per record with up to 1000 distinct values, about 190 when every value differs). +- **Global field index:** 4 to 8 bytes per value of the largest shard, per shard (a unique field over 10 million + records in shards of 10,000: 64 MiB on disk); a lookup reads six short rows. From its second operation the + rows are kept. A command-line process also builds an exact map of the shards' values (about 100 bytes per + value), within the budget, in a database of at most a million records; a web request doesn't. +- **10 million records with one indexed field:** an indexed query of 1000 results peaks at 150–210 MiB; with + PHP's default `memory_limit` of `128M` (a 64 MiB cache) it takes 0.3 s at 1000 shards and peaks below 128 MiB. + +### Memory of operations over many segments + +An operation holds the cache, one batch of the records of the segment it works on, and its answer: never the +whole database, nor a whole segment. Reads and aggregations, writes by filter, field index builds, +`rebuildIndex()`, `checkIntegrity()`, recounting shard counters, the switch to shards and the first pass of a join +go segment by segment and read a segment **10,000 records at a time** (the default shard size: a segment of that size +is read at once), so the memory does not grow with `shardSize` or with a database that is not sharded. Besides the +batch an operation keeps at most a few hundred bytes per key of that segment (its key index's arrays, offsets and +lengths, the buckets of a field index it builds); a write by filter holds a batch and its new versions. Two hold +more of the segment: `compact()` holds the bytes of its live records (the new data file), and a spatial index +build holds an entry per record with a geometry, about 512 bytes each (a segment's R-tree is built and loaded whole). +A sharded database's global field index is made in parts of at most a quarter of the cache limit. + +Shards of 100,000 records (PHP 8.2): a scan or an aggregation peaks at about 41 MiB, +`checkIntegrity()` at 43 MiB, a field index build at 49 MiB, `rebuildIndex()` at 52 MiB, `compact()` at 35 MiB. + +These peaks are without a cache, and the cache comes on top of them. On 10 million records in shards of 100,000 at +PHP's default `memory_limit` of `128M` (a 64 MiB cache), an unindexed `find()` takes 29 s and peaks at 123 MiB, a sum over +every record 32 s and 121 MiB, `createFieldIndex()` 36 s and 121 MiB. They finish with little left for what the script +itself holds. A `cacheMemoryLimit` of 32 MiB brings the field index build down to 99 MiB in the same time. + +A field whose every value differs is the most a field index costs, and a database that isn't sharded holds it in one +segment. On one segment of 300,000 records with a different value each (PHP 8.2), `createFieldIndex()` peaks at 97 MiB and +a range with a small answer at 119 MiB, within `128M`. `checkIntegrity()` takes 133 MiB and a range of 78,000 records +165 MiB, so those two need a larger `memory_limit`. Shards keep such a field small. + +What an answer holds is the caller's: `find()` of every record returns every record; `sort()` without a small +`limit()` sorts all matches; a join holds the joined records, `groupBy()` its groups. Besides the cache the +engine keeps a few KiB per segment (about 7 MiB for 1000 segments). -## Configuration Helpers - -### Check Config - -```php -// Check if config file exists -if (noneDB::configExists()) { - echo "Config file found"; -} - -// Get config template path -$template = noneDB::getConfigTemplate(); -// Returns: "/path/to/.nonedb.example" +--- -// Clear cached config -noneDB::clearConfigCache(); -``` +## Tuning -### Runtime Configuration Check +| Setting | When to change it | +|---|---| +| `autoCompactThreshold` | `0.2` for many deletes, `0.5`–`0.7` to write less often (more disk space) | +| `lockTimeout` | larger for long compactions or backups of large databases; `0` waits forever | +| `shardSize` | larger for queries that read every shard, smaller for cold lookups | +| `cacheMemoryLimit` | larger for long-running workers with large databases | -```php -$db = new noneDB(); - -// Check current settings -$db->isShardingEnabled(); // true/false -$db->getShardSize(); // 10000 -``` +Under PHP-FPM or `mod_php` the cache lives as long as one request: each request loads the index pages it needs. +In a long-running process (a CLI worker, a queue consumer) they stay loaded. For large databases in web +requests prefer queries that touch few shards: key lookups and indexed fields whose values sit in a few shards. --- -## Environment-Specific Configuration - -### Development - -```json -{ - "secretKey": "dev_secret_for_testing", - "dbDir": "./db/", - "autoCreateDB": true, - "shardingEnabled": false, - "autoCompactThreshold": 0.5 -} -``` +## Example configurations -### Staging +Development: ```json -{ - "secretKey": "staging_unique_secret", - "dbDir": "/var/data/staging/", - "autoCreateDB": true, - "shardingEnabled": true, - "shardSize": 10000 -} +{ "secretKey": "a random secret of your own", "dbDir": "./db/", "shardingEnabled": false } ``` -### Production +Production: ```json { - "secretKey": "GENERATE_UNIQUE_64_CHAR_STRING", - "dbDir": "/var/data/production/", + "secretKey": "a random secret of your own", + "dbDir": "/var/lib/app/nonedb/", "autoCreateDB": false, - "shardingEnabled": true, "shardSize": 50000, - "autoCompactThreshold": 0.3, - "lockTimeout": 10 + "lockTimeout": 10, + "durability": "durable", + "filePermissions": "0640", + "directoryPermissions": "0750" } ``` -Replace the placeholder key: noneDB refuses to start with it. - --- ## Troubleshooting -### "Config file not found" - -```php -// Check if file exists -var_dump(noneDB::configExists()); - -// Use dev mode for quick testing -noneDB::setDevMode(true); -``` - -### "Permission denied" - -```bash -# Fix directory permissions -chmod 755 db/ -chmod 644 .nonedb -``` - -### "timed out after Ns waiting for the lock" - -Another process held the database lock longer than `lockTimeout` (a long compaction or migration). - -```php -$db = new noneDB(['lockTimeout' => 30]); // or 0 to wait forever -``` - -### "'secretKey' is still the example placeholder" / "must be a non-empty string" - -Generate a key (`php -r "echo bin2hex(random_bytes(32));"`). If databases already exist under the -placeholder key, move them first with `noneDB::migrateKey($dbDir, $oldKey, $newKey)`. - -### "invalid database name" - -Only `A-Z a-z 0-9`, space, `'` and `-` are allowed. Before 3.2 other characters were removed silently, so -data written as `"my_db"` is stored under the name `"mydb"`. - -### "Database not accessible after upgrade" - -- Changed secretKey: database files are named `{hash}-{name}.nonedb` with a hash derived from the key. - Use `noneDB::migrateKey()` to change keys instead of editing the config only. -- Relative `dbDir`: it now follows the `.nonedb` file. Look for an `E_USER_DEPRECATED` notice naming the - old directory, then move the files or use an absolute path. -- Went back to 3.1 and then returned to 3.2: 3.1 doesn't maintain the lock files' state. Stop all - processes, delete the `*.nonedb.lock` files and start again. +| Message or code | Cause | What to do | +|---|---|---| +| `'secretKey' must be a non-empty string` / `is a publicly known key` | No key, or a placeholder from the documentation | Generate one; existing data under a placeholder: rotate it with `migrateKey()` | +| `KEY_NOT_REGISTERED` | The key is not registered for this store (a typo, another application's store, or a rotated key) | Use the registered key, or `registerKey()` (OPERATIONS.md §4) | +| `UNSUPPORTED_FORMAT` | `dbDir` holds files that are not noneDB 4 objects: databases of noneDB 1.x–3.x, or anything else | Migrate them (MIGRATION.md) or move them out | +| `FORMAT_DAMAGED` | A store record, FORMAT, CURRENT or generation can't be read and can't be repaired with proof | OPERATIONS.md §5 | +| `MAINTENANCE_IN_PROGRESS` | Key maintenance or a migration is running or was interrupted | Run it again to finish it (OPERATIONS.md §4) | +| `LOCK_FAILED` ("timed out ... waiting for the lock") | Another process held the lock longer than `lockTimeout` (a long write, a backup), or readers that never stop kept a writer out (no queue, PERFORMANCE.md "Known limits") | A larger `lockTimeout`; find the long holder; retry the write | +| `DURABILITY_FAILED` | A required sync failed before the commit point (durable mode): rolled back | Check the disk; `getDurability()['lastSyncError']` | +| `OUTCOME_UNKNOWN` | A failure after the commit point, or an undo that failed | Read the state again before retrying; the next access recovers | +| "could not create ... a quarantine file" | A write must move a damaged tail aside first and can't (disk full, permissions) | Free space or fix permissions; reads keep working | +| `Permission denied` | The PHP user can't read or write `dbDir` | Give the user that runs PHP (only) access: `chown -R www-data /var/lib/app/nonedb` | diff --git a/docs/DURABILITY.md b/docs/DURABILITY.md new file mode 100644 index 0000000..b41b879 --- /dev/null +++ b/docs/DURABILITY.md @@ -0,0 +1,153 @@ +# noneDB 4 Durability Contract + +**Engine:** noneDB 4.0.0 · **Status:** normative + +This document defines: +- what a successful, failed or uncertain write means; +- which syncs each mode performs, and in which order; +- what `getDurability()` reports. + +File formats and state machines are in [STORAGE_FORMAT.md](STORAGE_FORMAT.md). + +--- + +## 1. Modes + +| Mode | Setting | Guarantee | +|---|---|---| +| Performance (default) | `'durability' => 'performance'` | **Process crash:** a write that returned success stays applied after the process dies (SIGKILL, fatal error), as long as the operating system keeps running. Recovery is deterministic: complete lines roll forward, unterminated lines are never records, keys are never reused, and indexes are rebuilt from the data when they don't fit it. **Power loss:** no guarantee. Recent writes may be lost; damage is detected and recovered on a best-effort basis, and `checkIntegrity()` reports what remains. | +| Durable | `'durability' => 'durable'` | Everything in performance mode, **plus power loss:** a write that returned success is on stable storage, provided the file system and device honour `fsync`. Every sync the operation needs (§3) is performed. If one fails, the operation does not report success (§2); a cleanup sync after the commit point is the exception (§6). | + +Durable mode is never silently downgraded: + +| Platform | Durable mode | +|---|---| +| Linux, local file system (ext4, xfs, btrfs) | Supported. CI tests the real sync paths. | +| macOS | **Refused** with `UNSUPPORTED_DURABILITY` at construction. PHP's `fsync()` does not flush the drive cache, and PHP cannot call `F_FULLFSYNC`. | +| Windows | **Refused** with `UNSUPPORTED_DURABILITY`. A directory cannot be synced, so a new file or a rename is not durable. | +| Network file systems (NFS, SMB), stream wrappers | Not supported in either mode: `flock` and atomic rename are not reliable there. | + +A wrong `durability` value is an `InvalidArgumentException`, and so is a misspelled setting name +(CONFIGURATION.md "Settings"): a typo never means performance mode. + +The key maintenance methods (`migrateKey()`, `registerKey()`, `repairStore()`) take the same choice as an +option, `['durability' => 'durable']`, with the same validation (default `'performance'`). + +## 2. Outcomes + +Every write operation has one **commit point** (§3). It ends in exactly one of three outcomes: + +| Outcome | When | Caller sees | +|---|---|---| +| **committed** | The commit point was reached and every sync the mode requires succeeded, cleanup after the commit point aside (§6). | Success. | +| **rolled back** | The failure happened **before** the commit point and the undo completed. In durable mode, "completed" includes the sync of the undo itself (the `ftruncate` and its `fsync`). In performance mode, the undo is guaranteed at the process and operating-system level only. | `DURABILITY_FAILED` when a sync failed; otherwise the original error. Nothing of the operation is applied. | +| **unknown** | The failure happened **after** the commit point (for example, the directory sync after a rename failed), or the undo or its sync could not be completed. | `OUTCOME_UNKNOWN`. The next access recovers the database and decides. The caller must read the state again before retrying. | + +- An operation never returns success in the rolled-back or unknown cases. +- A rollback is never claimed when its sync did not succeed. +- `getDurability()['lastOperation']['outcome']` is `committed`, `rolled_back` or `unknown`. +- An *operation* is one public write call (`insert`, `update`, `delete`, `createDB`, `dropDB`, `flush`, + `compact`, `convertToSharded`, `restore`, the index methods, the query builder's `update`/`delete`/`removeFields`, and + `recovery` for recovery work a read had to do). A call that wrote nothing (an update that matched no + record) is not one and leaves the report as it was. Work deferred until an operation committed + (auto-compaction, a buffer flush, a migration to shards) is part of no operation's outcome: its failure + is logged (`deferred-failed`) and retried. +- An unknown outcome also sets `J` in the state record: the next session recovers the whole database. + +## 3. Commit points and sync order + +"Sync file" = `fsync` of the file. "Sync dir" = `fsync` of its directory. Performance mode performs none of them. Durable mode performs all of them, in the order listed; the operation's commit point is marked **⇒**. + +| Operation | Steps (durable mode) | +|---|---| +| Append to an existing data or buffer file | Write the lines and check the byte count and `fflush` → sync file **⇒ committed** | +| Append that creates the file (new shard, new buffer) | Create and write → sync file → sync dir **⇒ committed** | +| Set a state-record flag (`b`, `c`, `d`, `m`, `g`, `j`, `J`) or a new epoch | Write the 128-byte record → sync file. This always happens **before** the data change it protects. Clearing a flag needs no sync. | +| Atomic replacement (`FORMAT`, `CURRENT`, `metadata`, `shards.json`, `indexes.json`, STORE copies, journal) | Write the temp file → sync file → rename **⇒ committed in the process** → sync dir **⇒ durable**. A failing dir sync after the rename is *unknown*. | +| New directory | `mkdir` → sync its parent directory, for every directory created (a generation, `segments/`, `buffer/`, `recovery/`, dbDir itself) | +| Sharded write session | Set `j` (synced) → an insert reserves keys in `shards.json` (atomic replacement) → append (above) **⇒** → a delete writes `shards.json` → clear `j`. An update writes no `shards.json`. | +| Segment compaction (STORAGE_FORMAT §6.2) | New epoch (synced) → remove the checkpoint → temp data file, sync file → rename **⇒** → sync dir → fresh checkpoint | +| Generation switch (§5.2) | PREPARE: every file synced, every new directory synced into its parent → INTENT record (synced) → CURRENT replacement **⇒** → CLEANUP → sync `generations/` → DONE. `g` is cleared only after that sync; if it fails, `g` stays and the next session finishes the cleanup. A failure during PREPARE removes the new generation. | +| Database creation (§8.1) | `c` record (synced) → sync dbDir (the directory's entry) → generation 1, `CURRENT` (each file and new directory synced) → `FORMAT` (atomic replacement) **⇒** when its dir sync succeeded → clear `c` | +| Database drop (§8.2) | `d` and a new epoch (one record, synced) → remove `FORMAT` → sync dir **⇒** → remove the rest → sync dir → clear `d`. `d` stays when that last sync fails (the next session finishes the drop). If the `d` record can't be synced, `d` is withdrawn by a synced record (*rolled back*); if that fails too, *unknown*. | +| Quarantine before a cut (§7.2) | Copy → sync file → rename → sync dir → read back and verify, all **before** the source is cut. A failed sync: nothing is cut, the write fails (*rolled back*). | +| Backup (§10.2) | The database is not written. Every copied file synced, every new directory synced into its parent → `manifest.json` (atomic replacement) **⇒** | +| Restore (§10.3) | PREPARE: every file of X with the sync of its kind (data and buffer files synced in both modes, as replaced data files) → a present database: the generation-switch row; an absent one: the creation row; a damaged one: the state record with a new epoch (synced) → `CURRENT` replacement **⇒** → `FORMAT` replacement when it was invalid | +| Maintenance (§9) | `m` + epoch records (synced) → work (a rotation's renames, then sync dbDir) → journal `COMMITTED` (atomic replacement) **⇒** → STORE copies → clear `m` (synced) → remove the journal → sync its directory. A failure before the commit keeps the journal open: the store stays closed until the maintenance is run again (it continues) or aborted. Only a rotation whose verification fails undoes its renames and closes the journal. A failure after the commit: `MAINTENANCE_IN_PROGRESS` (run it again); a failed dir sync of the journal's commit or removal, or of a STORE copy: *unknown*. | + +Checkpoints and secondary indexes are **derived files**: +- they are written by atomic replacement but never synced, in either mode; +- they are validated against the data when loaded (`s`, `h`, `e`), so losing or corrupting one only costs a rebuild. + +**Performance mode** syncs nothing for its own guarantee, with three exceptions that cost one sync per rare +operation and keep a power loss from taking a whole data file or the list of shards: +- a data file that is **replaced** (compaction, the shard files of a generation switch) is synced before + its rename; if that sync fails the replacement is not made (`DURABILITY_FAILED`, *rolled back*); its + directory is synced after the rename, and a failure there is not an error in performance mode; +- a **quarantine copy** is synced; a failed sync is reported in the recovery log (`powerLossDurable`), the + cut goes ahead; +- `shards.json` is synced, with its directory, when compaction removes empty shards and when it is rebuilt. +`required` stays empty in performance mode: these syncs make no claim. + +## 4. Undo on failure + +- **Append failure** (an `fwrite` that makes no progress, failed `fflush`, failed sync file, failed dir sync of a created file): the appended bytes are cut back to the recorded size (`undoAppend`). A file the append created is cut to zero bytes, never removed, so the cut needs no directory sync. + - In durable mode, the truncation is synced as well. + - If the cut or its sync fails, the outcome is *unknown*: `OUTCOME_UNKNOWN`, and `J` is set so the next session recovers. +- **Rollback scope:** an exception that leaves the outermost exclusive `withLock()` cuts every file the scope appended to back to its recorded size. + - The same rules apply: synced cuts in durable mode, *unknown* when a cut fails. +- **Replacement failure before the rename:** the temp file is removed and the target is unchanged. The outcome is *rolled back*. +- **Replacement failure after the rename:** the outcome is *unknown*. In-process state treats the new file as current; the next process decides by reading the disk. + +## 5. Process crash recovery (both modes) + +Recovery runs at the next lock acquisition. Data is not returned before it completes. + +| Condition | Handling | +|---|---| +| Unterminated tail | Ignored by readers; quarantined and cut by the next writer. | +| Complete lines past a checkpoint | Replayed (roll forward). | +| `j` / `J` | Shard counters recounted from the data. | +| `g` | Generation recovery (STORAGE_FORMAT §5.2). | +| `c` / `d` | Creation is undone; a drop is finished (§8.3). | +| `m` or an open maintenance journal | Closed until the maintenance is run again. | + +**What is not atomic:** a multi-record API call (for example an insert of 1000 records) is **not** a transaction under SIGKILL. A prefix of it may be applied. +- In the sharded layout keys are reserved before data, so no key is reused. In the single layout the key of a line + that never became a record (torn, or rolled back) goes to the next insert; no reader ever saw it. +- Counters and indexes are recomputed from what was applied. +- An exception (as opposed to a kill) rolls the whole scope back (§4). + +## 6. `getDurability()` + + +```php +[ + 'mode' => 'performance' | 'durable', // this instance + 'platform' => ['os' => 'Linux', 'durableSupported' => true, 'notes' => [...]], + 'lastOperation' => null | ['op' => 'insert', 'db' => 'users', + 'outcome' => 'committed' | 'rolled_back' | 'unknown', + 'required' => ['file', 'dir'], 'completed' => ['file', 'dir'], 'reason' => null], + 'lastSyncError' => null | ['path' => ..., 'op' => 'file' | 'dir', 'time' => ...], + 'syncs' => ['file' => int, 'dir' => int, 'failed' => int], +] +``` + +- `lastOperation` is the last write operation of the **process** (§2), whichever instance ran it; `null` + before the first one. +- `required` lists the kinds of sync the last operation needed in this mode. It is empty in performance mode. +- `completed` lists the kinds whose every sync up to the commit succeeded. A cleanup sync after the commit point that + fails (`generations/` after a switch, the last one of a drop) changes neither `completed` nor the outcome: the call + returns success, `g` or `d` stays for the next session, and `lastSyncError` shows it. +- `outcome` follows §2; `reason` is the exception's message when the operation failed. +- `platform.notes` says why durable mode is not supported where it isn't (empty on Linux). +- `syncs` and `lastSyncError` count every sync of the process, the ones performance mode does too. +- Paths are relative to `dbDir`. No key material appears. + +## 7. Tests (summary) + +- **Every row of §3:** the failure of each sync is injected through the sync hook. The test checks the outcome, the exception code and the record set after the database is reopened, compared with the independent model (`tests/Crash/DurabilityContractTest.php`). +- **The crash and fault matrix** (`tests/Crash/FaultMatrixTest.php`): every write operation killed at every step and hit by every kind of I/O error; the reopened record set must be one of the states the operation may leave. +- **Every state-machine step in STORAGE_FORMAT:** SIGKILL before and after the step. The reopened database must be one of the two consistent states. +- **Durable mode on Linux:** tested in CI and in docker. On macOS and Windows, the `UNSUPPORTED_DURABILITY` refusal is tested. +- **Power loss:** simulated by placing the artefacts it can leave (zero-length files, NUL regions, missing renames). Each is checked to be recovered or reported. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 0000000..6f1aff8 --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,393 @@ +# Migrating to noneDB 4 + +**Tool:** `noneDB-migrate.php` · **Target:** storage format 4 (STORAGE_FORMAT.md) · **Status:** normative + +noneDB 4 does not read the files of earlier versions. `noneDB-migrate.php`, a separate tool shipped next to +`noneDB.php`, converts them: it reads the old files of one secret key, builds noneDB 4 databases beside +them, verifies them, and publishes them in one committed step. It never changes the old files while it +works, keeps a verified backup of them, and can put them back. + +The tool knows the old formats; the engine does not. Everything the tool writes is noneDB 4 as +STORAGE_FORMAT.md defines it, and it is verified with the engine before it becomes visible. + +--- + +## 1. Guarantees + +- **The live data of each version is kept as that version defines it** (§4): the records it returns, with + their keys. Records a version had deleted stay deleted, even where its data file still holds them. +- **Keys are never reused:** the next key of a migrated database is at least every key the old files ever + used (§5). +- **Fail closed:** when the files do not determine the live data (a lost or damaged index, an interrupted + flush of unknown state, a shard size that does not fit the files, …), the database is reported and + nothing is migrated for it, unless an option states the decision explicitly (§6). +- **Nothing is visible before it is verified:** each database is built in a staging area, checked with the + engine against the expected records, and only then published. A published database answers + `MAINTENANCE_IN_PROGRESS` until the whole run commits. +- **One commit point per run:** the maintenance journal (STORAGE_FORMAT.md §9) records `COMMITTED`. Before + it, a run can be resumed or rolled back (`--abort`); after it, only completed. +- **The old files are never deleted by a migration:** they are copied to a verified backup first and, after + the commit, moved into `.nonedb-retained/{runId}/`. Only `--cleanup` removes them, after verifying the + backup again. + +## 2. Usage + +```bash +# Check only: what would be migrated, what stops. Writes nothing. +NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --dry-run + +# Migrate +NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --backup-dir=/var/backups/nonedb --confirm-stopped + +# Also compare every migrated record with the backup after the commit +NONEDB_SECRET=... php noneDB-migrate.php --dir=./db --backup-dir=/var/backups/nonedb --confirm-stopped --verify +``` + +| Option | Meaning | +|---|---| +| `--dir=DIR` | The directory that holds the old databases (the engine's `dbDir`). | +| `--config=FILE` | A `.nonedb` file to take `secretKey` (and `dbDir` when `--dir` is absent) from. | +| `--dry-run` | Pre-check only (§7.1). Nothing is written anywhere. | +| `--backup-dir=DIR` | Where the backup of the old files is written (§8). Required to migrate, with `--restore` and with `--cleanup`. Must be outside `--dir`. | +| `--confirm-stopped` | Required to migrate: states that no application uses the directory. Versions before 3.2 take no database lock, so this cannot be checked for them (§7.2). | +| `--db=NAME[,NAME…]` | Migrate only these databases. The key is registered only by the run that leaves no old database of the key behind (§9). | +| `--verify` | After the commit, read every migrated database again with a new engine instance and compare every record with the backup (§7.8). Without it, the final check is `checkIntegrity()` and the record counts. | +| `--rebuild-from-data` | Consent for databases whose index is lost or damaged: their live set is rebuilt from the data file (§6.2). The report says that an index was replaced; it can't say which records come back. | +| `--flushing=import\|skip` | Decision for interrupted flushes of versions 2.3–3.1 (§6.3). | +| `--shard-size=N` | Shard size of databases that were not sharded (default: the `shardSize` of the config file, else 10000). Sharded databases keep their own. | +| `--abort` | Roll back an interrupted run that has not committed (§10). | +| `--restore=RUNID` | Put the old files of a run back and remove its noneDB 4 databases (§11). | +| `--cleanup=RUNID` | Remove `.nonedb-retained/RUNID/` after verifying it against the backup again (§12). Needs the key. | +| `--json` | Print the result as one JSON object (§13). | + +**Settings.** noneDB 4 refuses a setting it doesn't know, in a config array and in a `.nonedb` file +(CONFIGURATION.md "Settings"). The `.nonedb` of noneDB 3.0–3.2 may hold `autoMigrate`, which noneDB 4 doesn't +have (`shardingEnabled` alone decides whether a database switches to shards by itself): remove that line before +the run. Otherwise the pre-check stops with `FAILED`, naming the setting and the file, and nothing is written. That +holds for a `.nonedb` the engine finds itself; a file given with `--config` elsewhere is only read for its key and `dbDir`. + +**Keys.** The secret key is read from the environment or a config file, never from the command line: + +- `NONEDB_SECRET` (or the config file's `secretKey`): the key the migrated databases are registered with. +- `NONEDB_SOURCE_SECRET`: the key the old files were written with, when it differs. Releases before 3.0 shipped with the key `nonedb_123`, and noneDB 4 refuses that key (and every documented placeholder) for a store. Such databases are migrated with `NONEDB_SOURCE_SECRET=nonedb_123` and a real `NONEDB_SECRET`. A dev-mode source key is `nonedb_dev_mode_key_` followed by the MD5 of the directory the old `noneDB.php` was in. + +The tool loads `noneDB.php` from its own directory. + +## 3. Sources + +| Source | Recognised by | +|---|---| +| 1.4, 2.0–2.2 | `{h}-{name}.nonedb` whose content is one JSON object `{"data":…}`; sharded: `{h}-{name}.nonedb.meta` and `{h}-{name}_s{N}.nonedb` of the same form. | +| 2.3 | As 2.x, plus `.nonedb.idx`, write buffers `.nonedb.buffer`, `_s{N}.nonedb.buffer`, `.buffer.flushing`. | +| 3.0, 3.1 | JSONL data files, `.jidx` key indexes `"v":3` without `s`; 3.1 also `.sidx` spatial files. | +| 3.2 | A lock file `{h}-{name}.nonedb.lock` whose record has no epoch, `.jidx` `"v":3` with `s`, tombstone lines, one buffer per database with `.flushing.{K}`. | +| 3.3-dev | A lock record `{gen}:{epoch}`, `.jidx` `"v":4` checkpoints, `.fx`/`.rx` index files. | + +`{h}` is `hash_pbkdf2('sha256', name, key, 1000, 20)` (all versions). The tool lists `--dir` and groups the files of the +source key: a file belongs to database `name` when its name starts with `{h}-{name}.` or `{h}-{name}_s{N}.` +with `{h}` computed from `name` and the source key, or (spatial and quarantine files of 3.1–3.3-dev) with +`{h}.` or `{h}_s{N}.`. Names are what the old versions stored: `[A-Za-z0-9' -]`. + +- Files of other keys are not touched and are listed in the report. As long as they stay in the directory, + the engine refuses to answer "does not exist" for any database and refuses to create one (STORAGE_FORMAT.md + §3.4); run the tool with each key, or move them away. +- A 3.0/3.1 `.nonedb_hash_cache` can name files with the hash of another key. The tool does not follow it: + such files belong to that other key. +- Files of the store itself (`STORE*`, `*.ndb4/`, `.nonedb-maintenance/`, `.nonedb-retained/`, `.htaccess`, + `index.html`, `.gitkeep`, `.gitignore`, `.DS_Store`) are not sources. + +A **database** of the source key is a name with a data file `{h}-{name}.nonedb`, a meta `{h}-{name}.nonedb.meta`, +or a write buffer holding records. Files of the key that belong to no database (an info or lock file left by +a drop, shard files without a meta, …) are **leftovers**: §6.4. + +The version is decided **per segment** (a data file with its index), not per directory: versions from 3.0 on +convert files lazily, so one database can hold segments written by different versions. §4 names the rule +for each kind of segment. + +## 4. Live set + +Every database is read without the engine. Layout: **sharded** when `{h}-{name}.nonedb.meta` exists (every +version); its segments are the shards the meta lists, in its order; shard files it does not list are +leftovers (§6.4). Otherwise **single**: the segment `{h}-{name}.nonedb`. The live set is the union of the +segments' live sets plus the buffered records (§4.3). + +### 4.1 Segment kinds + +| Kind | How it is recognised | Live set | Key of a record | +|---|---|---|---| +| **W** — whole JSON (1.4–2.3, and segments no later version touched) | The file decodes as one JSON object whose only member is `data`, an array or an object | Every entry of `data` that is an array (a JSON object or list). `null` and other values are not records (they are reported); they count for the next key. | Single: the entry's integer index. Shard N: `N·S + index` (S: §4.2). | +| **J30** — JSONL with a 3.0/3.1 index | `.jidx` with `"v":3` and an `o` map, no `s` | The keys of `o`. The record is the line at `[offset, length]`, which must decode to an object whose `key` equals the map key. An entry of length 0 is not live (a failed write of 3.0/3.1; reported). Lines `o` does not point at are not live: 3.0/3.1 deleted by removing the key from `o` and wrote no tombstone. | The map key. | +| **J32** — JSONL with a 3.2 index | `.jidx` with `"v":3`, `o` and `s` | As J30. In addition `s` must not exceed the file size and must be a line end; complete lines after `s` are not live (3.2 never showed them; reported, kept in the backup). | The map key. | +| **J33** — JSONL of 3.3-dev | `.jidx` with `"v":4`, or, in a database whose lock record has an epoch, no `.jidx` or one of neither 3.0/3.1 nor 3.2 form | The data file is authoritative: every complete line from the start, the last line of a key wins, a tombstone `{"__nonedb_deleted":K}` removes K. A line that does not decode is salvaged as 3.3-dev did (STORAGE_FORMAT.md §6.1 salvage rule); one that cannot be salvaged is not live (reported). An unterminated last line is not a record (reported). Shard lines whose key is outside the shard's range are not live. | The line's `key` (an integer). | + +3.3-dev replaced a 3.0/3.1 index by its own checkpoint as soon as it opened the database, reading included, and +replayed the whole data file: from then on the files no longer tell which records 3.0/3.1 had deleted, and such a +segment is J33 (3.3-dev was never released; the fixture `3.1-3.3-dev-read` shows it). + +A JSONL segment that matches no row (no `.jidx` outside 3.3-dev, an unreadable `.jidx`, an index of an unknown +version, a J30/J32 entry that points at the wrong line, J32 `s` beyond the file) stops: §6.2. + +A record is carried over as stored: for J30/J32/J33 the bytes of its line are reused when the line ends with +`"key":K}`; otherwise (W segments, lines with `key` elsewhere) it is encoded the way the engine encodes a record +(STORAGE_FORMAT.md §6.1: `key` last, the encoding of §2). A JSON list record of 1.4 becomes an object with members `"0"`, `"1"`, … +(the same PHP array as before). + +### 4.2 Shard size + +For a sharded database S is the meta's `shardSize`, checked against the files, because versions before 3.2 +computed keys with the shard size of the running process, not the meta's: +- a W shard N must hold at most S entries; +- every key of a J30/J32 shard N must lie in `[N·S, (N+1)·S)`; a J33 line outside that range is not live, and its + key still raises the next key; +- the meta's shard ids must be distinct non-negative integers. +If any check fails, the database stops (§6.2). A single-layout database gets the shard size of +`--shard-size` (its keys don't depend on it). + +### 4.3 Write buffers + +Buffered records are live in every version that has buffers (each flushes them before it reads). The tool +assigns their keys as the version's flush would, in this order: + +1. **Shard buffers of 2.3–3.1** `{h}-{name}_s{N}.nonedb.buffer`: shard N must be listed in the meta. Keys + `N·S + c, N·S + c + 1, …` where `c` is the shard's slot count (W: entries of `data`; J30: `o`'s `n`). Their + records are already counted in the meta. +2. **Interrupted flushes of 3.2/3.3-dev** `….nonedb.buffer.flushing.{K}`, by ascending K: keys `K, K+1, …`; + a key the live set already holds is skipped (the flush had stored it). +3. **The current buffer** `{h}-{name}.nonedb.buffer` (single layout of 2.3–3.1; every layout of 3.2/3.3-dev): + keys from the next key (§5), in line order. +4. **Interrupted flushes of 2.3–3.1** `….buffer.flushing` and `_s{N}.nonedb.buffer.flushing`: only with + `--flushing=import` (§6.3), with fresh keys after everything else, so they never take a key an interrupted + flush of 3.2/3.3-dev reserved. + +A buffer line is a record when it decodes to an array; blank lines are ignored; any other line is reported and +not imported. A `key` member in a buffered record is replaced by the assigned key. A buffer of a layout the +version never flushes (a single-layout buffer next to a meta, a shard buffer of a shard the meta doesn't list) +stops: §6.2. + +## 5. Next key + +The next key of a migrated database is the maximum of: +- for W segments: single `count(data)` (the largest index + 1, null entries included); shard N: `N·S` + its + entry count; +- the `n` of a J30/J32 `.jidx` (single: as is; shard N: `N·S + n`), and of a J33 checkpoint (shard: `N·S + n`); +- the meta's `nextKey` (sharded); +- every key in every line of every JSONL segment + 1, record or tombstone, live or not; +- every key given to a buffered record + 1; +- 0. + +If the highest live key is below `next key − 1`, the database gets a tombstone for `next key − 1` (STORAGE_FORMAT.md +§6.2), so the engine never hands out a key the old version had used. + +## 6. Stops and decisions + +### 6.1 Results + +Each database gets one result in the pre-check (§7.1): + +| Result | Meaning | +|---|---| +| `READY` | It can be migrated as described in §4–§5. | +| `AMBIGUOUS_SOURCE` | The files don't determine its live data. Nothing is written for it. The report names the files and the decision that is missing. | +| `UNSUPPORTED_SOURCE` | Its files are recognised but cannot become noneDB 4: a name noneDB 4 cannot hold (an empty name, possible in 2.2), a meta of an unknown `version`. | +| `TARGET_EXISTS` | The noneDB 4 directory of this name and key already exists (a database created after an earlier run). Nothing is merged or overwritten: it is reported as `AMBIGUOUS_SOURCE`. | + +A run migrates the `READY` databases it is asked to migrate (all, or `--db`) only when every one of them is +`READY`; otherwise it stops before writing anything. + +### 6.2 `AMBIGUOUS_SOURCE` conditions + +- A JSONL segment without a usable index outside 3.3-dev: missing, empty, not JSON, unknown `v`, an `o` entry + whose line does not decode to its key, J32 `s` beyond the file or not at a line end. **`--rebuild-from-data`** + replaces the index by a replay of the data file (rule J33). For a segment last written by 3.0/3.1 this brings + back every record they deleted and not yet compacted; the report says the index was replaced, not what came back. +- A data file that decodes neither as W nor as JSONL lines of records. +- A sharded database whose meta is unreadable or fails §4.2 (§4.2 also keeps every key in one segment). +- A buffer the version never flushes (§4.3). +- An interrupted flush of 2.3–3.1 without `--flushing` (§6.3). +- The source files changed during the run (§7.6). + +### 6.3 Interrupted flushes of 2.3–3.1 + +2.3–3.1 renamed the buffer to `….flushing` while flushing and never read that file again. After a crash its +records are either only there, or already stored too, and nothing on disk tells which. `--flushing=import` +inserts them with new keys (what 3.2 and 3.3-dev did when they opened such a database); `--flushing=skip` leaves +them out of the migrated database (what 2.3–3.1 themselves showed). In both cases the file is backed up and +retained. + +### 6.4 Leftovers + +Files of the source key that belong to no database (§3), for example an info file, a lock file without data, +shard files without a meta, temp files of interrupted writes: they make the database's name `AMBIGUOUS_SOURCE` +when they contain data the tool cannot place (shard files without a meta), and are otherwise backed up and +retained with the run. The report lists every one. + +## 7. A run + +The steps follow STORAGE_FORMAT.md §9.2 and §9.5. The journal `{dir}/.nonedb-maintenance/journal.json` +(op `migrate`) is written after every step; its states are in parentheses. + +### 7.1 Pre-check (no journal) +Lists the directory, groups the files (§3), reads every database (§4–§5) and decides its result (§6.1). +Computes the space needed: the backup and the staged databases each take about the size of the old files (twice +that when both are on one file system); a run stops before writing anything when it is not free. Records each source file's +**fingerprint**: name, size, mtime, inode and SHA-256. With `--dry-run` the run ends here and prints the report. + +### 7.2 Locks +`STORE.LOCK` (created if missing) and the lock files of the source databases that have one (3.2, 3.3-dev) are taken +with `LOCK_EX|LOCK_NB`. If one is busy, every lock taken is +released and the run ends with `MAINTENANCE_BUSY`; nothing has been written but an empty `STORE.LOCK` where there was +none. The locks are held until the run ends. Then the journal is written (`DISCOVERED`) with the files of the store +the run makes: `STORE.LOCK`, `.htaccess`, `index.html` where they are missing, and a `STORE.LOCK` without a STORE record +(a run stopped before its journal left it; noneDB 4 creates none next to old files). + +### 7.3 Backup (`BACKED_UP`) +Every source file of the databases and leftovers of this run is copied into +`{backup-dir}/{runId}/files/`, with `manifest.json` written last: run id, source key's keyCheck, every file's +name, size, mtime and SHA-256, the databases of the run, and the STORE record when one exists. +Directories are created `0700`, files `0600`. The copy is read back and compared before the state is recorded. + +### 7.4 Staging (`STAGING`) +`{dir}/.nonedb-maintenance/staging/` is a noneDB 4 store of its own, holding the target key. For each database +the tool writes, by STORAGE_FORMAT.md: FORMAT (a new `databaseId`), the state record `1:{epoch}`, CURRENT, +`generations/1/` with `metadata`, `shards.json` (sharded) and the segments (§4.1, §5). It then opens the staging +store with the engine and creates the field and spatial indexes the source had (`createFieldIndex`, +`createSpatialIndex` with the source's node size), taken from the source's index list or index files. + +### 7.5 Verification (`VERIFIED`) +With the engine, for each database of the staging store: +`checkIntegrity()` is ok; every record, read in batches of keys, equals the expected record (`===`); `count()` +equals the number of live records; the next key (`getShardInfo()` / the key index) equals the computed one; +for every indexed field and a sample of values, the indexed lookup equals a scan; the recovery log is empty. +A difference stops the run with the database named (nothing published). + +### 7.6 Publish (`PUBLISHING`) +For each database: the fingerprints of its source files are checked again (a difference: `AMBIGUOUS_SOURCE`, the run is rolled +back). The staged database's state record gets `m`, and its directory is renamed to `{dir}/{locator}.ndb4`. The +target must not exist and must be on the same file system. From here until the commit the published +databases answer `MAINTENANCE_IN_PROGRESS`. + +### 7.7 Commit (`COMMITTED`) +The fingerprints of every file the run moves are checked a last time. The journal is written with state `COMMITTED` and the target STORE +record: the current one with the target key added if this run completes the key (§9), else unchanged; a new +store (`seq` 1) when the directory has none. **This is the commit point.** Then, in this order: +1. both STORE copies; +2. the source files of the migrated databases and leftovers are moved to `{dir}/.nonedb-retained/{runId}/` + with a `manifest.json` (file names, sizes, SHA-256, backup reference); +3. `m` is cleared (new epoch) on the published databases; +4. the staging directory is removed; +5. the journal is closed: it moves to `{backup-dir}/{runId}/journal.done.json`. + +### 7.8 Final verification (`FINAL_VERIFIED`) +After the journal is closed (the engine opens no store while one is open), a new engine instance opens every +migrated database: `checkIntegrity()` and `count()`; with `--verify` every record is compared, and every index with a +scan, against the live set read again from the backup. The report is then written next to the closed journal +(`report.json`), and the state `FINAL_VERIFIED` recorded in `journal.done.json`. The migration is finished when the +tool exits with 0: the applications stay stopped until then. A run whose key is not registered yet +(§9) has no final verification: the engine cannot open its databases until a later run registers the key. +`FINAL_VERIFIED` is recorded all the same, so `--cleanup` of such a run relies on the verified backup alone. + +## 8. Backup + +`{backup-dir}/{runId}/`: `files/` (the source files, byte for byte), `manifest.json`, `journal.done.json`, +`report.json`, and after a restore `v4/` and `restore.done.json` (§11). A backup is valid only with a `manifest.json` whose checksum and every file's SHA-256 verify. +The tool never deletes a backup. + +**Permissions.** The backup and its records are private: directories `0700`, files `0600` (`restore.done.json` +`0640`). The noneDB 4 files the +tool writes get the engine's default permissions, files `0640` and directories `0750` (CONFIGURATION.md "Files and +permissions"); other `filePermissions` or `directoryPermissions` apply to the files the engine creates later. +The retained old files (§12) keep their own. + +## 9. Key registration + +A key is added to the STORE record only by the run that leaves no database or leftover of that key in the old +format in the directory: a `--db` subset publishes its databases, but they stay unreachable (`KEY_NOT_REGISTERED`) +until a later run completes the rest. A run that finds no old file of the key registers it only when the directory +already holds noneDB 4 databases of that key (published by such a partial run); otherwise it does nothing, so a +mistyped key is never registered. A key that is already registered (databases created with it earlier) +is not added again; its published databases become reachable at the commit. + +## 10. Interruption + +The tool can be stopped at any point (a crash, `kill -9`). Run it again with the same options. A temp file a stopped +run left (STORAGE_FORMAT.md §2) is removed when the run writes that file again, or by `--abort` (the journal's): + +| Journal state | Run again | With `--abort` | +|---|---|---| +| none | starts over | nothing to do | +| `DISCOVERED`, `BACKED_UP` | makes the backup or verifies it, continues | releases the run: journal removed, the backup stays | +| `STAGING`, `VERIFIED` | removes the staging directory, stages again | removes the staging directory and the journal | +| `PUBLISHING` | checks the fingerprints, continues publishing (published directories still carry `m`) | removes the published directories; the old files are untouched | +| `COMMITTED` | completes §7.7 (each step is idempotent) and §7.8 | refused: the run is committed | +| closed, without `FINAL_VERIFIED` (a crash during §7.8) | runs §7.8 for that run (found through `.nonedb-retained/{runId}/manifest.json`) and reports it; nothing is migrated again | nothing to abort | + +While the journal exists, every process using the directory gets `MAINTENANCE_IN_PROGRESS`. A journal whose +state contradicts the files (a published directory missing, an unreadable journal) stops the tool: it reports what it found and changes nothing. + +## 11. Restore + +`--restore=RUNID` puts the directory back into its state before the run. It needs `--backup-dir`, the run's target key +(`NONEDB_SECRET`), and refuses, changing nothing, when the key has a database the run did not create: the key leaves +the STORE record, and that database would become unreachable. +1. verifies the backup (manifest and every SHA-256); +2. takes the locks (§7.2): `STORE.LOCK` and every `LOCK` of the run's databases, `LOCK_EX|LOCK_NB`; +3. writes the journal (op `restore-old`, state `RESTORING`); +4. moves the run's noneDB 4 directories into `{backup-dir}/{runId}/v4/`; removes the key from the STORE record + (the STORE files go there too when no key remains); +5. moves the retained files back from `.nonedb-retained/{runId}/` (or copies them from the backup after a + cleanup), checking every SHA-256; +6. closes the journal; when no STORE record is left, removes the files of the store the run made (§7.2) and the + empty `.nonedb-maintenance/` and `.nonedb-retained/`. + +The old files are then usable by the old version again; noneDB 4 does not read them. + +## 12. Cleanup + +`--cleanup=RUNID`, with `--backup-dir` and the key, verifies `.nonedb-retained/{runId}/` against the backup's manifest (every SHA-256) and removes +it. It can be repeated. A run without `FINAL_VERIFIED` (§10) is refused, nothing removed: run the tool again first. + +## 13. Result + +Exit status 0 when the requested operation completed; otherwise the status names the first stop: + +| Status | Result | +|---|---| +| 0 | done (or a dry run whose databases are all `READY`) | +| 2 | `AMBIGUOUS_SOURCE` / `UNSUPPORTED_SOURCE` (nothing written) | +| 3 | `MAINTENANCE_BUSY` (nothing written but an empty `STORE.LOCK`, §7.2) | +| 4 | `MAINTENANCE_IN_PROGRESS`: committed, not completed, run again; or another maintenance, or a run of other keys, is open | +| 5 | another failure; the message names it | +| 64 | usage error | + +The pre-check's report lists, per database: its status with the reasons, segment kinds, layout and shard size, live +records, buffered records, next key, the field and spatial indexes, and records and lines not carried over (with the +reason). The report of a run (`report.json`) lists the run, the backup, each database's records, whether the key was +registered, and the retained directory. `--json` prints either as one JSON object. + +## 14. Time, memory and space + +Measured with PHP 8.2. The sources were databases written by +3.2 itself: shards of 10 000 records, a field index on a field whose values are in every shard. + +| Records | Dry run | Run | Run with `--verify` | Peak memory (RSS) | +|---|---|---|---|---| +| 1M (137 MB, 306 files) | 6 s | 63 s | 98 s | 98 / 116 MiB | +| 10M (1.4 GB, 3006 files) | 61 s | 27 min | 47 min | 125 / 135 MiB | + +- **Memory:** the run holds one segment at a time and the engine's cache (`cacheMemoryLimit`, half of + `memory_limit` by default), so PHP's default `memory_limit` of 128M is enough at every size measured. + A smaller limit gives the cache less room and leaves less for one segment's work: 1M records ran at 64M. + More memory shortens the index builds: at 10M the global index of `group` takes 100 s at 128M and + 32 s without a limit. +- **Time** grows with the records: the tool reads every record twice (conversion and verification) and + `--verify` reads the published databases a third time. A field index of a sharded database built under a + memory limit takes one pass over the shards per part (§7.4; CONFIGURATION.md "Memory of operations + over many segments"). +- **Disk:** free space for the staged databases (about the size of the source) on the directory's file + system and for the backup (the same) on `--backup-dir`'s: twice the source's size when both are on one + file system, which then holds three times the source during the run (10M: 4.2 GB). The pre-check stops a + run without that room (§7.1). After the commit the old files are retained (moved, not copied) until + `--cleanup`. +- **Downtime:** the applications stay stopped for the whole run (§7.2): plan for the times above. A + `--dry-run` (a minute at 10M) checks first that every database is `READY` and, with `--json`, reports the bytes + of the old files. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..ca591a9 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,229 @@ +# noneDB 4 Operations + +**Engine:** noneDB 4.0.0 + +This document is for the people who run noneDB in production: what to monitor, how to back up and restore, +how to rotate keys, and what to do when the engine refuses to work. File formats are in +[STORAGE_FORMAT.md](STORAGE_FORMAT.md), outcomes of writes in [DURABILITY.md](DURABILITY.md). + +--- + +## 1. Metrics: `getMetrics(?string $db = null)` + +`getMetrics()` returns cheap counters of the process and, for a database, its metadata. Nothing is scanned +and nothing is written: the buffer is not flushed, no recovery runs, no checkpoint is written. A full check +of the data is `checkIntegrity()` (§2). + +```php +$db = new noneDB(); +$db->insert("users", [["name" => "John"], ["name" => "Jane"]]); +$metrics = $db->getMetrics(); // ['process' => [...]] +$metrics = $db->getMetrics("users"); // ['process' => [...], 'database' => [...]] +``` + +### 1.1 `process` + +Counters of the PHP process since it started, for every instance and database it used. They last one request +under a web server (PHP frees them with the request); a long-running CLI process keeps adding to them. + +| Key | Meaning | +|---|---| +| `indexCache.hits` | An operation needed an index of a segment (key index, field index, R-tree) or a global index, and the process had it in memory. | +| `indexCache.misses` | It had to read it from disk: the whole file, or the header and first page of a paged field index. | +| `indexCache.evictions` | Cached indexes and data bytes the process dropped to stay under `cacheMemoryLimit` (CONFIGURATION.md). A count that grows with every operation means the limit is too small for the working set. | +| `indexCache.limit` | The cache limit in bytes; 0: none. | +| `indexReads.pages` | 4 KiB pages read from index files: key-index checkpoints, field and global field indexes, R-trees (a whole file counts its pages). | +| `indexReads.bytes` | Bytes read from index files. | +| `checkpoints.written` | Derived files written: key-index checkpoints, field, global and spatial index files. | +| `checkpoints.seconds` | Time spent writing them. | +| `compactions.count` | Segments compacted (each data file replaced, STORAGE_FORMAT.md §6.2). | +| `compactions.seconds` | Time spent compacting them. | +| `locks.acquired` | Locks taken: database locks and `STORE.LOCK`. | +| `locks.waited` | Of these, the ones that had to wait for another process. | +| `locks.waitSeconds` | Time spent waiting. | +| `locks.timeouts` | Waits that ended with `LOCK_FAILED` after `lockTimeout`. | +| `recovery.events` | Events the process logged: a database's recovery log (STORAGE_FORMAT.md §7.1) and the store log (§3). | +| `recovery.actions` | The same, by `action` (for example `cut-tail`, `generation-recovered`, `restore`). | +| `durability` | `mode`, `lastOperation`, `lastSyncError` and `syncs`, as `getDurability()` (DURABILITY.md §6). | +| `integrity` | The last `checkIntegrity()` of the process: `database`, `ok`, `time`, `records`, `issues` (bad lines, torn tails and index mismatches together); `null` before the first. | +| `memory.usage`, `memory.peak` | `memory_get_usage()` and `memory_get_peak_usage()`. | + +### 1.2 `database` + +Read under the database's shared lock, from its files' metadata: the counters noneDB keeps, and one listing +of its directory for the sizes. A database that does not exist gives `['name' => ..., 'exists' => false]`, +and nothing is created. + +| Key | Meaning | +|---|---| +| `name`, `exists` | | +| `databaseId`, `generation`, `layout`, `shardSize` | From FORMAT, CURRENT and the generation's metadata. | +| `segments` | Data files: 1 in the single layout, the shards in the sharded one. | +| `records`, `nextKey` | Live records and the next key, as `getShardInfo()` reports them. | +| `bufferedRecords` | Records in the write buffer, not yet in the data files. | +| `fieldIndexes`, `spatialIndexes` | The indexed fields. | +| `bytes` | Sizes of the database directory's files: `data` (segment data files), `indexes` (checkpoints and index files), `buffer`, `metadata` (FORMAT, CURRENT, LOCK, metadata, shards.json, indexes.json), `recovery` (log and quarantine), `other` (anything else: an unreferenced generation, a set-aside shard file, a temp file), `total`. | +| `state.buffered` | The state record says the buffer may hold records (`b`). | +| `state.recoveryPending` | A crashed writer left work for the next one (`j`, `J`, `c`, `d`). | +| `integrity` | The last `checkIntegrity()` of this database in this process, as in §1.1; `null` before the first. | + +What `checkIntegrity()` can't read, `getMetrics()` can't either: a damaged database (the next read or write +repairs what it can prove), or one whose generation switch was interrupted and waits for the next read or write to +recover it, is `FORMAT_DAMAGED`; one under maintenance `MAINTENANCE_IN_PROGRESS`. + +```php +$process = $db->getMetrics()['process']; +$database = $db->getMetrics("users")['database']; +$hitRate = $process['indexCache']['hits'] / max(1, $process['indexCache']['hits'] + $process['indexCache']['misses']); +``` + +**What to watch:** +- `indexCache.evictions` and `misses` growing with every operation of a long-running process: the cache limit + is too small for the working set (CONFIGURATION.md "The cache"). +- `locks.waited` / `locks.timeouts`: writers waiting for each other, or for a backup. +- `recovery.events`: above 0 a crash was recovered, damage was found, or a restore or a shard list rebuild ran; + `recovery.actions` says which, `getRecoveryLog()` the details. +- `durability.lastSyncError` (durable mode): the disk reported an error. +- `database.bytes.other` above 0: leftovers (§6). `database.bytes.recovery`: quarantine files (§6). +- `database.state.recoveryPending`: a writer died; the next read or write recovers. + +## 2. Integrity checks and the recovery log + +`checkIntegrity($db)` reads every line of a database and compares its indexes and counters with the data; it +writes nothing. Run it after an unclean shutdown, after restoring files by other means, and from time to time +(it reads the whole database: run it when the load is low). `getRecoveryLog($db)` lists what recovery did +(API.md "getRecoveryLog()"). + +```php +$report = $db->checkIntegrity("users"); +if (!$report['ok']) { + // badLines, tornTail, indexMismatches, pendingRecovery say what +} +$events = $db->getRecoveryLog("users", 20); +``` + +| Finding | What it is | What to do | +|---|---|---| +| `tornTail` | An append that a crash cut | Nothing: the next write moves it to quarantine and cuts it | +| `badLines` | Lines that don't decode (damage) | Restore from a backup, or `compact()` (moves them to quarantine) | +| `indexMismatches` (key, field, spatial, meta) | An index or the counters disagree with the data | `rebuildIndex()`: it makes the key indexes again, the field and spatial indexes of every segment and the global ones, and recounts the shard counters | +| `indexMismatches` (`generation`) | A generation other than the active one (§6) | Inspect and remove it (§6) | +| `pendingRecovery` | A crashed writer's journal, an interrupted creation or drop | Any read or write recovers it | + +A field index or an R-tree that fits its segment is compared with the records through its own format, with the data lines +after it replayed; one whose pages can't be read counts as a mismatch. One of another epoch is not compared: its next use +rebuilds it. The global field and spatial indexes are made from the segments' files and are not compared. + +## 3. Backups + +[BACKUP_RESTORE.md](BACKUP_RESTORE.md): online backups of one moment, checks, restores, the store record. + +## 4. Keys + +A store lists the keys that may open it (STORAGE_FORMAT.md §3.1). Key operations are static methods that take +the store's directory; each returns `storeManifest`, the complete store record: keep the newest one somewhere +safe, `repairStore()` restores it. + + +```php +// A second application with a key of its own +$r = noneDB::registerKey('/var/lib/app/nonedb', $key, $otherKey); + +// Rotating a key: stop the application processes first +$r = noneDB::migrateKey('/var/lib/app/nonedb', $oldKey, $newKey); +if (!$r['success']) { + // $r['code']: MAINTENANCE_BUSY (a database is in use: nothing was written), MAINTENANCE_IN_PROGRESS + // (interrupted after its commit: run it again), FORMAT_DAMAGED (a directory didn't verify: rolled back) +} +file_put_contents('/secure/place/store-manifest.json', json_encode($r['storeManifest']) . "\n"); +``` + +**Rotating a key:** +1. Stop the processes that use the store (a database in use makes the rotation `MAINTENANCE_BUSY`, and nothing + is written). +2. Run `migrateKey()`. Every database directory of the old key is renamed to the new key's locator; contents + don't change. A process that still runs with the old key gets `MAINTENANCE_IN_PROGRESS` during the rotation and + `KEY_NOT_REGISTERED` after it, never an empty database. +3. Put the new key in the configuration; restart long-running processes (FPM workers). +4. Keep the returned `storeManifest`. + +An interrupted rotation leaves the store closed (`MAINTENANCE_IN_PROGRESS` for every process) until it is run +again with the same keys: before its commit it continues, or `['abort' => true]` undoes it; after the commit it +only finishes. + +**A store whose record is lost** (both copies damaged, `FORMAT_DAMAGED` naming the store record): +`repairStore($dir, $keys, ['manifest' => $storeManifestOrStoreJson])` restores it completely. Without a manifest, +`['confirmManualRebuild' => true]` rebuilds it from the keys given (`complete` false: keys registered before +that have no database are lost). Every database directory must belong to one of the keys, else nothing changes. +One damaged copy is repaired automatically and logged in `.nonedb-maintenance/log.jsonl`. + +## 5. When noneDB refuses to work + +noneDB fails closed: when it can't prove a state, it stops with a code instead of guessing. The data files are +kept in every case. + +| Code | Meaning | What to do | +|---|---|---| +| `UNSUPPORTED_FORMAT` | `dbDir` holds files that are not noneDB 4 objects: databases of noneDB 1.x–3.x, other files, a database directory that is a link. Existing databases keep working; no database is created or answered as missing | Migrate old databases (MIGRATION.md); move other files out; replace a linked database directory by the directory itself | +| `KEY_NOT_REGISTERED` | The configured key is not registered for the store | Use the registered key, or `registerKey()` | +| `MAINTENANCE_IN_PROGRESS` | A key operation or a migration is running or was interrupted (the journal `.nonedb-maintenance/journal.json` exists), or a database is marked by one | Wait, or run the interrupted operation again with the same arguments | +| `MAINTENANCE_BUSY` | A key operation found a database in use or the store locked (a backup holds it shared). Nothing was written | Stop the processes, wait for the backup, run it again | +| `LOCK_FAILED` | A lock wasn't taken within `lockTimeout`: a long holder, or readers that never stop in front of a writer (no queue) | A larger `lockTimeout`, find the long holder (`getMetrics()`: `locks`), retry the write | +| `OUTCOME_UNKNOWN` | A failure after a commit point, or an undo that failed | Read the state again before retrying; the next access recovers | +| `DURABILITY_FAILED` | A required sync failed (durable mode); rolled back | Check the disk (`getDurability()['lastSyncError']`) | +| `FORMAT_DAMAGED` | See below | See below | + +**`FORMAT_DAMAGED`** names what can't be proven: +- **The store record** (both `STORE` copies invalid, or the two differ): §4, `repairStore()`. +- **A database's `CURRENT`** is invalid, or an interrupted generation switch left a `CURRENT` that names neither of + its two generations: noneDB can't tell which generation was committed, so it removes neither. + - **The way out:** `restore()` from a backup (BACKUP_RESTORE.md §4, "Over a damaged database"). The damaged + generations stay in the directory for inspection. + - **Without a backup, by hand** (with every process stopped): + 1. Copy the whole database directory `{dbDir}/{locator}.ndb4/` away first. + 2. In `generations/`, each numbered directory is a complete generation. Its `metadata` names its number, and + `segments/` holds its data (JSON lines). Pick the generation to keep: usually the higher number, the + switch's target; check its data files are complete (every line ends with a newline). + 3. Write `CURRENT` for it: the single line + `{"magic":"NONEDB4","kind":"current","databaseId":"","generation":,"checksum":""}` + where the checksum is the SHA-256 of the line without its `,"checksum":...` member, followed by `}` and a + newline (STORAGE_FORMAT.md §2, §4.3). + 4. Replace the state record of `LOCK` (its first 128 bytes; don't delete the file) with `1:`, 8 new + lowercase hex digits (a new epoch), ` b` when the kept generation's `buffer/` holds a file, and ` J` + (recount the counters), padded with spaces to 128 bytes: for example `1:5e0c7a21 b J` (STORAGE_FORMAT.md + §4.2). + 5. Read the database once (until then `checkIntegrity()` says `pendingRecovery`), run `checkIntegrity()`; then + remove the other generation's directory. +- **A database's `FORMAT`** is invalid and `CURRENT` and its generation's metadata don't prove the identity: as for + `CURRENT`, `restore()` is the way out (it rewrites `FORMAT`). + +## 6. Leftovers + +Files noneDB keeps and never removes on its own, because nothing proves they are garbage: + +| Leftover | Where | When it can go | +|---|---|---| +| An unreferenced generation | `generations/{N}/` other than `CURRENT`'s: an interrupted generation switch, or the damaged generations a restore kept | Once you have checked it holds nothing you need (§5). `checkIntegrity()` reports it (`generation` mismatch); `getMetrics()` counts it under `bytes.other` | +| A set-aside shard file | `segments/s{N}.jsonl.orphan.{YmdHis}`: a shard file the shard list didn't name, found when that shard was created again | After inspection; `getDBs($name)` lists it (`orphanFiles`) | +| Quarantine files | `recovery/q.*.bin`: bytes cut from a data or buffer file (a torn tail, a damaged line), verified copies | When the recovery log's events about them are understood; `dropDB()` removes them with the database | +| Retained sources of a migration | `.nonedb-retained/{runId}/` | `noneDB-migrate.php --dir=… --backup-dir=… --cleanup=RUNID`, with the key set (MIGRATION.md) | + +Remove leftovers only while no process uses the database. + +## 7. Deployment checklist + +- PHP 8.1–8.5 with `json` (and `mbstring` for Unicode case folding in `like`). +- `dbDir` on a local file system, outside the web root; owned by the user PHP runs as. Files `0640`, + directories `0750` by default (`filePermissions`, `directoryPermissions`); group access for a second user + (CLI, cron) works through the group. +- `durability`: `durable` on Linux where a power loss must not lose acknowledged writes (every write waits for + the disk); `performance` otherwise. +- `lockTimeout` that covers your longest write and your backups. +- `cacheMemoryLimit`: the default (half of `memory_limit`) for web requests; more for long-running workers on + large databases. +- `autoCreateDB` `false` in production. +- Backups (BACKUP_RESTORE.md) and a restore drill. +- Monitoring: `getMetrics()` (§1), `checkIntegrity()` after unclean shutdowns (§2). +- After key maintenance, restart long-running processes. +- A new version of noneDB: stop every process of the old one first. Two versions on one store rewrite each other's index files. +- Upgrading from noneDB 1.x–3.x: MIGRATION.md (the engine refuses the old files until they are migrated). diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 0000000..974f3ca --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,361 @@ +# noneDB 4 Performance + +**Engine:** noneDB 4.0.0 · **Measured with:** PHP 8.2.0 (CLI) + +This document covers what is measured and against what, with the results and the limits found. The numbers +come from the scripts of §8, except the costs of the exact map in §4 and §7, which come from one-off measurements. + +**The baseline** is the frozen 3.3 development state (`build/baseline/noneDB-3.3dev.php`, sha256 `32c3d641…`): the +last code of the 3.x line. It was itself measured against 3.2.0: single writes to 10,000 records were 12–17 times +faster, with no verified regression. + +--- + +## 1. Summary + +- **The performance gate** (§2): two independent runs of 66 workloads on the release candidate. + - 46 workloads are verified improvements and 13 show no verified change. + - 5 are findings, significant in both runs: one insert into a small shard is 3.2–7.0% slower, the others at most 3.1%. + - 2 are fixed costs of the fail-closed store, within their budgets. + - None is a verified regression. + - Cold operations (a new process) are up to 30 times faster. One is slower: a miss on one shard, +0.19 ms, + within its budget. +- **Ten million records** (§3): the first indexed query of a process takes 0.04–0.27 s (3.3-dev: 1.9–7.3 s), and + queries need 68–244 MiB where 3.3-dev ran out of 512 MiB. All seven targets of the specification are met. +- **noneDB and SQLite** (§5): SQLite is faster everywhere, as a C library should be. noneDB needs no extension, and + its files are lines of JSON. + +## 2. The performance gate + +`tests/Performance/perf_gate.php` compares two libraries workload by workload. + +**Method** +- **Alternation:** the baseline and the candidate run alternately (ABBA). Each run is a child process on a fresh + copy of a fixture built by the same library. The shard matrix reuses one fixture per library: its lookups run + before its inserts. Seeds are fixed. +- **Runs:** 40 per library and workload (30 for the concurrent ones). +- **Kinds of workload:** + - *warm*: k timed operations after 3 untimed ones; + - *cold*: a new instance and its first operation; + - *bulk*: one timed operation after one key lookup. +- **Statistics:** + - Ratio = candidate / baseline (> 1 is slower), with a 99% bootstrap confidence interval of the median ratio. + - A one-sided Mann-Whitney test and the Hodges-Lehmann shift. + - p95 values pool up to 200 latencies per run, and use a cluster bootstrap over runs. +- **A regression** must show in two independent runs. It is one of: + - a median more than 5% slower and significant (p < 0.01 and CI above 1); + - a p95 more than 5% slower with its CI above 1; + - a throughput CI entirely below 0.95 (concurrent workloads); + - peak memory more than 5% and more than 64 KiB higher. + + A significant slowdown of 5% or less is a **finding**, explained here and not counted. +- **Fixed costs of the fail-closed store** are judged against absolute budgets, not a ratio: + - 0.25 ms for a cold workload (a new instance and its first operation); + - 0.1 ms for `find_missing_db_warm` (a read of a database that doesn't exist: the directory is checked every + time); + - 0.1 ms for `find_key_after_epoch_change`. + + A time regression within its budget in both runs is reported as "within budget". +- **Settings:** performance mode; child processes with `memory_limit=-1`; the operating system's file cache warm. + +**Result** (2026-11-01, release candidates `4.0.0-rc.5`, artifact `3f1b2205…`, 2 × 20 minutes, and `4.0.0-rc.6`, artifact `7b29f3cd…`: rc.6 changed the field index code, so the 15 workloads of that code were measured again on it, 2 × 2.5 minutes, and their rows below are rc.6's): + +| Workload | 3.3-dev median per run | Ratio run 1 [99% CI] | Ratio run 2 [99% CI] | p95 ratio (1 / 2) | Memory ratio (1) | Verdict | +|---|---|---|---|---|---|---| +| `find_key_warm` | 87.756 ms | 0.965 [0.961, 0.969] | 0.965 [0.962, 0.967] | 0.991 / 1.044 | 0.867 | improvement | +| `insert_one_100_warm` | 14.204 ms | 1.040 [1.007, 1.074] | 1.013 [0.987, 1.060] | 1.049 / 1.030 | 1.038 | no verified change | +| `insert_one_10k_warm` | 4.026 ms | 1.025 [0.980, 1.064] | 1.019 [0.985, 1.062] | 1.023 / 1.022 | 0.949 | no verified change | +| `update_one_10k_warm` | 5.377 ms | 0.950 [0.918, 0.966] | 0.953 [0.920, 0.973] | 0.960 / 0.954 | 0.852 | improvement | +| `delete_one_10k_warm` | 11.942 ms | 0.938 [0.908, 0.967] | 0.927 [0.904, 0.956] | 0.927 / 0.932 | 0.919 | improvement | +| `insert_one_indexed_warm` | 4.103 ms | 0.996 [0.968, 1.017] | 1.000 [0.986, 1.026] | 0.960 / 0.984 | 0.949 | no verified change | +| `update_indexed_10k_warm` | 52.550 ms | 0.937 [0.931, 0.945] | 0.905 [0.902, 0.912] | 0.951 / 0.911 | 0.912 | improvement | +| `range_indexed_warm` | 85.285 ms | 0.862 [0.860, 0.866] | 0.864 [0.863, 0.866] | 0.854 / 0.880 | 0.755 | improvement | +| `find_indexed_eq_warm` | 39.120 ms | 1.030 [1.029, 1.033] | 1.031 [1.028, 1.033] | 1.033 / 1.030 | 1.042 | finding to investigate | +| `count_filter_warm` | 170.948 ms | 0.934 [0.931, 0.936] | 0.934 [0.932, 0.937] | 0.925 / 0.944 | 0.948 | improvement | +| `sort_limit_warm` | 181.813 ms | 0.956 [0.951, 0.960] | 0.956 [0.954, 0.962] | 0.951 / 0.965 | 0.958 | improvement | +| `within_bbox_warm` | 45.696 ms | 1.017 [1.014, 1.019] | 1.016 [1.013, 1.019] | 1.016 / 1.014 | 1.010 | finding to investigate | +| `within_distance_warm` | 39.899 ms | 0.986 [0.985, 0.990] | 0.987 [0.985, 0.989] | 0.994 / 0.993 | 0.980 | improvement | +| `nearest_warm` | 161.566 ms | 0.993 [0.992, 0.995] | 0.993 [0.991, 0.995] | 1.015 / 1.016 | 1.001 | improvement | +| `sharded_find_key_warm` | 106.406 ms | 0.895 [0.892, 0.898] | 0.896 [0.894, 0.900] | 0.943 / 0.961 | 1.037 | improvement | +| `sharded_find_indexed_warm` | 52.033 ms | 0.871 [0.868, 0.875] | 0.870 [0.866, 0.873] | 0.878 / 0.864 | 1.033 | improvement | +| `sharded_insert_one_warm` | 38.311 ms | 1.037 [1.014, 1.058] | 1.013 [0.986, 1.040] | 1.060 / 0.993 | 1.005 | no verified change | +| `find_missing_db_warm` | 27.496 ms | 1.598 [1.595, 1.604] | 1.598 [1.595, 1.601] | 8.683 / 8.574 | 1.769 | within budget (+0.041 ms ≤ 0.10 ms per operation) | +| `find_key_after_epoch_change` | 62.243 ms | 0.243 [0.239, 0.252] | 0.242 [0.241, 0.243] | 0.256 / 0.248 | 0.197 | improvement | +| `find_key_cold` | 1.825 ms | 0.450 [0.445, 0.456] | 0.459 [0.455, 0.463] | - | 0.368 | improvement | +| `insert_one_cold` | 1.926 ms | 0.499 [0.495, 0.502] | 0.503 [0.499, 0.511] | - | 0.364 | improvement | +| `find_indexed_cold` | 3.396 ms | 0.555 [0.552, 0.557] | 0.556 [0.552, 0.559] | - | 0.506 | improvement | +| `within_bbox_cold` | 8.742 ms | 0.749 [0.747, 0.754] | 0.753 [0.749, 0.756] | - | 0.918 | improvement | +| `sharded_find_indexed_cold` | 7.241 ms | 0.485 [0.482, 0.489] | 0.485 [0.480, 0.491] | - | 0.396 | improvement | +| `bulk_insert_1k` | 8.330 ms | 0.970 [0.967, 0.976] | 0.969 [0.963, 0.977] | - | 0.976 | improvement | +| `bulk_insert_10k` | 82.531 ms | 0.959 [0.956, 0.961] | 0.958 [0.957, 0.961] | - | 0.986 | improvement | +| `bulk_insert_100k` | 1011.011 ms | 0.906 [0.905, 0.908] | 0.907 [0.903, 0.911] | - | 0.994 | improvement | +| `sharded_bulk_insert_10k` | 209.913 ms | 1.006 [1.003, 1.008] | 1.006 [1.003, 1.007] | - | 0.259 | finding to investigate | +| `find_all_10k` | 116.097 ms | 0.943 [0.942, 0.945] | 0.944 [0.943, 0.946] | 0.944 / 0.943 | 0.975 | improvement | +| `filter_unindexed_10k` | 128.849 ms | 0.942 [0.938, 0.947] | 0.943 [0.937, 0.947] | 0.954 / 0.941 | 0.965 | improvement | +| `bulk_update_10k` | 58.068 ms | 0.970 [0.967, 0.973] | 0.969 [0.967, 0.972] | - | 0.992 | improvement | +| `bulk_delete_10k` | 49.591 ms | 0.979 [0.977, 0.981] | 0.980 [0.973, 0.989] | - | 0.992 | improvement | +| `buffer_insert_flush` | 91.742 ms | 0.973 [0.955, 1.019] | 1.000 [0.984, 1.027] | - | 1.014 | no verified change | +| `compact_10k` | 65.825 ms | 0.998 [0.991, 1.002] | 0.998 [0.996, 1.001] | - | 0.632 | no verified change | +| `sharded_compact_10k` | 74.241 ms | 0.974 [0.971, 0.976] | 0.971 [0.967, 0.974] | - | 0.863 | improvement | +| `delete_autocompact_10k` | 45.954 ms | 0.985 [0.982, 0.987] | 0.985 [0.983, 0.988] | - | 0.991 | improvement | +| `create_field_index_10k` | 46.712 ms | 1.003 [1.001, 1.006] | 1.004 [1.002, 1.007] | - | 1.002 | finding to investigate | +| `rebuild_index_10k` | 45.311 ms | 0.996 [0.994, 1.000] | 0.997 [0.994, 1.000] | - | 0.834 | improvement | +| `conc_writers_4` | 65.553 ms | 1.044 [0.999, 1.104] | 1.009 [0.979, 1.188] | - | 1.000 | no verified change | +| `conc_writers_8` | 125.087 ms | 0.996 [0.976, 1.021] | 1.017 [0.996, 1.131] | - | 1.000 | no verified change | +| `conc_readers_4` | 103.960 ms | 0.972 [0.914, 1.034] | 0.980 [0.914, 1.032] | - | 1.000 | no verified change | +| `conc_mixed_8` | 150.450 ms | 1.035 [0.955, 1.090] | 1.060 [0.972, 1.113] | - | 1.000 | no verified change | +| `m1_lookup_hit_warm` | 10.245 ms | 1.007 [1.001, 1.009] | 1.005 [1.000, 1.010] | 1.049 / 1.019 | 0.700 | no verified change | +| `m1_lookup_miss_warm` | 4.387 ms | 0.959 [0.954, 0.964] | 0.961 [0.957, 0.964] | 0.965 / 0.963 | 1.000 | improvement | +| `m1_g_lookup_warm` | 12.512 ms | 0.993 [0.988, 0.996] | 0.995 [0.991, 0.998] | 0.992 / 0.998 | 0.632 | improvement | +| `m1_lookup_hit_cold` | 1.079 ms | 0.996 [0.985, 1.010] | 0.990 [0.979, 1.006] | - | 0.685 | no verified change | +| `m1_lookup_miss_cold` | 0.613 ms | 1.303 [1.288, 1.321] | 1.306 [1.291, 1.322] | - | 1.297 | within budget (+0.188 ms ≤ 0.25 ms per operation) | +| `m1_insert_warm` | 9.225 ms | 1.070 [1.007, 1.189] | 1.032 [1.013, 1.050] | 1.137 / 1.034 | 1.061 | finding to investigate | +| `m10_lookup_hit_warm` | 15.944 ms | 0.850 [0.846, 0.855] | 0.852 [0.846, 0.855] | 1.032 / 1.019 | 0.504 | improvement | +| `m10_lookup_miss_warm` | 4.396 ms | 0.962 [0.957, 0.968] | 0.964 [0.959, 0.968] | 0.965 / 0.963 | 1.000 | improvement | +| `m10_g_lookup_warm` | 38.128 ms | 0.917 [0.915, 0.920] | 0.918 [0.915, 0.923] | 0.918 / 0.926 | 0.647 | improvement | +| `m10_lookup_hit_cold` | 2.852 ms | 0.400 [0.397, 0.405] | 0.407 [0.402, 0.413] | - | 0.131 | improvement | +| `m10_lookup_miss_cold` | 2.350 ms | 0.374 [0.369, 0.378] | 0.371 [0.367, 0.376] | - | 0.130 | improvement | +| `m10_insert_warm` | 9.682 ms | 1.005 [0.977, 1.017] | 0.999 [0.969, 1.023] | 0.954 / 0.966 | 0.999 | no verified change | +| `m100_lookup_hit_warm` | 69.729 ms | 0.646 [0.644, 0.648] | 0.646 [0.644, 0.647] | 0.595 / 0.594 | 0.474 | improvement | +| `m100_lookup_miss_warm` | 4.411 ms | 0.965 [0.961, 0.969] | 0.964 [0.959, 0.967] | 0.970 / 0.967 | 1.000 | improvement | +| `m100_g_lookup_warm` | 102.906 ms | 0.943 [0.942, 0.945] | 0.943 [0.942, 0.945] | 1.025 / 1.016 | 0.481 | improvement | +| `m100_lookup_hit_cold` | 21.592 ms | 0.081 [0.081, 0.082] | 0.081 [0.080, 0.081] | - | 0.028 | improvement | +| `m100_lookup_miss_cold` | 20.775 ms | 0.070 [0.070, 0.071] | 0.070 [0.069, 0.070] | - | 0.025 | improvement | +| `m100_insert_warm` | 14.618 ms | 0.839 [0.823, 0.862] | 0.844 [0.821, 0.868] | 0.837 / 0.838 | 0.744 | improvement | +| `m1000_lookup_hit_warm` | 161.291 ms | 0.653 [0.651, 0.656] | 0.653 [0.651, 0.656] | 0.643 / 0.638 | 0.492 | improvement | +| `m1000_lookup_miss_warm` | 4.443 ms | 0.964 [0.960, 0.969] | 0.963 [0.961, 0.971] | 0.963 / 0.965 | 1.000 | improvement | +| `m1000_g_lookup_warm` | 718.900 ms | 0.612 [0.610, 0.614] | 0.612 [0.611, 0.615] | 0.565 / 0.565 | 0.468 | improvement | +| `m1000_lookup_hit_cold` | 215.761 ms | 0.036 [0.036, 0.036] | 0.036 [0.036, 0.036] | - | 0.018 | improvement | +| `m1000_lookup_miss_cold` | 215.431 ms | 0.034 [0.034, 0.034] | 0.034 [0.034, 0.034] | - | 0.017 | improvement | +| `m1000_insert_warm` | 54.718 ms | 0.713 [0.709, 0.717] | 0.701 [0.694, 0.708] | 0.725 / 0.623 | 0.779 | improvement | + +Workload names: `m{N}_*` is the shard matrix: N shards of 1000 records, both fields indexed. Field `u` is unique; +field `g` has 32 × N values, each in every shard at 10 shards and in about 31 shards at 100 and 1000. `*_10k` is a database of 10,000 records. The definitions are in +`perf_gate.php`. + +**At the 5% line:** single inserts sit near the line and move with the session: +- `m1_insert_warm`: +7.0% / +3.2% here. It is over the line in one run only, so it is not a verified regression. One + insert into a shard of 1,000 records with two indexed fields. The few µs are the steps of the v4 contract per write: the + operation's outcome (DURABILITY.md §6), generation and rollback marks, the store and directory checks. +- `insert_one_100_warm`: +4.0% / +1.3% and `insert_one_10k_warm` +2.5% / +1.9% here, no verified change. + +**Findings** (significant in both runs): +- **Single inserts:** `m1_insert_warm`, above. +- **Indexed reads:** `find_indexed_eq_warm` (100 results) +3.0–3.1%, `within_bbox_warm` +1.6–1.7%. + Every segment's key index is checked against its data (epoch, size) before it is used. +- **Within 1%:** `create_field_index_10k`, `sharded_bulk_insert_10k`. + +## 3. Ten million records + +`tests/Performance/large.php` runs this workload: 10,000,000 records `{group, metric, tag, payload}`, inserted in +batches, then `createFieldIndex('group')`. `group` has 10,000 values and every value is in every shard. That is +the worst case for an indexed lookup, which has to visit every shard. It is run with 100 shards of 100,000 records +and with 1000 shards of 10,000. + +- **Conditions:** the files are read once after the build, so the operating system's cache is warm. +- **Cold:** 40 new processes. +- **Warm:** 200 key lookups or updates, or 40 indexed queries, in a process that has already run one of the kind. +- **Memory:** `cacheMemoryLimit` is `'auto'` unless a row says otherwise. +- **What was measured:** the release candidate `4.0.0-rc.6` (artifact `7b29f3cd…`, 2026-11-01), and 3.3-dev + (`32c3d641…`). + +| | 4.0, 100 shards | 3.3-dev, 100 shards | 4.0, 1000 shards | 3.3-dev, 1000 shards | +|---|---|---|---|---| +| Bulk insert | 35.6 s | 40.9 s | 32.0 s | 36.4 s | +| `createFieldIndex('group')` | 34.4 s | 29.5 s | 37.7 s | 38.4 s | +| Peak memory of the build | 1512 MiB | 1609 MiB | 2582 MiB | 2878 MiB | +| Disk | 1157 MiB | 1328 MiB | 1280 MiB | 1420 MiB | +| Warm key lookup p50 / p95 | 0.064 / 0.769 ms | 0.326 / 44.0 ms | 0.275 / 0.285 ms | 4.89 / 5.01 ms | +| Warm indexed query, 1000 results, p50 / p95 / p99 | 12.4 / 13.1 / 21.9 ms | 19.4 / 26.8 / 28.2 ms | 24.1 / 74.5 / 219 ms | 30.8 / 100.4 / 307 ms | +| First indexed query of a process | 38 ms | 1.93 s | 270 ms | 7.27 s | +| Warm update by key p50 / p95 | 0.171 / 0.415 ms | 0.345 / 1.22 ms | 0.229 / 0.244 ms | 0.479 / 1.05 ms | +| Warm process peak | 218 MiB | 1480 MiB | 1380 MiB | 2392 MiB | +| Cold key lookup p50 / p95 | 1.33 / 1.36 ms | 45.2 / 46.6 ms | 1.39 / 1.43 ms | 6.40 / 7.17 ms | +| Cold indexed query p50 / p95 | 116 / 116 ms | 6.91 / 6.98 s | 303 / 305 ms | 7.97 / 8.24 s | +| Indexed query, `cacheMemoryLimit` 384 MiB, p95 | 16.2 ms | 5.98 s | 58.2 ms | 7.58 s | +| Query process at `memory_limit=512M` | works, peak 210 MiB | out of memory | works, peak 244 MiB | out of memory | +| Query process at `memory_limit=128M` | works, peak 112 MiB | — | works, peak 68 MiB | — | + +**The release targets** for 10 million records: + +| Target | Result | +|---|---| +| Bulk insert: no verified regression | Met: 12–13% faster | +| Warm key lookup: no verified regression | Met: 5–18 times faster | +| Warm indexed query: no verified regression | Met: 12.4 against 19.4 ms on 100 shards, 24.1 against 30.8 ms on 1000 | +| Cold key lookup: better, p95 ≤ 100 ms | Met: 1.36 / 1.43 ms | +| Cold indexed query of 1000 results: p95 ≤ 500 ms | Met: 116 / 305 ms | +| 384 MiB cache: no reload loop of seconds | Met: 16.2 / 58.2 ms | +| Queries within a 512 MiB process | Met: peak 210 / 244 MiB | + +The two columns of `createFieldIndex()` come from different sessions. Run one after the other in one session, at the +same memory, the two are level: 1.00–1.02 with PHP 8.2, 6–7% faster in a Linux container with PHP 8.4. Reading the +records takes most of the time, in both. + +**The 1000-shard query holds data files** under the automatic cache limit (§4). That is where its median comes from, and it has +a price: a warm process peaks at 1380 MiB, and the queries that load a file show as the p95 and p99. All three stay under +3.3-dev's. The previous release candidate, which held none, had a median of 42.4 ms at a peak of 380 MiB. + +## 4. First use and later use + +noneDB decodes an index into PHP arrays only once its use pays for the decoding. Until then it reads the index's +pages, or keeps its body as bytes. A single cold operation therefore stays cheap: the cold rows above are 4 to 60 +times faster than 3.3-dev, which decodes every index it touches at once. Later operations take the decoded form. +Where the work falls: + +- **Key indexes** of a database with at most 16 segments are decoded at the start of its second operation: about + 1.8 ms per 10,000 keys. In a larger database a segment's key index is decoded once its lookups pay for it. +- **A field index** of up to 64 KiB is read whole at its first use, except in the process's first operation. A + larger one is read whole once it has been used `c / 20` times (c: its values). +- **The global field index of a sharded database:** its filters answer. From its second operation the process holds + the index's body: 1–7 ms at a million records. A command-line process also builds an exact map of values to shards + at that operation, from the shards' field index files (a file it already holds as it is, it doesn't read again), when + the database has at most a million records. That takes 0.15–1 s at a million records, once; later lookups are a map + lookup. Above that size reading every shard's file would take seconds, and in a web request the map couldn't repay + itself before the request ends, so neither builds one. +- **Data files of a database of more than 16 segments:** reading one record costs about what reading 128 KiB at once + does, so a file in repeated use is read whole and held. It needs a process with a memory limit, and room under half of the + cache limit (CONFIGURATION.md "The cache"). A process with no limit holds none. +- **Field indexes built later:** a writer that only writes never loads field indexes. After a bulk load, the first + query of a field builds the indexes of the shards that have none, from their data, and the field's global index. + For a field of unique values over 100,000 records in 10 shards that took 0.8 s, once (3.3-dev: 0.46 s). + `createFieldIndex()` after the load does it in advance. + +Under PHP-FPM every request is a new process, so the cold numbers apply. A long-running worker reaches the warm +ones. + +## 5. noneDB and SQLite + +`tests/Performance/sqlite_comparison.php`, 2026-10-29, PHP 8.2.0, SQLite 3.43.2, 5 runs per engine and size, each in +its own process. Every workload returned the same records in the same order on all four engines. + +- **Engines:** + - **noneDB-indexed:** field indexes on `city`, `age` and `email`; from 10,000 records the database is sharded + (its default `shardSize`). + - **noneDB-scan:** the same indexes with `fieldIndexEnabled => false`: writes keep them, queries don't use them. + - **SQLite-doc:** JSON documents with expression indexes on the same fields: the closest to what noneDB stores. + - **SQLite-columns:** typed columns with B-tree indexes: an upper bound, not a document store. +- **Settings:** SQLite in WAL mode with `synchronous = OFF`, noneDB in performance mode. +- **Warm:** the calls a workload measures ran once unmeasured first. + +Milliseconds per operation, median of the runs (insert_500 and update_300: all 500 and 300 calls together): + +**10,000 records** + +| Workload | noneDB-indexed | noneDB-scan | SQLite-doc | SQLite-columns | +|---|---|---|---|---| +| find by key | 0.044 | 0.046 | 0.006 | 0.008 | +| find, equality (2000 results) | 7.76 | 30.1 | 0.81 | 1.41 | +| find, unique value | 0.053 | 29.8 | 0.008 | 0.008 | +| range, 4 ages (800 results) | 3.82 | 42.8 | 0.38 | 0.63 | +| count with a filter | 6.74 | 30.4 | 0.067 | 0.048 | +| first() | 0.228 | 0.512 | 0.008 | 0.009 | +| exists() | 0.071 | 0.507 | 0.007 | 0.006 | +| where + sort + limit 10 | 8.81 | 33.4 | 1.38 | 0.76 | +| distinct (977 values) | 35.2 | 34.9 | 4.65 | 0.92 | +| sum with a filter | 7.14 | 31.0 | 1.14 | 0.35 | +| join, limit 50 | 1.90 | 5.10 | 0.069 | 0.048 | +| 500 single inserts | 196 | 212 | 20.6 | 19.8 | +| 300 single updates | 91.6 | 86.7 | 12.4 | 3.65 | + +**1,000 and 100,000 records** (medians) + +| Workload | noneDB-indexed 1k | SQLite-doc 1k | noneDB-indexed 100k | SQLite-doc 100k | +|---|---|---|---|---| +| find by key | 0.042 | 0.005 | 0.046 | 0.006 | +| find, equality | 0.688 | 0.082 | 80.7 | 11.0 | +| find, unique value | 0.047 | 0.007 | 0.063 | 0.008 | +| range | 0.421 | 0.043 | 39.5 | 7.13 | +| where + sort + limit 10 | 0.934 | 0.144 | 90.8 | 15.4 | +| 500 single inserts | 57.5 | 15.9 | 208 | 18.7 | +| 300 single updates | 38.4 | 8.88 | 40.5 | 14.8 | + +SQLite is a C library with B-trees and prepared statements: +- A key lookup takes 0.006–0.008 ms against noneDB's 0.04–0.05 ms. +- Queries that return or count many records are 6–170 times faster in SQLite, because noneDB decodes every record + it reads in PHP. +- Single writes are 3–11 times faster in SQLite-doc (up to 25 times with typed columns). + +noneDB needs no extension. It stores plain JSON lines, which a crash can only cut at the end, and it runs where +`pdo_sqlite` isn't available. For heavy querying of large data, SQLite is the better tool. + +## 6. Durable mode + +`durability => 'durable'` is supported on Linux only: every write waits for its data, and for its directory when +it created or renamed a file, to reach the device (DURABILITY.md). Its cost is the device's sync latency per write. +That is a property of the disk and its cache, from tens of microseconds on an SSD with power-loss protection to +milliseconds elsewhere. PHP can't flush the drive cache on macOS, so durable mode isn't measured there. Performance mode, which every number of this document uses, issues no sync on the write path. + +The release candidate measured on Linux: PHP 8.5 in a Docker container, a database of 10,000 records, not sharded, 300 runs +of each single operation. The container's virtual disk syncs at two speeds, about 0.35 and about 1.3 ms, and the median of a run +can fall on either; a real device gives its own number. + +| Operation | performance, median | durable, median | Syncs (file + directory) | +|---|---|---|---| +| Insert one record | 0.018 ms | 0.388 ms | 1 + 0 | +| Insert 100 records | 0.074 ms | 0.508 ms | 1 + 0 | +| Update one record by key | 0.024 ms | 0.380 ms | 1 + 0 | +| Delete one record by key | 0.023 ms | 0.380 ms | 1 + 0 | +| Create a database | 0.784 ms | 6.990 ms | 5 + 8 | +| Compact 10,000 records | 16.3 ms | 14.5 ms | 2 + 1 | + +A batch costs one sync, whatever its size. A data file's replacement is synced in both modes, so `compact()` costs the +same in each. + +## 7. Known limits + +- **Memory and tail of a warm query over 1000 shards** (§3): a process under the automatic cache limit holds data files, and + peaks at 1380 MiB (3.3-dev: 2392). The queries that load a file are its p95 and p99, 74.5 and 219 ms (3.3-dev: 100 and 307). + `cacheMemoryLimit` bounds it: at 384 MiB the peak is 332 MiB and the median 56 ms. +- **The first query on a cold file cache:** a process whose files are not in the operating system's cache reads them from + disk. At 1000 shards the first indexed query took 400–834 ms (a Linux container, PHP 8.4), 101–120 ms with the cache + warm. The queries after it are the warm ones. +- **Close to `memory_limit`** (CONFIGURATION.md "Memory of operations over many segments"): an operation over large shards + fills what the automatic cache limit leaves. On 10 million records in shards of 100,000 at `memory_limit=128M`, an + unindexed `find()` peaks at 123 MiB, a sum at 121 MiB and `createFieldIndex()` at 121 MiB. They finish; a script that + holds much besides needs more memory or a smaller `cacheMemoryLimit`. One segment of 300,000 records with a different + value each needs more for `checkIntegrity()` (133 MiB) and for a range with a large answer. +- **The exact map in a short command-line script** (§4): a script that runs two or more indexed queries on a sharded + database of up to a million records pays for the map once, 0.15–1 s, and a map saves 7–20 µs per lookup: it repays + itself after 15,000–66,000 lookups. Long-running workers get there; a short script doesn't. Without the map the gate's + warm lookups, which leave its build out, would be 5–15% slower than 3.3-dev's (1–4 µs per lookup). +- **A writer behind readers that never stop:** locks are polled with `flock`, there is no queue, so readers that always + overlap can keep a writer out until its `lockTimeout` ends in `LOCK_FAILED`. With two nonstop readers on 2 million + records and a 16 MiB cache, 216 of 1.26 million updates timed out at a `lockTimeout` of 30 s; on 1.8 million records with + the data held in the cache, none of 22 million. A writer under such a load retries `LOCK_FAILED`; + `getMetrics()` counts them in `locks.timeouts`. +- **The gate's findings and budgeted costs** (§2). +- **Field indexes built at first use after a bulk load** (§4). +- **A cache limit below the working set** makes queries load indexes again. `getMetrics()` shows it as evictions + and misses that grow with every operation; size `cacheMemoryLimit` as CONFIGURATION.md "The cache" describes. + +## 8. Running the measurements + +Every script uses its own temporary directory and key; none of them touches your databases. + + +```bash +# The performance gate: A/B against another noneDB.php, about 20 minutes per run; a regression only when two +# independent runs show it +php tests/Performance/perf_gate.php run --baseline=build/baseline/noneDB-3.3dev.php --candidate=noneDB.php --out=run1.json --md=run1.md +php tests/Performance/perf_gate.php run --baseline=build/baseline/noneDB-3.3dev.php --candidate=noneDB.php --out=run2.json --md=run2.md +php tests/Performance/perf_gate.php verify run1.json run2.json + +# Large databases (10M records, one library per run; --purge empties the macOS file cache first) +php tests/Performance/large.php run --lib=noneDB.php --out=large --total=10000000 --shard-sizes=100000,10000 + +# noneDB and SQLite (needs pdo_sqlite) +php tests/Performance/sqlite_comparison.php 1000,10000,100000 --runs=5 --markdown + +# Single operations, operators, spatial queries, dataset sizes +php tests/Performance/operation_benchmark.php +php tests/Performance/operator_benchmark.php +php tests/Performance/spatial_benchmark.php +php tests/Performance/performance_benchmark.php 100,1000,10000 + +# The cost of durable mode (Linux only) +php tests/Performance/durability_cost.php + +# Soak: hours of work by several processes under random SIGKILL, checked by a process of its own +php tools/soak.php run --out=DIR --hours=N +``` diff --git a/docs/QUERY.md b/docs/QUERY.md index 0040049..695eadb 100644 --- a/docs/QUERY.md +++ b/docs/QUERY.md @@ -1,8 +1,7 @@ # noneDB Query Reference -Comprehensive documentation for noneDB's Query Builder API. - -**Version:** 3.2.0 +The query builder of noneDB 4.0.0: filters, operators, sorting, joins, grouping, aggregates, writes and query +plans. The examples run in order (CI runs them). --- @@ -18,8 +17,10 @@ Comprehensive documentation for noneDB's Query Builder API. 8. [Grouping & Having](#grouping--having) 9. [Spatial Queries](#spatial-queries) 10. [Terminal Methods](#terminal-methods) -11. [Real-World Examples](#real-world-examples) -12. [Performance Tips](#performance-tips) +11. [Query Planning & explain()](#query-planning--explain) +12. [Real-World Examples](#real-world-examples) +13. [Performance Tips](#performance-tips) +14. [Operator Quick Reference](#operator-quick-reference) --- @@ -54,7 +55,7 @@ $products = $db->query("products") ```php // Get query builder instance -$query = $db->query("database_name"); +$query = $db->query("articles"); // Chain methods and execute $results = $query @@ -101,9 +102,11 @@ noneDB supports MongoDB-style comparison operators in `where()` clauses. **How values compare** -- `$gt`, `$gte`, `$lt`, `$lte` (and `between()` / `notBetween()`): numbers and numeric strings compare - numerically (`"30" > 18`), other strings lexically (`'2024-02-01' > '2024-01-15'`). `null`, booleans, - arrays and a number against a non-numeric string never match. A missing field never matches. +- `$gt`, `$gte`, `$lt`, `$lte` and `between()`: numbers and numeric strings compare numerically + (`"30" > 18`), other strings lexically (`'2024-02-01' > '2024-01-15'`). A numeric string may have + leading and trailing whitespace (`"7 "` is 7). `null`, booleans, arrays and a number against a non-numeric string never match. A missing field never matches. A `NAN` operand never + matches. `notBetween()` returns the complement (see + [between()](#between--notbetween)). - `$eq`, `$ne`, `$in`, `$nin`, `whereIn()`, `whereNot()` and plain `['field' => value]` filters use strict comparison (`===`): `'25'` is not `25`. - A filter value that is an array of operators applies all of them; any other array is compared as a @@ -112,6 +115,11 @@ noneDB supports MongoDB-style comparison operators in `where()` clauses. / `orWhere()` is called. An unknown operator (`$gtee`), `$in`/`$nin` without an array or an invalid `$regex` throws it when the query runs, also on an empty database (filters are compiled once per query, before records are read). +- A `key` filter (`['key' => 5]`, `['key' => '5']`, `['key' => [1, 2]]`) means the same in every clause + (`where()`, `orWhere()`, several `where()` calls, with spatial filters) as in `find()`: integers and + canonical digit strings only (`"5\n"`, `" 5"` and `"05"` are not key 5). Operators on `key` + compare strictly, as on any field: `['key' => ['$eq' => '1']]` and `['key' => ['$in' => ['1']]]` match + nothing, because keys are integers. ### Comparison Examples @@ -178,6 +186,13 @@ $noPhone = $db->query("users") #### Pattern Matching +A pattern is literal except a leading `^` (starts with) and a trailing `$` (ends with: `son$` doesn't match +`"Jackson\n"`). +`$like`, `like()` and `notLike()` compare case-insensitively with Unicode case folding: +`straße` matches `STRASSE`, `İstanbul` matches `istanbul`. Folding is the same for every language, so +the Turkish dotless `ı` and `I` don't match each other. Without the mbstring extension the PCRE `/iu` +rules apply (they miss `ß`/`ss` and `İ`/`i`). + ```php // Contains (case-insensitive) $johns = $db->query("users") @@ -263,7 +278,9 @@ $db->query("users")->where([ ### orWhere() Adds OR groups: a record matches if **all** `where()` groups match **or any** `orWhere()` group matches. -Without `where()`, only the `orWhere()` groups decide. +Without `where()`, only the `orWhere()` groups decide. An empty group (`orWhere([])`) has no condition and +matches every record, so the query returns everything; this is intended (an empty `where([])` doesn't +filter either). ```php // Users who are admins OR have high reputation @@ -298,19 +315,32 @@ $db->query("products") ### whereNot() -Exclude records where **any** of the given fields equals its value (`===`): +Exclude records where **any** of the given fields matches: a plain value matches with `===`; an +operator array matches when **all** of that field's operators match. `whereNot(['role' => 'guest', 'dept' => 'hr'])` drops guests and everyone in HR. Several calls all apply. +A missing field doesn't match a plain value or most operators, so the record is kept; but `$ne`, `$nin` +and `$exists => false` do match a missing field: `whereNot(['role' => ['$ne' => 'admin']])` also drops +records without `role`. ```php // All users except guests $db->query("users") ->whereNot(['role' => 'guest']) ->get(); + +// Everyone except working-age users: all the operators of a field must match for a record to be excluded +$db->query("users") + ->whereNot(['age' => ['$gte' => 18, '$lt' => 65]]) + ->get(); ``` +`whereNot()` checks its operators when it is called: an unknown operator, `$in`/`$nin` without an array +or an invalid `$regex` throws `InvalidArgumentException` at once (`where()` checks them when the query +runs). + ### like() / notLike() -Pattern matching on string fields. +Pattern matching on string and number fields (booleans, arrays and missing fields never match). ```php // Names starting with 'A' @@ -351,12 +381,14 @@ can't be compared (`null`, booleans, text against numbers). ### search() -Full-text search across fields. +Case-insensitive substring search: a record matches when the whole term appears in one of its string +or number field values (booleans and arrays aren't searched; no word splitting or ranking: +`search('php tutorial')` doesn't match "tutorial on php"). ```php // Search in all fields $db->query("articles") - ->search('php tutorial') + ->search('php') ->get(); // Search in specific fields @@ -365,8 +397,9 @@ $db->query("articles") ->get(); ``` -Case-insensitive substring match on scalar values (`İzmir`, `IZMIR` and `izmir` match each other). -Without a field list, every field except `key` is searched. +Matching uses Unicode case folding, as in `like()` (`İzmir`, `IZMIR` and `izmir` match each other; so do +`straße` and `STRASSE`); without the mbstring extension only ASCII letters are folded. Without a field +list, every field except `key` is searched, and also not the `_distance` that a spatial filter adds. --- @@ -374,7 +407,7 @@ Without a field list, every field except `key` is searched. ### sort() -Sort results by field. +Sort results by field (`orderBy()` is an alias). ```php // Ascending (default) @@ -388,6 +421,7 @@ $db->query("products")->sort('price', 'desc')->get(); $db->query("products")->sort('category')->sort('price', 'desc')->get(); // Sort by distance (spatial queries; distances are in meters) +[$lon, $lat] = [28.9784, 41.0082]; $db->query("locations") ->withinDistance('coords', $lon, $lat, 10000) ->withDistance('coords', $lon, $lat) @@ -395,12 +429,22 @@ $db->query("locations") ->get(); ``` -Sorting is stable and uses a total order: missing/`null` first, then booleans, numbers, strings and -arrays (reversed for `desc`). +Sorting is stable and uses a total order, reversed for `desc`: + +1. missing / `null` +2. booleans (`false` < `true`) +3. numbers (int and float together) +4. numeric strings, by their number (`"9"` < `"10"` < `"1e3"`; leading and trailing whitespace allowed) +5. other strings, in byte order (`"1a"` < `"Z"` < `"a"`) +6. arrays (by their JSON) + +The order is transitive (unlike PHP's `<=>` on mixed strings, where `"9"` < `"10"` < `"1a"` < `"9"`), so a result +never depends on the order of insertion. `min()` and `max()` (the builder's and `$db->min()/max()`) use the same +order. ### limit() / offset() -Paginate results. +Paginate results (`skip()` is an alias of `offset()`). ```php // First 10 records @@ -418,6 +462,25 @@ function paginate($db, $dbname, $page, $perPage = 20) { } ``` +`limit(0)` returns nothing: `limit(0)->first()` is `null` and `limit(0)->exists()` is `false`. + +### select() / except() + +Choose the fields of the returned records: `select()` keeps only the given fields, `except()` drops them. + +```php +$db->query("users")->select(['name', 'email'])->get(); // [['name' => ..., 'email' => ..., 'key' => 3], ...] +$db->query("articles")->except(['content'])->get(); +``` + +- `select()` always keeps `key`; `except()` removes it only when it is listed. With both, only + `select()` applies. +- They only shape the result: every matching record is still read in full, and filters, `sort()` and + aggregates see all fields. +- The `_distance` added by `withDistance()` is kept by `select()`; the one that `withinDistance()` and + `nearest()` add is dropped unless selected. +- With `groupBy()` they apply to the group rows and their items (see [groupBy()](#groupby)). + --- ## Aggregation @@ -434,6 +497,9 @@ $activeUsers = $db->query("users") ->count(); ``` +`limit()` and `offset()` apply (`offset(2)->count()` on 6 matches is 4), as for every aggregate below; +with `groupBy()` it counts the groups. + ### sum() / avg() / min() / max() Numeric aggregations. @@ -454,6 +520,23 @@ $minPrice = $db->query("products")->min('price'); $maxPrice = $db->query("products")->max('price'); ``` +### distinct() + +The distinct values of a field among the matching records, in order of first appearance (records +without the field, or with `null`, are left out). + +```php +$cities = $db->query("users")->where(['active' => true])->distinct('city'); // ['Istanbul', 'Ankara', ...] +``` + +Aggregates (`count`, `sum`, `avg`, `min`, `max`, `distinct`) work on the matching records before +`select()`/`except()`: `select(['name'])->sum('price')` sums the prices. With `groupBy()` only `count()` is defined +(it counts the groups): the others see the group rows, not the records. +`sum()`/`avg()` add numbers and numeric strings (booleans, `null`, arrays and other strings are left out; +the result of none is 0). `min()`/`max()` compare every value that isn't `null` in the sort order above +(so `max()` of `5` and `"x"` is `"x"`) and return `null` when there is none. `distinct()` takes O(n) time; +values are distinct by type (`1`, `1.0` and `"1"` are three values; `-0.0` and `0.0` are one). + --- ## Joins @@ -480,11 +563,25 @@ $db->query("orders")->join("users", "user_id", "key", "buyer")->get(); // $order - The attached record replaces a field of the same name in the result only; stored records are never changed. - If several foreign records have the same `$foreignKey` value, the last one wins. - Joining a database that doesn't exist attaches `null` and creates nothing. +- How values match: + - integers, integral floats and canonical integer strings match each other (`5`, `5.0`, `"5"`), so + `user_id = users.key` works whether `user_id` is stored as a number or a string; + - other floats match only the same float (`1.5` doesn't match `1`); + - other strings match only the same string (`"007"` doesn't match `7`); + - `true` matches only `true` and `false` only `false`; + - `null` and arrays match nothing. +- Joins on `key` read only the needed foreign records, joins on another field look the + values up through that field's index when it has one (`createFieldIndex("users", "email")`). Without + `groupBy()` and when `sort()` doesn't use the joined field, joins run after `sort()`/`limit()/offset()`, + so only the returned records are joined. +- One consistent state: a query with joins holds shared locks on all its databases at once, so + a writer can't change the foreign database half-way through. The locks are taken in a fixed order and + never while waiting for a writer, so queries and writers can't deadlock. ### Multiple Joins ```php -$data = $db->query("order_items") +$data = $db->query("order-items") ->join("orders", "order_id", "key") // $item['orders'] ->join("products", "product_id", "key") // $item['products'] ->get(); @@ -512,8 +609,18 @@ $ordersByStatus = $db->query("orders") // ] ``` -Group values keep their type (`1`, `"1"` and `1.5` are separate groups); records without the field -are grouped under `null`. +Groups come in the order their value first appears; `sort('_group')`, `sort('_count')`, `limit()` and +`offset()` apply to the groups. Group values keep their type (`1`, `"1"` and `1.5` are separate groups); records without the +field are grouped under `null`. `-0.0` and `0.0` are one group, and the `serialize_precision` setting doesn't +merge groups. + +`select()`/`except()` with `groupBy()`: `_group`, `_count` and `_items` select or drop parts of each +group row, and any other field applies to the records in `_items`. + +```php +$db->query("orders")->groupBy('status')->select(['_group', 'total'])->get(); +// [['_group' => 'pending', '_items' => [['total' => 12.5, 'key' => 3], ...]], ...] +``` ### having() @@ -534,188 +641,43 @@ $goodCategories = $db->query("products") ``` Aggregates: `count`, `sum:field`, `avg:field`, `min:field`, `max:field`; operators: `>`, `<`, `>=`, `<=`, -`=`, `!=`. `having()` only applies to groups: without `groupBy()`, or with an unknown aggregate or -operator, the condition is ignored. +`=`, `!=`. `sum`, `avg`, `min` and `max` use the group's numbers and numeric strings: `sum` and `avg` of +none are 0, a group without any has no `min`/`max` and doesn't match. `having()` only applies to groups: +without `groupBy()`, or with an unknown aggregate or operator, the condition is ignored. --- ## Spatial Queries -noneDB supports geospatial queries with R-tree indexing. - -### Creating Spatial Index +Geometries are stored as GeoJSON (`[longitude, latitude]`); distances are in meters. Index management, +the GeoJSON formats, the direct methods (`$db->withinDistance()`, ...) and the exact matching rules are in +[SPATIAL.md](SPATIAL.md). Spatial queries work without an index; `createSpatialIndex()` makes them fast. ```php -// Create index on location field (optional: spatial queries work without it, the index makes them fast) -$db->createSpatialIndex("restaurants", "location"); - -// Check if index exists -$db->hasSpatialIndex("restaurants", "location"); // true/false - -// List all spatial indexes -$db->getSpatialIndexes("restaurants"); // ['location'] - -// Drop spatial index -$db->dropSpatialIndex("restaurants", "location"); -``` - -### GeoJSON Data Format - -```php -// Point -$point = [ - 'type' => 'Point', - 'coordinates' => [28.9784, 41.0082] // [longitude, latitude] -]; - -// LineString -$line = [ - 'type' => 'LineString', - 'coordinates' => [ - [28.97, 41.00], - [28.98, 41.01], - [28.99, 41.02] - ] -]; - -// Polygon -$polygon = [ - 'type' => 'Polygon', - 'coordinates' => [[ - [28.97, 41.00], - [29.00, 41.00], - [29.00, 41.03], - [28.97, 41.03], - [28.97, 41.00] // First point repeated to close - ]] -]; -``` - -### withinDistance() - -Find records within radius of a point. - -```php -// Direct method -$nearby = $db->withinDistance("restaurants", "location", - 28.9784, // longitude - 41.0082, // latitude - 5000 // radius in meters -); - -// Query builder -$nearby = $db->query("restaurants") - ->withinDistance('location', 28.9784, 41.0082, 5000) // 5 km - ->where(['open_now' => true]) - ->get(); -``` - -### withinBBox() - -Find records whose geometry's bounding box overlaps the box (a point inside it, or a line/polygon that -reaches into or touches it). - -```php -// Direct method -$inArea = $db->withinBBox("restaurants", "location", - 28.97, 41.00, // minLon, minLat - 29.00, 41.03 // maxLon, maxLat -); - -// Query builder -$inArea = $db->query("restaurants") - ->withinBBox('location', 28.97, 41.00, 29.00, 41.03) - ->where(['category' => 'cafe']) - ->get(); -``` - -### nearest() - -Find K nearest records. - -```php -// Direct method -$closest = $db->nearest("restaurants", "location", - 28.9784, 41.0082, // reference point - 5 // number of results -); - -// Query builder: the 10 nearest records among those rated 4+ (k is applied after the other filters) -$closest = $db->query("restaurants") - ->nearest('location', 28.9784, 41.0082, 10) - ->where(['rating' => ['$gte' => 4.0]]) - ->withDistance('location', 28.9784, 41.0082) - ->sort('_distance', 'asc') - ->limit(5) - ->get(); -``` - -### withinPolygon() - -Find records within polygon boundary. - -```php -$polygon = [ - 'type' => 'Polygon', - 'coordinates' => [[ - [28.97, 41.00], - [29.00, 41.00], - [29.00, 41.03], - [28.97, 41.03], - [28.97, 41.00] - ]] -]; - -// Direct method -$inPolygon = $db->withinPolygon("restaurants", "location", $polygon); - -// Query builder -$inPolygon = $db->query("restaurants") - ->withinPolygon('location', $polygon) - ->where(['price_range' => ['$lte' => 3]]) - ->get(); -``` - -### withDistance() - -Add `_distance` to every result: meters from the point to the geometry, measured like -`withinDistance()` / `nearest()` (0 for a point inside a polygon, otherwise to the geometry's nearest -point), rounded to 4 decimals. It is computed before sorting and kept by `select()`. - -In the query builder, `withinDistance()` and `nearest()` already add `_distance` (to their own center, -rounded to 2 decimals); `withDistance()` replaces it with the distance to its point. - -```php -$results = $db->query("restaurants") - ->withinDistance('location', $userLon, $userLat, 10000) // 10 km - ->withDistance('location', $userLon, $userLat) - ->sort('_distance', 'asc') - ->get(); - -foreach ($results as $r) { - echo "{$r['name']}: {$r['_distance']} m away\n"; -} -``` - -### Combining Spatial + Operators - -```php -// Food delivery app: find nearby open restaurants with delivery +$polygon = ['type' => 'Polygon', 'coordinates' => [[[28.95, 41.00], [29.01, 41.00], [29.01, 41.03], [28.95, 41.03], [28.95, 41.00]]]]; +[$userLon, $userLat] = [28.9784, 41.0082]; +$db->query("restaurants")->withinDistance('location', 28.9784, 41.0082, 5000)->get(); // within 5 km +$db->query("restaurants")->withinBBox('location', 28.97, 41.00, 29.00, 41.03)->get(); // bounding boxes overlap +$db->query("restaurants")->withinPolygon('location', $polygon)->get(); // entirely inside +$db->query("restaurants")->nearest('location', 28.9784, 41.0082, 10)->get(); // the 10 nearest + +// Spatial filters combine with every other filter; nearest() picks its k after the other filters $restaurants = $db->query("restaurants") ->withinDistance('location', $userLon, $userLat, 5000) - ->where([ - 'open_now' => true, - 'delivery' => true, - 'rating' => ['$gte' => 4.0], - 'price_range' => ['$lte' => 3], - 'cuisine' => ['$in' => ['turkish', 'italian', 'chinese']] - ]) + ->where(['open_now' => true, 'rating' => ['$gte' => 4.0], 'cuisine' => ['$in' => ['turkish', 'italian']]]) ->withDistance('location', $userLon, $userLat) - ->sort('rating', 'desc') + ->sort('_distance', 'asc') ->limit(20) ->get(); ``` +- `withinBBox()` compares bounding boxes: a line or polygon can match without entering the box itself. +- `withinDistance()` and `nearest()` add `_distance` (meters to their own center, 2 decimals); + `withDistance()` adds or replaces it with the distance to its point (4 decimals), before sorting, and + `select()` keeps it. +- The spatial filter runs first, whatever the order of the calls; the other filters are checked on its + results. + --- ## Terminal Methods @@ -738,14 +700,18 @@ Returns first matching record or null. $user = $db->query("users")->where(['email' => 'john@example.com'])->first(); ``` -### count() +### last() -Returns number of matching records. +Returns the last record of the result (after `sort()`, `offset()` and `limit()`) or null. ```php -$count = $db->query("users")->where(['role' => 'admin'])->count(); +$newest = $db->query("posts")->sort('created_at')->last(); ``` +### count() / sum() / distinct() / ... + +The aggregates are terminal methods too; see [Aggregation](#aggregation). + ### exists() Returns boolean indicating if any records match. @@ -767,7 +733,7 @@ $result = $db->query("users") `sort()`, `offset()` and `limit()` choose which records are written (e.g. "the 10 oldest"); `select()`, `except()` and `join()` don't matter. Selecting and writing happen under one database lock. -`update()`/`delete()` can't be combined with `groupBy()` (`LogicException`). +`update()`, `delete()` and `removeFields()` can't be combined with `groupBy()` (`LogicException`). ### delete() @@ -792,8 +758,68 @@ $db->query("users")->where(['legacy' => true])->removeFields(['old_password', 't --- +## Query Planning & explain() + +A query reads only what its filters need: + +- **Indexes narrow the records down.** Plain values and `$eq`, `$in`, `whereIn()` on an indexed field + read the keys from its field index; `$gt`, `$gte`, `$lt`, `$lte` and `between()` use the same index as a + range index. `key` filters (plain value, `$eq`, `$in`) read only those keys. Several `where()` calls + intersect their candidates; `orWhere()` uses indexes when every branch can. Negations (`$ne`, `$nin`, + `whereNot()`, `whereNotIn()`, `notLike()`, `notBetween()`), `$exists`, `$like`/`like()`, `$regex`, + `$contains`, ranges on `key` and `search()` read every record (of the segments that the other filters + don't rule out). Every record read is checked against every filter, so an index only decides what is + read, never what matches. Field indexes are used while `fieldIndexEnabled` is on (the default). +- **Shards are skipped** when their global field index rules the value out (sharded databases). +- **Records are read in batches** (32, then 128, ...) and the scan stops when enough match: + `first()`, `exists()` and `limit()` without `sort()` read only what they need. +- **Top-k:** `sort()` with `offset + limit` up to 1000 keeps only the best rows while scanning instead of + sorting every match. The result is the same as a full sort (ties keep key order). Not used with + `withDistance()`, `groupBy()` or joins that must run before the sort (`explain()` then shows + `"sort" => "full"`). A spatial filter or a plain `key` filter sorts all its matches too, though `explain()` shows + `"top-k"` for it. + +`explain()` shows what a query would do, without reading records or building anything: + +```php +// 2000 users in 4 shards of 500, one city per shard; "city" and "age" are indexed +$plan = $db->query("users")->where(['city' => 'Istanbul', 'age' => ['$gte' => 30]])->limit(10)->explain(); +// [ +// "access" => "index", // index | scan | mixed | spatial | none +// "segments" => ["total" => 4, "skipped" => 3, "indexed" => 1, "scanned" => 0], +// "plan" => [["segment" => 0, "access" => "index", "candidates" => 380], ["segment" => 1, "access" => "skip", ...], ...], +// "estimatedRead" => 380, // candidates of the plan (limit() stops earlier) +// "sort" => "none", // none | top-k | full +// "earlyExit" => true, +// "joins" => [], // per join: db, on, strategy (key-lookup | in-lookup), when (after-slice | before-sort) +// ... // also: explain (1), db, sharded, planTruncated, filter, residual, spatial, groupBy +// ] + +$plan = $db->query("users")->where(['city' => 'Istanbul'])->explain(true); // also runs the query +// $plan['actual'] = ["recordsRead" => 500, "segmentsRead" => 1, "returned" => 500, "ms" => 1.2] +``` + +An index that a query would build first (missing or out of date) counts as a scan in `explain()`; the +query itself builds it. + +--- + ## Real-World Examples +The values a request gives: + +```php +$searchTerm = 'laptop'; +$page = 0; +$email = 'jane@example.com'; +$password = 'what the user typed'; +[$startDate, $endDate] = ['2026-10-01', '2026-10-31']; +[$userLon, $userLat] = [28.9784, 41.0082]; +$maxBudget = 50; +$selectedCategories = ['plumbing', 'electrics']; +$term = 'php'; +``` + ### E-commerce Product Search ```php @@ -816,13 +842,9 @@ $products = $db->query("products") ```php $user = $db->query("users") - ->where([ - 'email' => $email, - 'password_hash' => $passwordHash, - 'status' => 'active', - 'email_verified' => true - ]) + ->where(['email' => $email, 'status' => 'active', 'email_verified' => true]) ->first(); +$ok = $user !== null && password_verify($password, $user['password_hash']); ``` ### Analytics Dashboard @@ -877,7 +899,7 @@ $articles = $db->query("articles") // Search with category filter $results = $db->query("articles") ->where(['category' => ['$in' => ['tech', 'science']]]) - ->search($query, ['title', 'content', 'excerpt']) + ->search($term, ['title', 'content', 'excerpt']) ->sort('published_at', 'desc') ->get(); ``` @@ -889,14 +911,16 @@ $results = $db->query("articles") ### 1. Use Indexes ```php -// Create field index for frequently queried fields +// Field indexes for fields you filter on with plain values, $in, whereIn(), ranges or between() $db->createFieldIndex("users", "email"); $db->createFieldIndex("products", "category"); -// Create spatial index for location queries +// Spatial index for location queries (the R-tree instead of scanning every record) $db->createSpatialIndex("locations", "coords"); ``` +`explain()` shows whether a query uses them (see [Query Planning & explain()](#query-planning--explain)). + ### 2. Limit Results ```php @@ -904,49 +928,24 @@ $db->createSpatialIndex("locations", "coords"); $db->query("logs")->limit(100)->get(); // Use first() instead of get()[0] -$db->query("users")->where(['id' => $id])->first(); +$db->query("users")->where(['email' => 'jane@example.com'])->first(); ``` -### 3. Select Only Needed Fields +### 3. Filters Decide What Is Read -```php -// Only fetch name and email -$db->query("users")->select(['name', 'email'])->get(); - -// Exclude large fields -$db->query("articles")->except(['content', 'raw_html'])->get(); -``` +The order of the calls doesn't matter: every `where()`, `whereIn()` and `between()` filter is planned +together, and filters on indexed fields narrow down what is read. `select()`/`except()` only shape the +returned arrays; they don't reduce reading. -### 4. Filter Early +### 4. Batch Operations ```php -// Good: Filter with where() first -$db->query("orders") - ->where(['status' => 'completed']) // Filter first - ->between('total', 100, 500) // Then range - ->sort('created_at', 'desc') - ->get(); -``` - -### 5. Use Spatial Indexes for Location Queries - -```php -// Create the spatial index once -$db->createSpatialIndex("restaurants", "location"); - -// Then queries use the R-tree instead of scanning every record -$db->withinDistance("restaurants", "location", $lon, $lat, 5000); // 5 km (meters) -``` - -### 6. Batch Operations - -```php -// Batch insert instead of individual inserts -$db->insert("logs", $arrayOf1000Records); // Single operation +// Batch insert instead of individual inserts: one operation, one lock, one append per shard +$db->insert("logs", array_map(function ($i) { return ['n' => $i, 'level' => 'info']; }, range(1, 1000))); // Batch update $db->query("orders") - ->where(['status' => 'pending', 'created_at' => ['$lt' => $oldDate]]) + ->where(['status' => 'pending', 'created_at' => ['$lt' => '2026-01-01']]) ->update(['status' => 'expired']); ``` @@ -956,6 +955,7 @@ $db->query("orders") ### Comparison Operators + ```php ['$gt' => value] // Greater than ['$gte' => value] // Greater than or equal @@ -973,8 +973,9 @@ $db->query("orders") ### Spatial Methods + ```php -->withinDistance($field, $lon, $lat, $meters) +->withinDistance($field, $lon, $lat, $distanceMeters) ->withinBBox($field, $minLon, $minLat, $maxLon, $maxLat) ->nearest($field, $lon, $lat, $k) ->withinPolygon($field, $polygon) @@ -983,6 +984,7 @@ $db->query("orders") ### Filter Methods + ```php ->where([...]) ->orWhere([...]) @@ -998,42 +1000,34 @@ $db->query("orders") ### Modifier Methods + ```php -->sort($field, 'asc'|'desc') +->sort($field, 'asc'|'desc') // alias: orderBy() ->limit($count) -->offset($count) +->offset($count) // alias: skip() ->select([...]) ->except([...]) -->join($db, $localKey, $foreignKey) +->join($foreignDb, $localKey, $foreignKey, $alias = null) ->groupBy($field) ->having($aggregate, $operator, $value) ``` ### Terminal Methods + ```php -->get() // Array of records -->first() // Single record or null -->count() // Integer count -->exists() // Boolean -->sum($field) // Numeric sum -->avg($field) // Numeric average -->min($field) // Minimum value -->max($field) // Maximum value -->update([...])// Update result -->delete() // Delete result +->get() // Array of records +->first() / ->last() // Single record or null +->count() // Integer count +->exists() // Boolean +->sum($field) // Numeric sum +->avg($field) // Numeric average +->min($field) // Minimum value +->max($field) // Maximum value +->distinct($field) // Distinct values +->explain($analyze) // The query plan +->update([...]) // ['n' => ...] +->delete() // ['n' => ...] +->removeFields([...]) // ['n' => ..., 'fields_removed' => [...]] ``` ---- - -## Version History - -| Version | Changes | -|---------|---------| -| 3.2.0 | `where()` calls combine with AND; `orWhere()` without `where()` fixed; type-aware comparisons; unknown operators throw; safe `like`/`$regex`; multi-key stable `sort()`; `nearest()` applies k after filters; `withDistance()` before sort and measured like `withinDistance()`; typed `groupBy()`; `removeFields()` on stored records only | -| 3.1.0 | Added comparison operators ($gt, $gte, $lt, $lte, $ne, $eq, $in, $nin, $exists, $like, $regex, $contains) | -| 3.1.0 | Added spatial indexing with R-tree (withinDistance, withinBBox, nearest, withinPolygon) | -| 3.1.0 | Added withDistance() for distance calculations | -| 3.0.0 | JSONL storage with O(1) key lookup | -| 3.0.0 | Single-pass filtering optimization | -| 3.0.0 | Static cache sharing across instances | diff --git a/docs/SPATIAL.md b/docs/SPATIAL.md index f23f695..8e61fff 100644 --- a/docs/SPATIAL.md +++ b/docs/SPATIAL.md @@ -1,8 +1,8 @@ # noneDB Spatial Index Reference -Comprehensive documentation for noneDB's geospatial capabilities using R-tree indexing. +GeoJSON geometries, the R-tree spatial index and spatial queries in noneDB. -**Version:** 3.2.0 +Geospatial indexes and queries of noneDB 4.0.0. The examples run in order (CI runs them). --- @@ -16,18 +16,20 @@ Comprehensive documentation for noneDB's geospatial capabilities using R-tree in 6. [Combining Spatial with Filters](#combining-spatial-with-filters) 7. [Performance Optimization](#performance-optimization) 8. [Real-World Examples](#real-world-examples) +9. [Error Handling](#error-handling) --- ## Overview -noneDB v3.1 introduces spatial indexing with R-tree data structure for efficient geospatial queries. Features include: - **R-tree indexing** with O(log n) query performance -- **GeoJSON support** for Point, LineString, Polygon, and Multi* types -- **Distance calculations** using Haversine formula +- **GeoJSON support** for Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon and + GeometryCollection +- **Distances in meters**, using the Haversine formula (earth radius 6,371 km) - **Query builder integration** for combining spatial + attribute filters -- **Automatic CRUD synchronization** - index updates on insert/update/delete +- **Always current:** queries see every insert, update and delete; the index catches up from the data + when it is loaded (see [R-tree Index Structure](#r-tree-index-structure)) --- @@ -35,17 +37,21 @@ noneDB v3.1 introduces spatial indexing with R-tree data structure for efficient ### createSpatialIndex() -Create an R-tree spatial index on a geometry field. +Create an R-tree spatial index on a geometry field. A missing database is created when `autoCreateDB` +is on. Field names may contain any characters (`home-loc`, `konum.ev`); `''` and `key` are rejected. ```php $db = new noneDB(); // Create spatial index on 'location' field $result = $db->createSpatialIndex("restaurants", "location"); -// Returns: ["success" => true, "indexed" => 150] +// Returns: ["success" => true, "indexed" => 150, "error" => null] // If index already exists: -// Returns: ["success" => false, "indexed" => 0, "error" => "Spatial index already exists..."] +// Returns: ["success" => false, "indexed" => 0, "error" => "Spatial index already exists for field 'location'"] + +// Optional: entries per R-tree node (an integer of at least 4, default 32); rebuilds keep it +$result = $db->createSpatialIndex("zones", "boundary", ['nodeSize' => 16]); ``` **Important:** Records with invalid or missing GeoJSON are silently skipped during indexing: @@ -65,6 +71,7 @@ $result = $db->createSpatialIndex("places", "location"); echo "Indexed: {$result['indexed']} records\n"; // May be 0! // To validate before insert: +$geometry = ['type' => 'Point', 'coordinates' => [28.9784, 41.0082]]; $validation = $db->validateGeoJSON($geometry); if (!$validation['valid']) { echo "Error: " . $validation['error']; @@ -78,6 +85,9 @@ Check if a spatial index exists. ```php $exists = $db->hasSpatialIndex("restaurants", "location"); // Returns: true or false + +// With a shard id: true when the index exists and the database has that shard +$exists = $db->hasSpatialIndex("restaurants", "location", 2); ``` ### getSpatialIndexes() @@ -95,18 +105,30 @@ Remove a spatial index. ```php $result = $db->dropSpatialIndex("restaurants", "location"); -// Returns: ["success" => true] +// Returns: ["success" => true, "error" => null], also when there was no such index +``` + +### rebuildSpatialIndex() + +Delete the index files and build them again from the data, with the same `nodeSize`. Returns the same +array as `createSpatialIndex()`; a field without an index gets one. + +```php +$result = $db->rebuildSpatialIndex("restaurants", "location"); +// Returns: ["success" => true, "indexed" => 150, "error" => null] ``` --- ## GeoJSON Data Format -noneDB supports standard GeoJSON geometry types. Coordinates are `[longitude, latitude]`. +noneDB supports standard GeoJSON geometry types. Coordinates are `[longitude, latitude]`. A geometry +that `validateGeoJSON()` rejects is stored but never indexed or matched (see +[Error Handling](#error-handling)). ### Point -A single location. +A single location. A third value (altitude) is accepted and ignored. ```php $location = [ @@ -142,7 +164,8 @@ $db->insert("routes", [ ### Polygon -An area with boundary. First and last points must be identical (closed ring). +An area with boundary. A ring needs at least 4 positions, its first and last points must be identical +(closed ring), and it must not intersect itself. ```php $area = [ @@ -164,7 +187,7 @@ $db->insert("zones", [ ### Polygon with Hole -Polygon with inner exclusion zone. +Polygon with inner exclusion zone. Every vertex of a hole must lie inside the outer ring or on it. ```php $parkWithLake = [ @@ -221,6 +244,21 @@ $zones = [ ]; ``` +### GeometryCollection + +Geometries of any type in one value. An empty collection is valid but has no location: it is never +indexed and never matches a spatial query. + +```php +$site = [ + 'type' => 'GeometryCollection', + 'geometries' => [ + ['type' => 'Point', 'coordinates' => [28.98, 41.00]], + ['type' => 'LineString', 'coordinates' => [[28.97, 41.00], [28.98, 41.01]]] + ] +]; +``` + --- ## Spatial Query Methods @@ -252,15 +290,17 @@ $nearby = $db->withinDistance("restaurants", "location", 28.9784, 41.0082, 5000, 'includeDistance' => true // Add _distance field to results ]); -// Each result now has _distance in meters +// Each result now has _distance in meters, rounded to 2 decimals foreach ($nearby as $r) { echo "{$r['name']}: {$r['_distance']} meters\n"; } ``` Distance is measured to the **nearest point of the geometry**: 0 for a point inside a polygon, the -closest vertex or edge otherwise (a Point is just that point). Results are sorted by distance. -Circles that cross the antimeridian (±180°) or reach a pole are searched correctly. +closest vertex or edge otherwise (a Point is just that point; an edge is straight in longitude/latitude). +The distance is exact, also to an edge across the antimeridian from the center. Results are sorted by the distance rounded to centimeters (`_distance`), +ties by key; `nearest()` takes its k in that order. Circles that cross the antimeridian (±180°) or +reach a pole are searched correctly. ### withinBBox() @@ -278,7 +318,9 @@ $inArea = $db->withinBBox( $pacific = $db->withinBBox("ships", "location", 170, -10, -170, 10); ``` -Returns geometries whose bounding box overlaps the box. +Compares bounding boxes: returns the records whose geometry's bounding box overlaps or touches the box, +so a line or polygon can match without entering the box itself. The latitudes may be given in either +order; `minLon > maxLon` means a box across the antimeridian. ### nearest() @@ -307,11 +349,19 @@ $closest = $db->nearest("restaurants", "location", 28.9784, 41.0082, 5, [ ``` `nearest()` adds `_distance` unless `'includeDistance' => false`; `withinDistance()` adds it only with -`'includeDistance' => true`. +`'includeDistance' => true`. Both round it to 2 decimals. The options are exactly these, `includeDistance` a +boolean and `maxDistance` a number or `null`; another option or another type is an `InvalidArgumentException`. +The same holds for `createSpatialIndex()`'s `nodeSize`, an integer of at least 4. With `maxDistance`, the records within that distance are returned even when there are fewer than k. `k = 0` returns `[]`; with fewer than k records in total, all of them are returned. +Arguments that aren't usable numbers: +- a NaN longitude, latitude, radius or `maxDistance` returns `[]` in every spatial query; +- an infinite radius or `maxDistance` means "the whole earth": every geometry; +- infinite box coordinates are clamped to ±180/±90; +- an infinite center, a negative radius or a negative `maxDistance` returns `[]`. + ### withinPolygon() Find records within a polygon boundary. @@ -331,15 +381,39 @@ $polygon = [ $inPolygon = $db->withinPolygon("restaurants", "location", $polygon); ``` -`$polygon` may be a Polygon (holes are excluded) or a MultiPolygon. A geometry matches when it lies -entirely inside: every vertex inside and no edge crossing the boundary (a line that leaves and re-enters -a concave polygon does not match). +`$polygon` may be a Polygon (holes are excluded) or a MultiPolygon; anything else, or an invalid +polygon, returns `[]`. A geometry matches when it lies entirely inside: every vertex inside and no edge +crossing the boundary (a line that leaves and re-enters a concave polygon does not match). + +Rules: +- **The polygon is closed:** points on its boundary are inside, on every edge. For points, a rectangle + matches the same records as `withinBBox()` (which also returns lines and polygons that only overlap + the box). +- **Holes are respected:** a record that covers a hole of the query polygon (a polygon whose area + holds it, a line through it) is not within the polygon. A ring that goes around a hole without covering it is. A record may touch a hole's boundary. +- **MultiPolygon query:** each component of a multi-part record (each point of a MultiPoint, each part + of a MultiPolygon or GeometryCollection) may lie in a different part. A single component that spans + two adjacent parts doesn't match: the union of the parts isn't computed. +- **Empty GeometryCollection:** never within anything; empty members of a collection are ignored. +- A polygon can't cross the antimeridian (GeoJSON coordinates are plain lon/lat); split it into a + MultiPolygon at ±180°. +- Containment uses floating-point arithmetic: a point exactly on an edge counts as inside when its + coordinates are exact in binary (integers, halves, ...); a decimal coordinate such as 41.02 on an edge + can land on either side of it. + +### haversineDistance() + +Distance in meters between two points (Haversine formula, earth radius 6,371 km). + +```php +$meters = $db->haversineDistance(28.9784, 41.0082, 29.0, 41.0); // lon1, lat1, lon2, lat2 +``` --- ## Query Builder Integration -All spatial methods are available in the query builder. +The four spatial filters and `withDistance()` are available in the query builder (rules below). ### withinDistance() @@ -389,16 +463,30 @@ $results = $db->query("restaurants") // Each result has _distance in meters, computed before sort() ``` -`_distance` is measured like the spatial filters: 0 for a point inside a polygon, otherwise to the -geometry's nearest point (4 decimals). In the query builder `withinDistance()` and `nearest()` already -add `_distance` to their own center (2 decimals); `withDistance()` replaces it with the distance to its -point. +### Rules + +- **The spatial filter runs first**, whatever the order of the calls; `where()` and the other filters + are then checked on its results (field indexes are not used for them). +- `withinBBox()` compares bounding boxes and `withinPolygon()` requires the geometry to lie inside, as + the direct methods do. +- `withinDistance()` and `nearest()` add `_distance`: meters to their own center, rounded to 2 + decimals. `nearest()` takes its k after the other filters and has no `maxDistance` option; use + `withinDistance($field, $lon, $lat, $max)->nearest($field, $lon, $lat, $k)`. +- `withDistance()` adds `_distance`, or replaces it, with the distance to its point, measured like the + spatial filters (0 for a point inside a polygon, otherwise to the geometry's nearest point) and + rounded to 4 decimals. It is computed before `sort()`, and `select()` keeps it (it drops the + `_distance` of `withinDistance()`/`nearest()`). +- Several spatial filters must all match, except that of several `nearest()` filters only the smallest `k` and + the last one's order apply. With `nearest()` among them, its order and `_distance` are + used: `withinDistance(A)->nearest(B)` orders and cuts by the distance to B. Of two `withinDistance()` + filters, the first one's `_distance` is kept. +- A record whose field holds no geometry gets no `_distance` from `withDistance()`. --- ## Combining Spatial with Filters -The real power comes from combining spatial queries with attribute filters using comparison operators. +Spatial filters combine with `where()` and its comparison operators. ### Spatial + Simple Where @@ -480,25 +568,7 @@ $results = $db->query("restaurants") ->get(); ``` -### Complex Combined Query - -```php -// Food delivery app: nearby, open, delivers, good rating, affordable -$restaurants = $db->query("restaurants") - ->withinDistance('location', $userLon, $userLat, 5000) // 5000 meters - ->where([ - 'open_now' => true, - 'delivery' => true, - 'rating' => ['$gte' => 4.0], - 'price_range' => ['$lte' => 3], - 'cuisine' => ['$in' => ['turkish', 'italian', 'chinese']], - 'review_count' => ['$gt' => 20] - ]) - ->withDistance('location', $userLon, $userLat) - ->sort('rating', 'desc') - ->limit(20) - ->get(); -``` +A complete combined query is in [Food Delivery App](#food-delivery-app). --- @@ -506,12 +576,27 @@ $restaurants = $db->query("restaurants") ### R-tree Index Structure -noneDB uses an optimized R-tree with: - -- **Node size:** 32 entries per node (reduces tree depth) -- **Linear split algorithm:** O(n) instead of O(n²) -- **Parent pointer map:** O(1) parent lookup -- **Dirty flag pattern:** Batched disk writes +noneDB uses an R-tree with at most 32 entries per node (the `nodeSize` option): + +- **Bulk loading (STR):** `createSpatialIndex()` and rebuilds sort the boxes into nodes + filled to 3/4 (24 of 32 entries) instead of inserting one by one. +- **Integer boxes:** coordinates are stored as integers in units of 1e-7 degrees (about 1 cm), rounded + outwards, so a box always covers its geometry, whatever `serialize_precision` is. +- **A checkpoint, not a file per write:** a write appends to the data file only; the R-tree file + (`spatial/{seg}.{token}.rx` in the generation, STORAGE_FORMAT.md §6.5) covers the data up to a byte offset and the lines after it are added + when the index is loaded. Once the new data reaches both 64 KiB and a quarter of the index file's + size, the process that loads the R-tree writes a new checkpoint (a writer only if it has the R-tree + loaded). +- **Sharded databases:** one R-tree per shard (`spatial/s{N}.{token}.rx`) and one global file per field + (`spatial/{token}.grx`) with each shard's bounding box. A shard is skipped + only when its box and its newer lines can't match; a shard without an entry is always searched. + Indexes follow the automatic migration to shards and cover shards created later. +- **File names:** `{token}` is the field name when it matches `[a-z0-9_]{1,40}`, otherwise a lower-case + prefix and 12 hex digits of its SHA-256 (`home-loc` → `home_loc-17e07f2f0d56`). +- **Fewer checks:** candidates found in an R-tree are not validated again (it only holds geometries that + passed validation); candidates from a scan are. + +Measurements (the spatial workloads of the performance gate): [PERFORMANCE.md](PERFORMANCE.md). ### Best Practices @@ -524,29 +609,20 @@ noneDB uses an optimized R-tree with: 2. **Use withinDistance for radius search** ```php // Good: Uses R-tree to narrow candidates + [$lon, $lat] = [28.9784, 41.0082]; $db->withinDistance("locations", "coords", $lon, $lat, 10000); // 10000 meters ``` -3. **Combine spatial with where for efficiency** +3. **Combine spatial with where** ```php - // Spatial filter first, then attribute filter + // The spatial filter runs first (whatever the call order); where() is checked on its results $db->query("locations") - ->withinDistance('coords', $lon, $lat, 5000) // 5000 meters, spatial first - ->where(['active' => true]) // Then filter + ->withinDistance('coords', $lon, $lat, 5000) // 5000 meters + ->where(['active' => true]) ->get(); ``` -4. **Use withDistance + sort for distance-ordered results** - ```php - $db->query("locations") - ->withinDistance('coords', $lon, $lat, 10000) // 10000 meters - ->withDistance('coords', $lon, $lat) - ->sort('_distance', 'asc') - ->limit(10) - ->get(); - ``` - -5. **Use nearest() for K-nearest queries** +4. **Use nearest() for K-nearest queries** ```php // More efficient than withinDistance + limit for finding closest $db->nearest("locations", "coords", $lon, $lat, 5); @@ -563,6 +639,21 @@ noneDB uses an optimized R-tree with: Where n = total records, k = matching records. +### Validation Cost + +Validation costs O(n log n) for any polygon (n = its vertices): a ring is +valid when no two non-adjacent edges share a point, a hole when each of its vertices is inside or on the +outer ring, computed with exact orientation tests. Usual rings take a quick sort-and-sweep; rings built +to defeat it (e.g. a comb whose edges all overlap in x) and polygons with many holes switch to a sweep +line over a balanced tree. + +Inserts and updates don't validate. Geometries are validated when an R-tree is built or adds new data +lines (the first query of a process after a write), and by queries that don't find them in an R-tree +(a scan), so polygons with tens of thousands of vertices can still cost noticeable +time per query (`validateGeoJSON()` of one polygon, PHP 8.2: 0.06 s for a 20,000-vertex circle, 1.5 s +for a 16,000-edge comb, 6.5 s for a 64,000-edge comb). `withinPolygon()` then compares each candidate +with the query polygon: O(n·m) for a candidate with n and a query with m vertices. + --- ## Real-World Examples @@ -681,7 +772,6 @@ class GeofenceService { return false; } - // Check if current location is within zone $zone = $this->db->query("zones") ->where(['key' => $user['assigned_zone']]) ->first(); @@ -690,24 +780,15 @@ class GeofenceService { return false; } - // Use polygon intersection + // Store the current position, then check it against the zone (Polygon or MultiPolygon). + // A spatial index on users.position (created once) keeps the check fast. $point = ['type' => 'Point', 'coordinates' => [$userLon, $userLat]]; + $this->db->update("users", [['key' => $userId], ['set' => ['position' => $point]]]); - return $this->isPointInPolygon($point, $zone['boundary']); - } - - private function isPointInPolygon($point, $polygon) { - // Create temp record - $this->db->insert("temp_check", ['location' => $point]); - $this->db->createSpatialIndex("temp_check", "location"); - - $result = $this->db->withinPolygon("temp_check", "location", $polygon); - - // Cleanup - $this->db->delete("temp_check", []); - $this->db->dropSpatialIndex("temp_check", "location"); - - return count($result) > 0; + return $this->db->query("users") + ->withinPolygon('position', $zone['boundary']) + ->where(['key' => $userId]) + ->exists(); } } ``` @@ -731,6 +812,9 @@ if (!$validation['valid']) { | Error | Cause | Solution | |-------|-------|----------| | "Spatial index already exists for field 'x'" | Duplicate createSpatialIndex | Check with hasSpatialIndex first | +| "Spatial indexing is disabled" | `spatialIndexEnabled` is false (create/rebuild; existing indexes are kept, queries scan) | Enable it | +| "Invalid field name" | Field `''` or `key` | Use another field | +| "Database does not exist" | `autoCreateDB` is false | Create the database first | | "Missing type property", "Unknown geometry type: X", "Coordinates must be an array", ... | Malformed geometry | Validate with validateGeoJSON() | | "Ring 0 must be closed (first and last positions must be identical)" | Polygon not closed | Ensure first == last point (compared numerically: `29` equals `29.0`) | | "Ring 0 must have at least 4 positions" | Too few positions | A closed ring needs 4+ positions | @@ -741,23 +825,8 @@ if (!$validation['valid']) { | "Latitude must be between -90 and 90" | lat > 90 or < -90 | Use valid coordinates | Queries work without a spatial index too (they scan the records); an index only makes them faster. -Records whose geometry is invalid are not indexed and never match spatial queries. - -### Sharded databases and field names - -Spatial indexes work the same on sharded databases (one R-tree per shard plus each shard's bounding -box for shard skipping), follow automatic migration to shards, and cover shards created later. -Field names may contain any characters (`home-loc`, `konum.ev`). - ---- - -## Version History +Records whose geometry is invalid are not indexed and never match spatial queries; such records can be +inserted and are kept. Invalid are also: flat coordinates (`[1, 2]` for a LineString), string-keyed +coordinates (`{"a": ...}`) and a non-string `type` (`true`). +Validation costs: see [Validation Cost](#validation-cost). -| Version | Changes | -|---------|---------| -| 3.2.0 | Sharded databases supported; buffered inserts indexed; antimeridian/pole searches; nearest() maxDistance returns partial results; distance to the nearest point of a geometry; MultiPolygon queries; edge-aware withinPolygon; stricter validation; crash-safe index maintenance | -| 3.1.0 | Initial spatial indexing with R-tree | -| 3.1.0 | GeoJSON validation | -| 3.1.0 | withinDistance, withinBBox, nearest, withinPolygon | -| 3.1.0 | Query builder spatial integration | -| 3.1.0 | Performance optimizations (parent pointers, linear split) | diff --git a/docs/STORAGE_FORMAT.md b/docs/STORAGE_FORMAT.md new file mode 100644 index 0000000..d118bf6 --- /dev/null +++ b/docs/STORAGE_FORMAT.md @@ -0,0 +1,770 @@ +# noneDB 4 Storage Format + +**Storage format:** 4 · **Engine:** noneDB 4.0.0 · **Status:** normative + +This document is the contract for every file noneDB 4 reads or writes. The engine recognises +only what is described here. Anything else in a store directory is a *foreign entry* (§3.4) and +is never interpreted. Sync requirements and operation outcomes are in [DURABILITY.md](DURABILITY.md). + +--- + +## 1. Terms + +| Term | Meaning | +|---|---| +| store | The configured `dbDir`. It holds the databases of one or more secret keys. | +| key | A configured `secretKey`. | +| keyCheck | `substr(hash_hmac('sha256', 'nonedb4/key', key), 0, 32)`. It identifies a key without revealing it. | +| locator | `substr(hash_hmac('sha256', 'nonedb4/db/' . name, key), 0, 32)`: the directory name of a database. It is stable for a name and key, unpredictable without the key, and changes when the key is rotated. | +| databaseId | A random UUIDv4 assigned once when a database is created. It never changes: it survives key rotation, backup and restore. | +| generation | A complete, self-consistent file set of one database layout. `CURRENT` names the active one. | +| segment | One data file with its key-index checkpoint: `main` (single layout) or `s{N}` (shard N of the sharded layout). | +| lock epoch | 8 lowercase hex digits in the lock record (§4.2). They change before any file a process may have cached is replaced. | +| segment epoch `e` | 8 lowercase hex digits in a key-index checkpoint. They change whenever the segment's data file is replaced or its index rebuilt without a trusted predecessor. | + +## 2. Encodings + +- **Metadata files** (`STORE`, `FORMAT`, `CURRENT`, `metadata`, `shards.json`, `indexes.json`, journals, manifests): + - UTF-8 JSON on a single line, terminated by `"\n"` (`indexes.json` alone has no final `"\n"`). + - Encoded with `JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION` and `serialize_precision = -1`. + - Object members are written in the order listed in this document. +- **Checksum:** + - `checksum` is the last member: the lowercase hex SHA-256 of the encoding of the object **without** its `checksum` member, followed by `"\n"`. + - To verify, use the bytes as written: the file must end with `,"checksum":"<64 hex>"}\n`; the bytes before `,"checksum"` followed by `}\n` must hash to it. A re-encoding is never compared (an empty object would come back as `[]`). + - A file that does not decode, lacks a required member, has a wrong `magic`/`kind`, or fails the checksum is **invalid**. +- **Atomic replacement:** + - The new content is written to `{target}.tmp{8 hex}` in the target's directory. + - That file is flushed and, in durable mode, synced; it is then renamed over the target; in durable mode the directory is synced. + - A temp file left by a crash is never read; until it is removed it only takes disk space. It is removed by the next writer that owns its directory exclusively and works on that file: the temp of a STORE copy or of the maintenance journal by the next write of that file under `STORE.LOCK` held exclusively, and the journal's also when a maintenance closes (an abort writes none); the temp of a segment's data file by the segment's next compaction (§6.2); every temp of a database, a key index's too, by the recovery of a session that finds `j`/`J` (§7.3); a generation's with the generation (§5.2). No other operation lists a directory for them. +- **Data and buffer lines:** see §6. + +## 3. Store directory + +```text +{dbDir}/ + STORE store record, copy 1 (§3.1) + STORE.2 store record, copy 2 + STORE.LOCK store lock (flock only, empty; never replaced or deleted) + .htaccess, index.html deny web access (written when the store is created) + {locator}.ndb4/ one directory per database (§4) + .nonedb-maintenance/ maintenance journal, migration staging (§9) and log.jsonl: store-level events + (copy repaired, store restored or rebuilt; JSONL, no key material) + .nonedb-retained/ migrated source files awaiting cleanup (§9.6) +``` + +- **Path:** `dbDir` is used by its canonical path: symbolic links and `.`/`..` are resolved when an instance is + configured (for a directory that doesn't exist yet, its nearest existing parent is). Two spellings of one + directory are one store, with one lock per file in a process. +- **Permissions:** every file noneDB creates in `dbDir` gets `filePermissions` (default `0640`), every + directory `directoryPermissions` (default `0750`), whatever the process's umask (CONFIGURATION.md). Files that + exist keep theirs. Backups are `0600`/`0700` (§10.1). + +### 3.1 STORE record + +```json +{"magic":"NONEDB4","kind":"store","storageFormat":4,"storeId":"","seq":7,"keys":["",...],"checksum":""} +``` + +- `keys` is sorted and has no duplicates. +- Every write goes through `STORE.LOCK` held exclusively. It writes `STORE.2` and then `STORE`, both with the same content and a `seq` one higher than before, each by atomic replacement. +- **Invariant:** while no maintenance journal is open, both copies are valid and identical. + +**Reading, with no open maintenance journal (§9.1):** + +| Copies | Result | +|---|---| +| Both valid and identical | Use them. | +| One valid; the other missing or invalid | The invariant says they were identical. The missing or invalid copy is rewritten from the valid one under `STORE.LOCK` (held exclusively; a process that holds it shared uses the valid copy and leaves the repair to the next opening), and a `store-copy-repaired` event is appended to `.nonedb-maintenance/log.jsonl`. This also covers a first creation interrupted between the two copies. | +| Both valid, but `seq` or content differ | The invariant is broken: `FORMAT_DAMAGED`. Nothing is guessed. | +| None valid | `FORMAT_DAMAGED`. Only `repairStore()` can fix it (§9.4). | + +### 3.2 Opening a store + +The engine opens a store once per process and caches the result. The cache is re-checked on the paths named in §3.3. + +1. **A STORE copy exists:** read it by §3.1. + - If the configured key's keyCheck is listed, the store is open. + - Otherwise: `KEY_NOT_REGISTERED`. No file is created. +2. **No STORE copy exists:** scan `dbDir` read-only. + - Allowed entries: `.htaccess`, `index.html`, `.gitkeep`, `.gitignore`, `.DS_Store`, `STORE.LOCK`, and `STORE.tmp*` / `STORE.2.tmp*`. + - Any other entry: if `STORE.LOCK` exists, step 1 is repeated holding it shared (a store created by another process since the first read is found there; its creation holds the lock exclusively). Still no STORE copy: `UNSUPPORTED_FORMAT`; if the entries include noneDB 4 objects (`*.ndb4`, `.nonedb-maintenance`), the store record itself is missing: `FORMAT_DAMAGED` (see `repairStore()`). **No file or directory is created on this path.** +3. **Empty, or `dbDir` does not exist:** create the store. + 1. Create `dbDir` if needed. + 2. Open or create `STORE.LOCK` and lock it exclusively. + 3. Re-check step 1: another process may have created the store. + 4. Re-scan with the step-2 allow-list. + 5. Remove stale `STORE*.tmp*` files. + 6. Write the STORE record with `seq` 1 and this keyCheck (two copies). + 7. Write `.htaccess` and `index.html`. + 8. Unlock. + + `STORE.LOCK` never makes a directory non-empty. A crash before the STORE record leaves a directory that the next opening still treats as empty. + +### 3.3 Re-validation of an open store + +The STORE record and the journal check (§9.1) are read again: + +- after a database's lock epoch changed (the cache-invalidation path); +- every time the engine is about to answer "this database does not exist", or to create a database. This is the missing-database path. It holds `STORE.LOCK` shared, so maintenance (exclusive) cannot run at the same time. + +CRUD on an existing database never reads the STORE record, and lists a directory only in the rare cases of §12. + +### 3.4 Foreign entries + +- **v4 allow-list of a store with a STORE record:** `STORE`, `STORE.2`, `STORE.LOCK`, `STORE*.tmp*`, directories named `[0-9a-f]{32}.ndb4`, `.nonedb-maintenance`, `.nonedb-retained`, `.htaccess`, `index.html`, `.gitkeep`, `.gitignore`, `.DS_Store`. Every other entry is **foreign**. +- **The engine never inspects foreign entries.** On the missing-database path (§3.3), however, it lists `dbDir` directly, every time, with no cache keyed on mtime or inode. If any foreign entry exists: + - the engine answers neither "does not exist" nor creates the database; + - the operation fails with `UNSUPPORTED_FORMAT`; + - the message says the directory holds files that are not noneDB 4 objects, names up to five of them, and says no database is created or answered as missing while they are there. +- Databases that exist are unaffected. Their directories are found without a listing. +- **Links:** a `{locator}.ndb4` entry that is a symbolic link, or not a directory, is never a database. It is + a foreign entry; an operation on its database fails with `UNSUPPORTED_FORMAT` and follows no link; + `getDBs()` skips it. + +## 4. Database directory `{locator}.ndb4/` + +```text +FORMAT identity (§4.1) +LOCK lock + state record (§4.2) +CURRENT active generation (§4.3) +generations/{N}/ generation N (§5) +recovery/ recovery log and quarantine (§7) +``` + +### 4.1 FORMAT + +```json +{"magic":"NONEDB4","kind":"database","storageFormat":4,"databaseId":"","name":"users","shardSize":10000,"createdAt":1793103240,"formatOptions":{},"checksum":""} +``` + +- FORMAT carries no key material. A directory belongs to a key K when `locator(K, FORMAT.name)` equals its name. +- FORMAT is written once, at the end of creation (§8.1). It is never rewritten except by the repair in §8.3 + and by `restore()` over a damaged database (§10.3). +- `shardSize` is fixed at creation: the configuration's `shardSize` applies only to new databases. + +### 4.2 LOCK and the state record + +`LOCK` is opened with `c+` and locked with `flock`. Its inode never changes while the database directory exists, and `dropDB` keeps it. The first 128 bytes are the state record, space padded and overwritten in place: + +```text +record = gen ":" epoch *( " " flag ) +gen = 1*18DIGIT ; generation counter of the lock file, monotonic +epoch = 8 lowercase hex digits +flag = "b" ; the write buffer may hold records + / "c" ; creation in progress (§8.1) + / "d" ; drop in progress (§8.2) + / "m" ; maintenance in progress (§9) + / "g=" N ">" X ; generation switch from N to X in progress (§5.2), 1*9DIGIT each + / "j=" seg *( "," seg ) ; journal: these segments' shard counters may be stale + / "J" ; journal: all segments +seg = "main" / "s" 1*9DIGIT ; a segment (§1) +``` + +- A record that is empty, all NUL bytes, or all spaces is a **new record**. The next exclusive session writes `1:{new epoch}`; its file writes keep generation 1. +- `gen` grows by one in every exclusive session that writes a file of the database. A new or damaged record starts again at 1 with a new epoch: caches are valid only for the same generation **and** epoch, so a restarted counter never validates an old cache. +- A record that does not match the grammar is **damaged**. The next exclusive session treats it as `J`: full recovery of the database (§7.3) and a new epoch. + - It does not treat a damaged record as `g`: a generation that is not `CURRENT` is reported by `checkIntegrity()` and never removed on this path. +- Every operation reads the record when it takes the lock. When the epoch differs from what the process cached, every cache of the database is dropped and `CURRENT` is read again. +- `m` → `MAINTENANCE_IN_PROGRESS`. +- `c`, `d`, `g`, `j` and `J` each trigger their recovery under the exclusive lock before any data is returned (§8, §5.2, §7.3). + +### 4.3 CURRENT + +```json +{"magic":"NONEDB4","kind":"current","databaseId":"","generation":3,"checksum":""} +``` + +`CURRENT` is **valid** when: +- its checksum verifies; +- `databaseId` equals FORMAT's; +- `generations/{generation}/metadata` is valid (§5.1) with the same `databaseId`. + +It is only ever replaced atomically. + +## 5. Generations + +### 5.1 Generation directory `generations/{N}/` + +```text +metadata immutable after creation of the generation +shards.json sharded layout only: shard list and counters (mutable) +indexes.json index definitions (mutable; absent = no indexes) +segments/{seg}.jsonl data (§6) +segments/{seg}.kix key-index checkpoint (§6.4; derived) +indexes/{seg}.{token}.fx field index of a segment (derived) +indexes/{token}.gfx global field index (sharded layout; derived) +spatial/{seg}.{token}.rx R-tree of a segment (derived) +spatial/{token}.grx global spatial index (sharded layout; derived) +buffer/pending.jsonl write buffer +buffer/flushing.{K}.jsonl buffer being flushed from key K on +``` + +**`metadata`:** + +```json +{"magic":"NONEDB4","kind":"generation","storageFormat":4,"databaseId":"","name":"users","shardSize":10000,"createdAt":1793103240,"formatOptions":{},"generation":3,"layout":"sharded","checksum":""} +``` + +- The identity members are a copy of FORMAT's; they let FORMAT be repaired (§8.3). +- `layout` is `single` (one segment, `main`) or `sharded` (segments `s{N}`, keys `[N·shardSize, (N+1)·shardSize)`). + +**`shards.json`:** + +```json +{"kind":"shards","v":1,"shardSize":10000,"nextKey":123456,"totalRecords":120001,"deletedCount":17,"shards":[{"id":0,"count":9998,"deleted":2},...]} +``` + +- Replaced atomically by inserts, deletes, compaction and recounts. An update changes no counter and leaves it. +- Keys are reserved in it before data is appended. +- Its counters are protected by the `j`/`J` journal (§7.3). + +**`indexes.json`:** + +```json +{"kind":"indexes","v":1,"fields":["city"],"spatial":["loc"],"spatialOptions":{"loc":{"nodeSize":32}}} +``` + +- `spatialOptions` is `[]` when no index has options. +- Written after the index files it lists exist; a layout switch writes it first and builds them after PUBLISH. +- An empty or invalid file is derived again from the headers of the generation's `.fx`/`.rx` files. + +**Field tokens:** +- A field name matching `[a-z0-9_]{1,40}` is its own token. +- Otherwise the token is a lower-case prefix (non-`[a-z0-9_]` characters become `_`, at most 24 characters), then `-`, then the first 12 hex digits of the name's SHA-256. +- Every index header carries the exact field name, and a header whose field differs is ignored. + +### 5.2 Generation switch (state machine) + +A generation switch is used for: +- the change from the single to the sharded layout: PREPARE writes the shards one at a time, each read from the + single segment by its key range, with its key-index checkpoint (§6.4) covering the whole file, and verifies that + every live record is a line of its shard; the field and spatial indexes are built after PUBLISH; +- `restore()` of a present database (§10.3). + +Creation has its own steps (§8.1), without INTENT or `g`. + +Every step runs while `LOCK` is held exclusively. In durable mode, every persistent write is synced as in DURABILITY.md. + +| Step | Action | After a crash here | +|---|---|---| +| PREPARE | Build `generations/X` completely (X = N+1) and verify it (a layout switch: every live record is a line of its shard; a restore: checksums, `checkIntegrity()`, expected record count). CURRENT still names N. A PREPARE that fails (an exception) removes X before it rethrows. | The record has no `g`. X is an unreferenced directory: reported by `checkIntegrity()`; the next PREPARE that reuses the number removes it first. | +| INTENT | Write the state record with `g=N>X` and a **new lock epoch** (synced), and `b` when X's `buffer/` holds a file (a `b` already set stays). From now on, no process can use a cache of the old epoch, and every process that takes the lock runs recovery before returning data. | Recovery (below). | +| PUBLISH | Replace `CURRENT` atomically with generation X. **Commit point.** | Recovery. | +| CLEANUP | Remove `generations/N` (in durable mode, then sync `generations/`). Readers read only under the lock and keep no data handle between operations, so no reader can be inside N. | Recovery. | +| DONE | Clear `g`, only once N is gone (and, in durable mode, its removal synced), and `b` unless X's `buffer/` holds a file. Otherwise `g` stays and the next session's recovery finishes the cleanup. | None. | + +**Recovery** (`g=N>X` found; exclusive lock): + +1. Read `CURRENT` and check validity (§4.3). Its generation must also be N or X. +2. CURRENT valid and names X → X is committed: remove N if it still exists, then clear `g`. +3. CURRENT valid and names N → X was never published: remove X, then clear `g`. + - In both cases the recovery log gets `generation-recovered`. If the obsolete generation can't be removed (permissions, I/O), it gets `generation-cleanup-failed` instead. `g` is cleared anyway: CURRENT decides, so the data is consistent. The leftover is reported by `checkIntegrity()` rather than retried by every operation. +4. Otherwise (CURRENT unreadable, invalid, or naming anything else) → which generation was committed cannot be proven: + - **nothing is removed and no generation is chosen**; + - every read and write fails with `FORMAT_DAMAGED`, naming both candidate generations; + - `checkIntegrity()`, `explain()` and `getMetrics()` fail with `FORMAT_DAMAGED` too, without naming them. They do so for every pending `g`, a recoverable one as well, and for an invalid CURRENT without `g`. + - The documented ways out are `restore()` from a backup, or the manual procedure in OPERATIONS.md. + +An invalid `CURRENT` without `g` is handled the same way: fail closed, nothing removed. + +## 6. Segments + +### 6.1 Data file `{seg}.jsonl` + +- One JSON object per line, terminated by `"\n"`. Every record line carries `"key":K` as its **last** member. +- Lines are encoded as metadata is (§2): `JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRESERVE_ZERO_FRACTION`, `serialize_precision -1`. +- A delete appends a tombstone `{"__nonedb_deleted":K}`. +- An update appends the new record; the last line of a key wins. +- A line counts only if: + - it is complete (ends with `"\n"`); + - it decodes; + - its key lies in the segment's range: `main` = `[0, ∞)`, `s{N}` = `[N·shardSize, (N+1)·shardSize)`. +- An unterminated last line is never a record. Readers ignore it. +- The next writer copies it to a verified quarantine file (§7.2) and cuts it before appending. +- A complete line that does not decode as a whole is analysed by the salvage rule: from right to left, the first `{` from which a record or tombstone with a valid key decodes up to the line end (tried first at the line's offset in the old index, then for at most 4096 `{`). If found, it is applied and logged; otherwise the line is skipped and logged. + +### 6.2 Data replacement (segment compaction) + +Compaction stays inside a generation (§5.2 is not used). Steps under the exclusive lock, after the temps an earlier +crash left of `{seg}.jsonl` are removed (§2): + +1. **EPOCH:** new lock epoch (synced). +2. **DROP-CHECKPOINT:** remove `{seg}.kix`. +3. **REPLACE:** write the compacted data to a temp file, sync it, rename it over `{seg}.jsonl`, and sync the directory. +4. **CHECKPOINT:** write a fresh checkpoint with a new segment epoch `e`. + +- A crash between any two steps leaves either the old or the new data file, never a mix. +- A missing checkpoint means a full replay of whichever file is there. +- Secondary indexes whose `e` differs are rebuilt. +- Compaction keeps keys. It writes a top tombstone when the highest live key is below the segment's highest assigned key, so keys are never reused. + +### 6.3 Shard creation + +- A shard is created by the first append to its key range: `{seg}.jsonl` is created in the active generation. +- The key reservation in `shards.json` and the `j` journal entry are written **before** the append. +- Data files that `shards.json` does not list are set aside as `{name}.orphan.{YmdHis}` (never deleted) before a shard of that number is created. + +### 6.4 Key-index checkpoint `{seg}.kix` (derived) + +The key index maps every live key of a segment to its record: byte offset and length of the line in +`{seg}.jsonl`. Its checkpoint is a binary file: a header, then the slots in checksummed blocks. + +**Header:** the first 512 bytes, a JSON object padded with spaces to 511 bytes, then `"\n"`. Its last member is `crc` (§6.5). + +```json +{"kind":"kix","v":1,"layout":"dense","base":30000,"slots":10000,"live":9998,"n":10000,"d":2,"t":2,"s":S,"h":"","e":"<8 hex>","created":T,"updated":T,"z":10000,"sid":3,"crc":C} +``` + +| Member | Meaning | +|---|---| +| `layout` | `dense` or `sparse` (below). | +| `base` | The key of local key 0: `sid × z` for a shard, 0 for `main`. | +| `slots` | Dense: number of slots (local keys `0 … slots-1`). Sparse: number of entries. | +| `live` | Live keys in the body. | +| `s` | Bytes of the data file the checkpoint covers; always a line end. | +| `h` | crc32b of the up to 256 bytes before `s`. | +| `e` | Segment epoch. | +| `n` | Next local key. | +| `d` | Garbage lines. | +| `t` | Tombstones. | +| `z`, `sid` | Shard size and shard id; sharded layout only. | + +**Body:** starts at byte 512, in blocks of 4096 bytes. The last block may be shorter. +- Each block ends with the crc32 of its other bytes: 4 bytes, little-endian, `hash('crc32b')` as an unsigned integer. +- **Dense layout:** a block holds the slots of 341 consecutive local keys (the last block: the rest, `m` slots). + - First the `m` offsets (u64, little-endian), then the `m` lengths (u32, little-endian). + - Local key `k` is in block `b = ⌊k / 341⌋` at index `i = k mod 341`. Its offset is at `8i` and its length at `8m + 4i` of the block (`m` = 341 except in the last block). + - Length 0 means the key has no record (never written, or deleted). + - Separate offset and length runs let a whole block be decoded in a few operations. +- **Sparse layout:** a block holds up to 255 entries, ascending by key; only the last block may hold fewer (`m`). + - First the `m` local keys (u32), then the `m` offsets (u64), then the `m` lengths (u32), all little-endian. + - Then zero bytes up to 4092, then the crc. +- **Choice of layout:** dense while `slots ≤ max(1024, 4 × live)`, otherwise sparse. A segment of a deletion-heavy database thus stays bounded by its live keys. +- **Limit:** local keys (key − `base`) are below 2^32. + +**Rules:** +- **Fits** its data file when `kind`, `v` and `layout` are known, `s ≤ size`, byte `s-1` is `"\n"`, and `h` matches. +- A block whose crc doesn't match makes the whole file invalid. So does a body whose length doesn't match the header. +- Loading reads the checkpoint and replays the complete lines after `s`. +- **Does not fit** (`s` or `h`): a full replay rebuilds it under the exclusive lock, with a new `e`. +- **Absent, invalid, or `v` unknown:** no checkpoint. The lines are replayed from the start in memory, under the shared lock too, and the next write session writes a checkpoint. +- Writes append data only. Checkpoints are written at the end of a write session once the tail reaches 64 lines + live/16, or 1 MiB. +- In memory a key index is the verified body (12 bytes per slot) plus the keys written or deleted since the checkpoint. + - A body is decoded into per-key arrays when it has at most 250 000 keys and memory allows (the process, with about 80 bytes per key more, stays under half of `cacheMemoryLimit`; no limit: always): + - in a database of at most 16 segments (each is in constant use): at its first use when the body is at most 64 KiB (about 5000 keys, under 1 ms to decode), except in the process's first operation (lock session), where a single cold operation keeps the body; otherwise at the start of the database's next operation, whether that operation reads the index or not (`rebuildIndex()`, which makes the key indexes again, decodes none); + - otherwise once its use has paid for it. + - Paid for: its operations (lock sessions) plus the keys looked up in bulk reach an eighth of its live keys. Decoding costs about 0.17 µs per key; a lookup costs about 1.3 µs more on a body than on arrays. + - Light use keeps the body: a cold query over many segments, a few reads of each segment. + +### 6.5 Secondary indexes (derived) + +- **Header and validity:** + - The header of every derived file (`.kix`, `.fx`, `.gfx`, `.rx`, `.grx`) ends with the member `crc`: the crc32, a decimal integer, of the header's JSON without that member. A header whose `crc` doesn't match makes the file invalid. + - The body of an `.rx` file is one line of JSON, id → node, then the entries of its leaves in the order of those nodes, each five 64-bit little-endian integers (key, then its box). A node is `[level, box, items]`: a box is four integers (null only without items), the items of a leaf (level 0) the number of its entries, the items above a leaf the ids of nodes of a lower level, each named once. The header's `bc` is the crc32 of the body. A body that doesn't match it, or that is not such a tree, makes the file invalid. (The other formats check their pages or blocks; a `.grx` has no body.) + - A number in a file never decides how much is read: a page, a body or a record the file doesn't hold makes the file invalid. + - Line 1 of an `.fx`, `.gfx`, `.rx` or `.grx` file is a JSON header that starts with `kind`, `v` and `field`; the body follows. An `.fx` or `.rx` header holds `s`, `e` and `h` (an `.fx` has `w` before them, an `.rx` `q` and `ns`); a global header holds one such entry per shard in `sh`. + - A segment index (`.fx`, `.rx`) is valid when `kind`, `v` and `field` match, `e` equals the segment's checkpoint epoch, `s` ≤ the checkpoint's `s`, and `h` matches. Loading replays the data lines after `s`. + - An invalid index is rebuilt from the data under the exclusive lock. +- **Global indexes:** + - `.gfx` and `.grx` hold, per shard, `{s, e, h}` (`.grx`: and the shard's box) in the header member `sh`, and the lock epoch `le` they were written under. An entry describes the shard's segment index at `s`. + - A shard is skipped only when its entry fits its data, its summary rules the value out, and the newer lines don't hold it. A shard without an entry is never skipped. +- **Version rule:** an index or checkpoint file whose `v` is unknown is rebuilt from the data. Derived files are never trusted without the checks above. + +#### 6.5.1 Field indexes: paged files (`.fx`, `v` 1) + +A field index maps each value of a field to the keys of a segment that hold it. The file is a static B+tree, so a lookup reads a few small pages instead of the whole file. + +- **Entries:** one per value, identified by its bucket id (the typed identity `Values::valueKey`, a string that begins with a type letter: `n`, `t`, `f`, `i`, `d` or `s`). Entries are sorted by bucket id in byte order. +- **Header:** the JSON line starts with `kind`, `v`, `field` and `w`, in this order. `w` is a token of this write: 16 random hex digits. + - It also holds `pg`: `{"root": [offset, length], "levels": L, "pages": P, "data": D}`. + - Offsets are counted from the first byte after the header line. + - `data` is the number of bytes of the data pages, which come first. + - It adds `mk` (highest key covered), `c` (entries) and `n` (keys). +- **Pages:** a page holds about 4096 bytes in all. An entry larger than that is a page of its own. Every page ends with a crc (below). + - **Data page:** its entries as one JSON object, then the crc. + - **Directory page:** its entries, then a table of where they start, then the crc. The table is one u32 per entry, the offset of the entry from the page's first byte, ascending; then the number of entries (u16). +- **Data pages:** a JSON object with one member per entry, `"":`, in bucket id order, without spaces. + - `` is the key (a JSON integer) when the entry has one key, else a JSON array of its keys, ascending. + - **Canonical JSON:** names are written as PHP `json_encode` writes them with `JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS`: only `"`, `\` and the control characters below U+0020 are escaped. Bucket ids are valid UTF-8 (a record that isn't can't be stored). + - Decoding a page whole is one `json_decode`; a decoded page is already the in-memory form of its entries. +- **Directory pages:** entries of the first bucket id of a child page (u32 length + bytes: a string value has no length limit), then the child's offset (u32) and length (u32). All integers little-endian. + - A directory page holds at least two entries, even when they make it larger than 4096 bytes (bucket ids of more than about 2 KiB): each level has fewer pages than the one below. Only the last page of a level may hold one. + - Level 1 points at data pages, level 2 at level-1 pages, and so on. + - The root is the only page of the top level. + - With one data page there is no directory: the root is that data page (`levels` 0). +- **Checksums:** every page ends with the crc32 of its other bytes (u32, `hash('crc32b')` as an unsigned integer). A page whose crc doesn't match, or a directory entry that leaves its page, makes the file invalid. + - A whole read also checks each page: a table names exactly the entries of its page; a data page decodes to a JSON object whose canonical JSON is the page's bytes. + - A lookup checks that the offsets it follows lie inside the entries. +- **Lookup:** + 1. Read the root. + 2. In each directory page, follow the last entry whose first bucket id is ≤ the one searched. + 3. Read the data page and find the entry: the bytes `"":` (canonical JSON) at a position whose preceding byte is `{` or `,`. Such a match starts a member name, since a bucket id begins with a type letter (a closing quote is followed by `:`, not by a letter), and the name is exactly the bucket id. The value follows. Equivalently: the name without its opening quote, preceded by `{"` or `,"` (inside a name a quote is always escaped as `\"`, and a value holds no quote). The search never leaves the page's bytes. + 4. Replay the data lines after `s`: a key whose newest line holds another value, or a tombstone, is not a key of this value; a newer line with this value adds its key. +- **Whole read:** the data pages in order (bytes `0 … data`). + - A process loads an index whole for ranges and other uses of all its values. + - It also loads one in use, only while memory allows (the process, with the decoded index, under half of `cacheMemoryLimit`; no limit: always): + - an `.fx` file of at most 64 KiB at its first use in an operation other than the process's first (lock session); in the first, a lookup reads its pages. Decoding pays off only when more operations follow, and the decodes a process needs then happen in the same operation as its key indexes' (§6.4); + - a larger `.fx` once its operations have paid for the load: at least two, and `c / 20` of them (a whole load costs about one paged lookup per twenty entries). +- **Pages kept between lookups:** a reader may write a new checkpoint under the shared lock, so the file can be replaced while another process keeps pages of the old one. + - Before reading another page, that process reads the header up to the end of `w` from the same handle. + - If those bytes differ from the header it holds, the file was replaced. The kept pages are dropped and the lookup starts again from the new header. + - Pages of one file are never combined with pages of another. + - **Body kept as bytes:** an `.fx` in use from its second operation whose body (bytes `0 … root end`) is at most 256 KiB, while memory allows twice its size, is read whole at its next page miss: from the lookup's handle, after the header check. From then on its pages come from those bytes: a page's crc is checked at its first use, then the entry is searched for in place (lookup step 3). Nothing is decoded, so this is not a whole read; the bytes are the pages of one file and follow the rules above. +- **Unchanged:** these files are still derived, written whole by atomic replacement and never synced. A writer never changes them in place. + +#### 6.5.2 Global field index: shard filters (`.gfx`, `v` 1) + +The global field index of a sharded database tells which shards may hold a value: per shard, a Bloom filter of the bucket ids of the shard's `.fx`. The filters are stored bit-sliced: row `r` holds bit `r` of every shard's filter, one bit per shard, so a lookup ANDs six short rows, whatever the number of values. + +- **Size:** four to eight bytes per value of the largest shard, per shard (≥ 32 bits per value). Building, updating and looking up never hold more than the file. +- **Header:** the JSON line starts with `kind` (`"gfx"`), `v` (`5`), `field` and `w`, in this order, then: + - `le`: the lock epoch it was written under (§6.5); + - `bits` M (of each shard's filter) and `width` W (shards a row has room for): powers of two, M at least 64, W at least 8; + - `sh`: `{"": {"s":…,"e":…,"h":…}}`, the `.fx` checkpoint each shard's filter was made from (§6.5). A shard id is below W. +- **Body:** M rows of W / 8 bytes; bit `i mod 8` (value `1 << (i mod 8)`) of byte `⌊i / 8⌋` of row `r` is bit `r` of shard `i`'s filter. The bits of a shard without an entry mean nothing. + - Rows are grouped into pages of P = min(M, max(1, 4096 / (W / 8))) rows, in order: page `p` holds rows `p · P … p · P + P − 1`, then the crc32 of those bytes (u32 little-endian). Page `p` starts at `p · (P · W / 8 + 4)` from the first byte after the header line. +- **Probes of a bucket id:** `d` = `hash('xxh3', id, true)` (8 bytes); `x` = bytes 0–3 of `d` as a u32 big-endian; `y` = bytes 4–7 as a u32 big-endian, its lowest bit set. For `i` = 0 … 5: row `(x + i · y) mod 2³² mod M`. +- **Filter of a shard:** M bits, those of every probe row of every bucket id of its `.fx` (at `s`) set, and no other. +- **Lookup of a value:** the AND of its six probe rows; a shard may hold the value when its bit is set in it. It is skipped as §6.5 says: its entry fits, its bit is clear, and the newer lines don't hold the value. A shard that may hold it costs a lookup in its `.fx` (of the shards whose filter rules the value out, at most about 1 in 40 000 answer "may"), never a wrong answer. + - Rows read from the file: the header up to the end of `w` is read again first from the lookup's handle (§6.5.1: a replaced file is read again), then the pages that hold the probe rows, each crc checked. + - A process that doesn't make the exact map below loads the body whole at the file's second operation (lock session) when memory allows (as an `.fx`, §6.5.1: under half of `cacheMemoryLimit` with it), checking every page's crc; its lookups then read the rows from memory. A body held so is that of one file: the entries it came with say which bytes of each shard it describes (§6.5). + - **Exact map:** at the same second operation, in a process of PHP's command line (SAPI `cli`) and a database of at most 1 000 000 records (its metadata's `totalRecords`), a process with room for it (under half of `cacheMemoryLimit` once it is built) reads every shard's `.fx` file that fits its data and keeps a map of their bucket ids to the shards that hold them, with an entry per shard (`s`, `e`, `h` of that file). The build reads those files whole, every key they list; in a larger database that takes seconds, so no map is made there and nothing is read for one. A web server's request ends with its statics, before a map could pay for itself, so no other SAPI makes one either. The file's sizes bound the map first: at most `bits / 32` values per entry of `sh`, about 32 bytes a pair; a map whose bound doesn't fit the room left is not built, and nothing is read for it. A file whose buckets the process holds exactly as the file has them (read whole, no line replayed since) is not decoded again. Its lookups are a map lookup; §6.5's rule applies to its entries as to the file's. When the process replaces a shard's filter, the map takes the shard's new entry and its values (a value the shard no longer holds may stay listed: the shard is then read for it). Not built in `checkIntegrity()`, `explain()` or a join; it changes nothing on disk. +- **Sizes:** `bits` is the smallest power of two ≥ max(64, 32 · c) for the largest `c` (entries) of the shards' `.fx`; `width` the smallest power of two ≥ max(8, highest shard id + 1). A build takes `c` and each entry from the header of a shard's `.fx` that fits its data, and decodes each file for its bucket ids, once per part of the build (the cache limit decides the parts). +- **Update:** when a shard's `.fx` is checkpointed, its filter and entry are replaced; once per session, after its last change (§6.5). If its `32 · c` exceeds M or its id is ≥ W, the file is rebuilt with the new sizes from every shard's `.fx`. Otherwise every page is copied with the shard's bits replaced. Either way the file is written whole by atomic replacement, derived and never synced. +- **Damage:** a page whose crc doesn't match, or a header that isn't valid, makes the file invalid. It is rebuilt from the shards' `.fx` under the exclusive lock; until then it skips no shard. + +## 7. Recovery files and journal flags + +### 7.1 Recovery log `recovery/log.jsonl` + +- One JSON object per event: `{"t":…,"action":…,…}`, with `seg` on salvage and skip events. The file rotates to `log.1.jsonl` past 1 MiB. +- Events record what an exclusive session did: + - quarantine and cut, + - salvage, skip line, dropped line, dropped buffer, + - FORMAT repair, shard metadata rebuilt, index dropped, + - a generation recovered, or its cleanup failed, + - deferred job failure, + - restore, foreign restore. +- Store-level events (a STORE copy repaired, a manual rebuild) go to `.nonedb-maintenance/log.jsonl` (§3). +- The log never contains record values or key material. + +### 7.2 Quarantine `recovery/q.{seg}.{off}.{len}.{sha256}.bin` + +Bytes that must be removed from a data or buffer file are first copied here. `{seg}` is the segment (`main`, `s{N}`) or `buffer` for the pending buffer and the flushing files; recovery events about the bytes use `{seg}.{off}.{len}.{sha256}` as their id. The copy is read back and its length and SHA-256 are verified before the source is cut. A failure leaves the source unchanged and fails the write. + +### 7.3 Journal `j` / `J` + +- `j` (or `J` for all segments) is set before a sharded segment's data changes. It is cleared at the end of the session. +- When found at lock time, the next exclusive session: + - recounts the shard counters from the data, + - removes stale temp files, + - re-validates the listed segments. +- A failed rollback sets `J`. + +## 8. Database lifecycle + +### 8.1 Creation + +Creation takes `STORE.LOCK` shared and re-validates the store (§3.3). Then: + +1. `mkdir {locator}.ndb4`. A creator that loses this race goes on: `LOCK` serializes them. +2. Open `LOCK` and lock it exclusively. +3. Write the state record `{gen}:{epoch} c` (synced). The epoch is new for a new record; a re-creation keeps the one the drop wrote. +4. Generation 1 (§5.2 PREPARE) and `CURRENT`. Generation 1 is empty, or a backup's copy (`restore()`, §10.3); + a copy that holds a buffer file sets `b` (synced) before `CURRENT`. +5. `FORMAT` (atomic). Its existence means "creation complete". +6. Clear `c`. + +No data is written before step 6, except the copy of a restore, which its backup still holds. + +### 8.2 Drop + +1. Write the state record with `d` and a new epoch (synced). +2. Remove `FORMAT`. The database no longer exists (commit point). If `FORMAT` can't be removed, `d` is cleared, nothing else is touched and `dropDB()` fails naming the files. +3. Remove `CURRENT`, `generations/` and `recovery/`. +4. Clear `d`. If files are left, `d` stays and the next exclusive session removes them (the drop is committed). + +`LOCK` and the directory stay. A directory holding only `LOCK` is an empty location (§8.3). + +### 8.3 Interpreting a database directory + +On every path that needs the database, the engine resolves the directory's state as follows. Resolving may upgrade to the exclusive lock. `checkIntegrity()`, `explain()` and `getMetrics()` resolve nothing and write nothing: they report `c` and `d` as pending recovery, and fail with `FORMAT_DAMAGED` on a Damaged directory, with or without the proof a repair needs. The next read or write repairs what it can prove. + +| State | Evidence | Behaviour | +|---|---|---| +| Absent | No directory, or only `LOCK` and no `c`/`d` flag | Reads return empty results and create nothing; writes create the database (§8.1). | +| Interrupted creation | No FORMAT, `c` set | Nothing was committed: creation writes no data, and a restore's copy is still in its backup. Remove the partial `CURRENT`/`generations`, clear `c`, then treat as **Absent**. | +| Interrupted drop | `d` set | Finish the drop (§8.2), then treat as **Absent**. | +| Creation finished, flag left | FORMAT valid, `c` set | Clear `c`. | +| Damaged | FORMAT missing or invalid, no `c`/`d`, entries other than `LOCK` | **Never treated as absent.** | +| Present | FORMAT valid | Open; CURRENT is checked by §4.3 and §5.2. | + +**Repairing a Damaged directory:** +- A repair happens only with full proof: a valid CURRENT whose generation `metadata` is valid and whose identity members agree. +- If there is proof, FORMAT is rewritten from `metadata` and the repair is logged. +- If not, `FORMAT_DAMAGED`. No file changes and the data stays. +- The way out without proof is `restore()` from a backup (§10.3), which keeps the damaged generations. + +## 9. Maintenance + +### 9.1 Maintenance journal `.nonedb-maintenance/journal.json` + +```json +{"magic":"NONEDB4","kind":"journal","op":"rotate","id":"","state":"COMMITTED","targets":[...],"store":{...},"clear":[...],"checksum":""} +``` + +- `op` is one of: `migrate`, `rotate`, `register`, `repair-store`, `restore-old`. +- `store` is the complete target STORE record, `clear` the directories whose `m` the commit clears. A rotation writes `store` from `STARTED` on, `register` and `repair-store` start at `COMMITTED`. `repair-store` adds `mode`. +- **While the file exists:** + - opening a store fails with `MAINTENANCE_IN_PROGRESS`; + - re-validating a store (§3.3) fails the same way. +- A finished journal is moved out as `journal.done.json` (`restore.done.json` after `restore-old`) into the operation's backup directory, or removed. + +### 9.2 Order (normative) + +1. Lock `STORE.LOCK` exclusively (`LOCK_EX|LOCK_NB`). +2. Lock every target database's `LOCK` in locator order, also `LOCK_EX|LOCK_NB`. + - If any lock fails: release all, `MAINTENANCE_BUSY`. **No journal, epoch or `m` flag has been written.** +3. Write the journal, state `STARTED` (a migration: `DISCOVERED`, §9.5), with the operation and its targets. +4. Write `m` and a new epoch into every target's state record (synced). A rotation releases the database locks here: `m` keeps every process out, and step 7 takes each lock again to clear it. +5. Do the work. A re-run works out from the file system how far it came. +6. **Commit:** write the journal with state `COMMITTED` and the target STORE record. **This is the logical commit point.** +7. Apply: + 1. write the STORE copies (§3.1) with the target content; + 2. clear `m` with a new epoch; + 3. close the journal; + 4. release `STORE.LOCK`. + +**On re-run:** +- Before `COMMITTED`: roll forward, or roll back with the `abort` option. +- From `COMMITTED` on: only roll forward (idempotent). +- A failure while applying step 7 is not reported as success: `MAINTENANCE_IN_PROGRESS`, with a message that says it is committed but not completed and must be run again (a failed directory sync of a STORE copy: `OUTCOME_UNKNOWN`). The journal stays open, so the store stays closed until the re-run. +- If the `COMMITTED` journal write cannot be confirmed durable: `OUTCOME_UNKNOWN`. The re-run decides from the journal on disk. + +**Lock order rule for the whole engine:** `STORE.LOCK` before any database `LOCK`, and database `LOCK`s in locator order. CRUD takes only a database `LOCK`. The missing-database path and creation take `STORE.LOCK` shared first. + +### 9.3 Key operations + +- **`registerKey($dbDir, $registered, $new)`:** + - Needs `$registered` to be in the STORE record. + - Journal → `COMMITTED` (STORE with the new keyCheck added) → copies → close. + - No database lock is needed. +- **`migrateKey($dbDir, $old, $new, $options)`:** + - Targets are every database of `$old`. If `{new locator}.ndb4` already exists for one of them, the rotation is refused before anything is written (`MAINTENANCE_BUSY`, naming the database). + - In step 5, each `{old locator}.ndb4` is renamed to `{new locator}.ndb4`. FORMAT and metadata are key-independent and unchanged. + - Every renamed directory is then verified: FORMAT, CURRENT, metadata and `databaseId`. A failure reverses the renames, clears `m`, closes the journal and ends with `FORMAT_DAMAGED`. If a rename back fails, the journal stays open (the store stays closed) and the result is `MAINTENANCE_IN_PROGRESS`; `['abort' => true]` finishes the reversal. + - Commit: the STORE record without the old keyCheck and with the new one. + - Running processes are covered in both cases: + - **During the rotation:** a process finds `m` in the state record once it has the `LOCK`, so it gets `MAINTENANCE_IN_PROGRESS`. + - **After the rotation:** a process that still uses the old locator finds the path gone. Every lock acquisition checks that its path still names the locked file. The process then takes the missing-database path (§3.3), which re-validates the store and fails with `KEY_NOT_REGISTERED` (or `MAINTENANCE_IN_PROGRESS` while the journal is open). It never gets an empty result and never creates a directory. + +### 9.4 `repairStore($dbDir, array $keys, array $options)` + +Only for the case where both STORE copies are invalid. One valid copy is repaired automatically (§3.1). The result names the mode and whether it is complete. + +**Ownership rule (both modes):** every `*.ndb4` directory must belong to one of `$keys`, i.e. `locator(key, FORMAT.name)` equals its name. Otherwise: `FORMAT_DAMAGED`, the unattributable directories are listed, and nothing changes. + +| Mode | Input | Result | +|---|---|---| +| A — verified manifest | `['manifest' => path]` or `['manifest' => storeManifest]` | The manifest is a complete STORE record from a maintenance result, a `backup()` (`store.json`) or the migration tool's backup; its checksum must verify. The new record is the manifest's keys ∪ `$keys`. Result `complete: true`: the manifest's state is restored completely. Keys registered after the manifest that have no database are outside this guarantee by definition. | +| B — manual rebuild | `['confirmManualRebuild' => true]` | The new record is `$keys`. Result `complete: false`, with the warning that earlier keys without databases cannot be recovered. Logged as `manual-store-rebuild`. Without the confirmation option: refused. | + +Every successful key operation returns the new STORE record, checksum included, as `storeManifest`; it can be passed to `repairStore()` as it is or saved as a file. The documentation tells operators to keep it. + +### 9.5 Migration publish (tool) + +The migration tool (MIGRATION.md) follows §9.2 and adds: + +- **Before each publish:** the source fingerprint is re-checked under the locks. The staged database's state record carries `m` **before** it is renamed into `dbDir`. + - A published database therefore answers `MAINTENANCE_IN_PROGRESS` until the commit, even when its key is already registered. +- **Commit:** the target STORE record has the key added; if the key was already registered, it is unchanged. +- **Apply after commit:** STORE copies → move the sources to `.nonedb-retained` (§9.6) → clear `m` on the published databases → close the journal. +- **Collision:** if the target `{locator}.ndb4` already exists, that database is `AMBIGUOUS_SOURCE`. Nothing is merged or overwritten. +- **Key registration:** a key is registered only by the run that completes the migration of all of its old databases in `dbDir`. + +### 9.6 Retained sources `.nonedb-retained/{runId}/` + +- After the commit, the migrated databases' source files are moved here by rename, the files under `files/`, together with a `manifest.json` (file names, sizes, SHA-256, backup reference). +- They are safe to ignore when both hold: + 1. every database they belong to is committed; + 2. their bytes equal the verified backup. +- The engine never looks inside. +- `--cleanup` re-verifies the backup and removes the directory. `--restore` moves the files back. + +## 10. Backup and restore of a database + +`backup($db, $dir)` copies one database, as it is at one moment, into a directory of its own. +`restore($dir, $db, $options)` makes such a copy the content of a database, with one commit. + +### 10.1 Backup directory + +```text +{dir}/ 0700; files 0600 + FORMAT the database's FORMAT and CURRENT, byte for byte + CURRENT + generations/{G}/... the files of the active generation G that §5.1 names, byte for byte; a data or + buffer file only up to its last "\n" (an unterminated tail is never a record, §6.1) + recovery/... the database's recovery log and quarantine files (for inspection only) + store.json the STORE record (§3.1), sealed as STORE is: a manifest for repairStore() (§9.4) + manifest.json written last +``` + +The layout is that of a database directory (§4), so the copy is checked by the same rules. Temp files and +data files the database set aside (`*.orphan.*`, §6.3) are not part of it. + +**`manifest.json`:** + +```json +{"magic":"NONEDB4","kind":"backup","storageFormat":4,"databaseId":"","name":"users","generation":3,"nextKey":123456,"records":120001,"bufferedRecords":0,"takenAt":1793103240,"files":[{"path":"CURRENT","size":118,"sha256":""},...],"checksum":""} +``` + +- `nextKey` and `records` are the database's counters when the copy was taken (those `getShardInfo()` + reports as `nextKey` and `totalRecords`). `bufferedRecords` counts the records of its buffer files. +- `files` lists every other file of the backup, sorted by `path` (byte order): the path relative to `{dir}` + with `/` separators, the size and the SHA-256. + +A backup is **valid** when all of these hold; otherwise it is incomplete or damaged: +1. `manifest.json` is valid (§2), with `kind` `backup` and `storageFormat` 4. +2. Every path in `files` is one of: + - `FORMAT`, `CURRENT`, `store.json`; + - `recovery/{name}`, a name of `[A-Za-z0-9._-]` that is not `.` or `..`; + - `generations/{G}/` (G = the manifest's `generation`) followed by `metadata`, `shards.json`, + `indexes.json`, `segments/{seg}.jsonl`, `segments/{seg}.kix`, `indexes/{seg}.{token}.fx`, + `indexes/{token}.gfx`, `spatial/{seg}.{token}.rx`, `spatial/{token}.grx`, `buffer/pending.jsonl` or + `buffer/flushing.{K}.jsonl` (seg and token as in §1 and §5.1). +3. `files` holds `FORMAT`, `CURRENT`, `store.json` and `generations/{G}/metadata`. +4. Every listed file exists, is a regular file, and has its size and SHA-256. +5. `FORMAT` is valid with the manifest's `databaseId` and `name`; `CURRENT` is valid (§4.3) and names G; + `generations/{G}/metadata` is valid with the same identity. + +Files the manifest does not list are ignored. The checksums detect damage, not a deliberate change: a +backup directory needs the protection dbDir has. + +### 10.2 `backup($db, $dir)` + +1. `$dir` must lie outside `dbDir` once symbolic links are resolved, and must not exist or be an empty + directory. Otherwise: `InvalidArgumentException`, and nothing is created. +2. Lock `STORE.LOCK` shared and check the store: an open maintenance journal is `MAINTENANCE_IN_PROGRESS`, + an unregistered key `KEY_NOT_REGISTERED`. Then lock the database's `LOCK` shared. Acquiring it recovers + what any operation would (§4.2). A database that does not exist is a `RuntimeException`, and nothing is + created. Both locks are held until step 5: + - the copy is the state of one moment: data, buffer, counters, indexes and the STORE record; + - writers of this database wait (`lockTimeout`), and maintenance fails with `MAINTENANCE_BUSY`; other + databases are not affected; + - nothing of the database is written: the buffer is not flushed, and no checkpoint is written. +3. Copy the files of §10.1, hashing what is copied. +4. Read every copy back. Its size and SHA-256 must be what was copied. +5. Write `manifest.json` by atomic replacement. In durable mode, every copy and every directory is synced + before it. + +A failure removes what the call created, then rethrows. An interrupted backup has no `manifest.json`, and +`restore()` refuses it. + +### 10.3 `restore($dir, $db, ['force' => false])` + +**1. Verify** the backup by §10.1. An invalid backup is `RESTORE_REFUSED`, and nothing is written. + +**2. Lock** the database's `LOCK` exclusively, as a write does. An absent database takes the creation path +(§8.1, under `STORE.LOCK` shared). Inside a write to the database that has not committed, `restore()` is a +`LogicException`. The state is resolved as for any operation, with one exception: a **damaged** database +(FORMAT or CURRENT invalid and no proof, §5.2, §8.3) is handed to `restore()` instead of failing. This is +the documented way out of `FORMAT_DAMAGED`. + +**3. Identity** of the restored database. It is decided before anything is written: a refusal leaves the +store byte for byte as it was. + +| Target | Without `force` | With `force` | +|---|---|---| +| Present or damaged, FORMAT valid, the backup's `databaseId` | FORMAT's identity | The same. | +| Present or damaged, FORMAT valid, another `databaseId` | `RESTORE_REFUSED` | FORMAT's identity; FORMAT stays as it is. A `foreign-restore` event. | +| Absent, or damaged with FORMAT invalid; the backup's `name` is `$db` | The backup's identity. | The same. | +| Absent, or damaged with FORMAT invalid; another `name` | `RESTORE_REFUSED` | A new identity: a new `databaseId`, `createdAt` now, the backup's `shardSize` and `formatOptions`. A `foreign-restore` event. | + +A sharded backup whose `shardSize` differs from the identity's is `RESTORE_REFUSED`, with `force` as well: +its shard files would not fit the key ranges. + +**4. PREPARE** the new generation X: +- present: X = N + 1 (§5.2); +- absent: X = 1, inside the creation (§8.1); +- damaged: X = 1 + the highest number under `generations/`. + +Then: +1. **Copy** the backup's `generations/{G}/` into `generations/X/`, except `metadata`. Each file is written + with the durability of its kind (DURABILITY.md §3): data and buffer files as replaced data files, + `shards.json` and `indexes.json` as records, derived files unsynced. Every copy is read back: it must + hold the backup's bytes (size and SHA-256). +2. **Metadata:** write X's `metadata` with the identity of step 3, generation X and the layout of the + backup's `metadata`. +3. **Next key.** T is the database's next key: + - present: N's next key; + - absent: 0; + - damaged: 1 + the highest key of any complete line in any segment file under `generations/`. The keys a + damaged database handed out are not known otherwise. + + When T is above the manifest's `nextKey`: + - a top tombstone `{"__nonedb_deleted":T−1}` is appended to the segment of key T−1: `main` in the single + layout, `s{⌊(T−1)/shardSize⌋}` in the sharded one (created, with a shard entry, when the backup has + none); + - the sharded `shards.json` gets `nextKey` T. + + New keys then start at max(T, the manifest's `nextKey`): a restore never lets keys go back. The records + of a `flushing.{K}` file keep the keys from K on that its name gave them (§5.1). +4. **Verify X.** `checkIntegrity()` runs over X as if it were the active generation, and writes nothing. + - Its `records` and `bufferedRecords` must equal the manifest's. + - The shard list and the counters must fit the data (no `meta` mismatch). + - The next key must be max(T, the manifest's `nextKey`). + + Otherwise: `RESTORE_REFUSED`. X is removed, and the database stays as it was. + +**5. Commit:** +- **Present:** INTENT → PUBLISH → CLEANUP → DONE (§5.2). Generation N is removed only after X is published. +- **Absent:** `CURRENT`, then `FORMAT` (the commit point), then `c` is cleared (§8.1). +- **Damaged:** + 1. the state record gets a new epoch, loses `g`, and gets `b` when X's `buffer/` holds a file (synced); + 2. `CURRENT` is replaced to name X: the commit point; + 3. an invalid FORMAT is rewritten with the identity. A crash before that is repaired from X's `metadata` + (§8.3). + + The earlier generations stay as unreferenced directories: nothing proves which of them was committed. + `checkIntegrity()` reports them, and OPERATIONS.md says how to inspect and remove them. + +**6. Log** a `restore` event in `recovery/log.jsonl` (`foreign-restore` when `force` changed the identity): +generation X, the backup's `databaseId` and `takenAt`, and `records`. The database's earlier `recovery/` +stays. + +**After a crash**, the database opens either as it was or as restored: +- present: by the recovery of §5.2; +- absent: an interrupted creation is removed (§8.3); +- damaged: it stays damaged until `CURRENT` names X. + +## 11. Errors + +| Code | Constant | Meaning | +|---|---|---| +| 3301 | `OUTCOME_UNKNOWN` | A failure after a commit point, or an undo that could not be completed. The next access recovers. | +| 3302 | `LOCK_FAILED` | A lock was not acquired within `lockTimeout`, or was lost. | +| 3304 | `UNSUPPORTED_FORMAT` | The location is not a noneDB 4 store or database, or it holds foreign entries on the missing-database path. | +| 3305 | `FORMAT_DAMAGED` | A noneDB 4 identity or pointer record is invalid and cannot be repaired with proof. Data files are kept. | +| 3306 | `UNSUPPORTED_DURABILITY` | Durable mode was requested on a platform that cannot provide it. | +| 3307 | `DURABILITY_FAILED` | A required sync failed before the commit point; the operation was rolled back. | +| 3308 | `KEY_NOT_REGISTERED` | The configured key is not in the STORE record. | +| 3309 | `MAINTENANCE_IN_PROGRESS` | A maintenance journal is open, or the database's state record has `m`. | +| 3310 | `RESTORE_REFUSED` | The backup is incomplete or damaged, belongs to another database (without `force`), or has another shard size than the database (§10.3). Nothing was changed. | +| 3311 | `MAINTENANCE_BUSY` | Maintenance could not take every lock it needs. Nothing was changed. | + +All are thrown as `RuntimeException` with the code above, except that `migrateKey()`, `registerKey()` and `repairStore()` return theirs in the result array (`success` false, `error`, `code`). + +## 12. Directory listings + +The engine lists a directory only in these cases: +- when a store is first created (§3.2); +- on the missing-database and creation paths (§3.4); +- in `getDBs()`; +- in recovery triggered by `g`, `j`, `J` or a damaged record; +- in `checkIntegrity()`, `getMetrics()` of a database, `backup()`, `restore()`, `dropDB()`, a generation switch and maintenance; +- when a write creates a shard, for files an earlier crash left under its number; +- before a compaction, for the temps of §6.2; +- when a buffer is flushed, for its `flushing` files; +- when `shards.json` or `indexes.json` is derived again from the files. + +Other reads and writes of an existing database list no directory. A test counts the listings. diff --git a/docs/index.html b/docs/index.html index adb6cf9..3be3168 100644 --- a/docs/index.html +++ b/docs/index.html @@ -243,7 +243,7 @@
-
v3.2 - Multi-process safe & crash-safe
+
v4.0 - Crash-safe writes, backups & a verified storage format

The Zero-Dependency
Database for PHP

Not every project needs a database server.
For small to mid-scale applications, noneDB delivers the performance you need
without the infrastructure overhead.

@@ -257,11 +257,11 @@

The Zero-Dependency

-
~9KLines of Code
+
1File to Include
0Dependencies
-
O(1)Key Lookup
+
0.05 msKey Lookup
@@ -272,15 +272,15 @@

The Zero-Dependency
-

Blazing Fast

O(1) key lookups with JSONL byte-offset indexing. Static cache sharing across instances.

+

Fast by Design

Key-index checkpoints and paged field indexes: a lookup reads only the pages it needs. Indexes stay cached within a memory limit you set.

-

Auto-Sharding

Automatically splits large datasets into shards for optimal performance at scale.

+

Scales to Millions

Automatic sharding (10,000 records per shard by default) and bounded memory: 10 million records answer queries in a 512 MiB process.

-

Multi-Process Safe

Database-level locking, atomic file replacement and self-healing indexes: no lost updates, even when a process is killed mid-write.

+

Crash-Safe

Locks across processes, failed writes rolled back, and a killed process loses no acknowledged write. Durable mode survives a power loss (Linux); backups are consistent.

@@ -292,7 +292,7 @@

Query Builder

Fluent interface with MongoDB-style operators. Chain me

-

Secure by Default

PBKDF2 hashed filenames. No external exposure of database structure.

+

Keys and Permissions

Only the keys registered for a store open it; directory names are HMACs of the names. Files are 0640. Data is not encrypted: keep the directory outside the web root.

@@ -309,7 +309,7 @@

Secure by Default

PBKDF2 hashed filenames. No external exposure of da

📊

Analytics & Logs

Event tracking, activity logs

🗺️

Location Apps

Store locators, geo-based features

-

10K+ records? No problem. Auto-sharding keeps performance consistent as your data grows.

+

10K+ records? Auto-sharding keeps the files small, and a key lookup takes about 0.05 ms from 1,000 to 100,000 records.

@@ -329,7 +329,7 @@

Secure by Default

PBKDF2 hashed filenames. No external exposure of da -

Requires PHP 7.4+ • No extensions needed • Works on any hosting

+

Requires PHP 8.1+ (64-bit) • No extra extensions • Local file system

@@ -344,7 +344,7 @@

Secure by Default

PBKDF2 hashed filenames. No external exposure of da

-
// Initialize
+                        
// Initialize (reads the .nonedb config file)
 $db = new noneDB();
 
 // Create
@@ -417,7 +417,10 @@ 

Secure by Default

PBKDF2 hashed filenames. No external exposure of da Query Builder Spatial Queries Configuration - Benchmarks + Performance + Migrating to 4.0 + Backup & Restore + Operations

@@ -449,14 +452,12 @@

Ready to simplify your stack?

diff --git a/examples/chaining.php b/examples/chaining.php index c885b09..f26df2f 100644 --- a/examples/chaining.php +++ b/examples/chaining.php @@ -207,16 +207,16 @@ echo " Users before: $before, after: $after\n"; // ========================================== -// COMPARISON: OLD vs NEW API +// COMPARISON: DIRECT METHODS vs QUERY BUILDER // ========================================== -echo "\n=== OLD vs NEW API ===\n\n"; +echo "\n=== DIRECT METHODS vs QUERY BUILDER ===\n\n"; -echo "OLD API (still works):\n"; +echo "Direct methods:\n"; echo ' $results = $db->find("users", ["active" => true]);' . "\n"; echo ' $sorted = $db->sort($results, "score", "desc");' . "\n"; echo ' $limited = $db->limit($sorted, 5);' . "\n"; -echo "\nNEW API (fluent chaining):\n"; +echo "\nQuery builder:\n"; echo ' $results = $db->query("users")' . "\n"; echo ' ->where(["active" => true])' . "\n"; echo ' ->sort("score", "desc")' . "\n"; diff --git a/examples/database-management.php b/examples/database-management.php index e69a7d2..c2f4c4c 100644 --- a/examples/database-management.php +++ b/examples/database-management.php @@ -170,17 +170,17 @@ echo "1. Database Naming:\n"; echo " - Use lowercase letters and hyphens\n"; -echo " - Avoid special characters (they get removed)\n"; +echo " - Only letters, digits, space, apostrophe and hyphen are allowed\n"; echo " - Be consistent: 'user-profiles' not 'userProfiles'\n"; echo "\n2. Database Organization:\n"; echo " - Separate concerns: users, orders, logs\n"; echo " - Don't mix unrelated data in one database\n"; -echo " - Consider sharding for large datasets\n"; +echo " - Large databases are sharded automatically (shardSize records per shard)\n"; echo "\n3. Security:\n"; -echo " - Change \$secretKey before production!\n"; -echo " - Protect /db directory with .htaccess\n"; +echo " - Put a random secretKey in your .nonedb file (php -r \"echo bin2hex(random_bytes(32));\")\n"; +echo " - Keep dbDir and .nonedb outside the web root\n"; echo " - Validate database names from user input (invalid names throw InvalidArgumentException)\n"; echo "\n=== Database Management Examples Complete ===\n"; diff --git a/examples/delete.php b/examples/delete.php index 08b4da9..a589471 100644 --- a/examples/delete.php +++ b/examples/delete.php @@ -45,9 +45,9 @@ echo " Remaining: " . $db->count("users") . " users\n"; // ----------------------------------------- -// 3. Delete by Key (Index) +// 3. Delete by Key // ----------------------------------------- -echo "\n3. Delete by Key (Index):\n"; +echo "\n3. Delete by Key:\n"; // First, find current keys $users = $db->find("users", 0); @@ -118,7 +118,7 @@ echo " Sessions after: " . $db->count("session") . "\n"; // ----------------------------------------- -// 7. Deleted Records Become NULL +// 7. What Happens to Deleted Records // ----------------------------------------- echo "\n7. Technical Note - Deleted Records:\n"; echo " - Deleted records disappear from find() immediately\n"; @@ -134,6 +134,8 @@ $info = $db->getShardInfo("users"); echo " Before compact - Total records: " . ($info['totalRecords'] ?? 'N/A') . "\n"; +// Auto-compaction already ran when deleted records passed 30% of the file (autoCompactThreshold), +// so compact() only frees what was deleted after that $result = $db->compact("users"); echo " Freed slots: " . $result['freedSlots'] . "\n"; echo " After compact - Total records: " . $result['totalRecords'] . "\n"; diff --git a/examples/find.php b/examples/find.php index c95adb1..e8ff1c1 100644 --- a/examples/find.php +++ b/examples/find.php @@ -56,9 +56,9 @@ echo " Active users in Istanbul: " . count($activeIstanbul) . "\n"; // ----------------------------------------- -// 4. Find by Key (Index) +// 4. Find by Key // ----------------------------------------- -echo "\n4. Find by Key (Index):\n"; +echo "\n4. Find by Key:\n"; // Single key $user = $db->find("users", ["key" => 0]); diff --git a/examples/performance.php b/examples/performance.php index 651fe36..84d7cc4 100644 --- a/examples/performance.php +++ b/examples/performance.php @@ -1,11 +1,15 @@ = 1000 ? ($s/1000).'K' : $s; }, $sizes)) . " |\n"; @@ -291,12 +295,11 @@ function getFileSize($db, $dbName) { echo "\n"; } -echo "\n### Method Chaining (v2.1)\n"; +echo "\n### Query Builder\n"; echo "| Operation | " . implode(' | ', array_map(function($s) { return $s >= 1000 ? ($s/1000).'K' : $s; }, $sizes)) . " |\n"; echo "|-----------|" . str_repeat("------|", count($sizes)) . "\n"; -foreach (['whereIn', 'orWhere', 'search', 'groupBy', 'select', 'complex_chain' => 'complex chain'] as $key => $label) { - if (is_int($key)) { $key = $label; } - echo "| {$label}() | "; +foreach (['whereIn' => 'whereIn()', 'orWhere' => 'orWhere()', 'search' => 'search()', 'groupBy' => 'groupBy()', 'select' => 'select()', 'complex_chain' => 'complex chain'] as $key => $label) { + echo "| {$label} | "; foreach ($sizes as $size) { echo formatTime($results[$size][$key]) . " | "; } diff --git a/examples/query-methods.php b/examples/query-methods.php index c6f0674..e18abf3 100644 --- a/examples/query-methods.php +++ b/examples/query-methods.php @@ -41,7 +41,7 @@ // ----------------------------------------- // 2. Get Unique Email Domains // ----------------------------------------- -echo "\n2. Distinct on Nested Processing:\n"; +echo "\n2. Unique Email Domains:\n"; $emails = $db->distinct("users", "email"); echo " Emails: " . json_encode($emails) . "\n"; diff --git a/examples/sharding.php b/examples/sharding.php index 90c9af2..a991ddd 100644 --- a/examples/sharding.php +++ b/examples/sharding.php @@ -2,7 +2,7 @@ /** * noneDB - Sharding Example * - * Demonstrates auto-sharding features: getShardInfo(), compact(), migrate() + * Demonstrates auto-sharding features: getShardInfo(), compact(), convertToSharded() * and sharding behavior with large datasets. */ @@ -58,24 +58,24 @@ echo " Result: " . ($info === false ? "false (database not found)" : json_encode($info)) . "\n"; // ========================================== -// MANUAL MIGRATION +// SWITCHING TO SHARDS BY HAND // ========================================== -echo "\n=== MANUAL MIGRATION ===\n\n"; +echo "\n=== SWITCHING TO SHARDS BY HAND ===\n\n"; -echo "1. Migrate non-existent database:\n"; -$result = $db->migrate("nonexistent"); +echo "1. convertToSharded() on a database that doesn't exist:\n"; +$result = $db->convertToSharded("nonexistent"); echo " " . json_encode($result) . "\n"; -echo "\n2. Migrate small database (will create shards):\n"; -$result = $db->migrate("small-db"); +echo "\n2. convertToSharded() on a small database (creates its shards):\n"; +$result = $db->convertToSharded("small-db"); echo " " . json_encode($result) . "\n"; -// Check after migration +// Check after the switch $info = $db->getShardInfo("small-db"); -echo " After migration - Sharded: " . ($info['sharded'] ? "true" : "false") . "\n"; +echo " After the switch - Sharded: " . ($info['sharded'] ? "true" : "false") . "\n"; -echo "\n3. Try to migrate again (already sharded):\n"; -$result = $db->migrate("small-db"); +echo "\n3. Again (already sharded):\n"; +$result = $db->convertToSharded("small-db"); echo " " . json_encode($result) . "\n"; // ========================================== @@ -83,8 +83,10 @@ // ========================================== echo "\n=== COMPACT ===\n\n"; -// Create database with deletions -$db->insert("with-deletions", [ +// Create database with deletions. A high autoCompactThreshold keeps auto-compaction (by default once +// deleted records pass 30%) from running first, so compact() has something to do +$manual = exampleDb(["autoCompactThreshold" => 0.9]); +$manual->insert("with-deletions", [ ["id" => 1, "name" => "Keep 1"], ["id" => 2, "name" => "Delete me"], ["id" => 3, "name" => "Keep 2"], @@ -93,18 +95,18 @@ ]); echo "1. Before deletion:\n"; -echo " Records: " . $db->count("with-deletions") . "\n"; +echo " Records: " . $manual->count("with-deletions") . "\n"; // Delete some records -$db->delete("with-deletions", ["name" => "Delete me"]); -$db->delete("with-deletions", ["name" => "Delete me too"]); +$manual->delete("with-deletions", ["name" => "Delete me"]); +$manual->delete("with-deletions", ["name" => "Delete me too"]); echo "\n2. After deletion (deleted records are hidden immediately; their bytes stay in the file until compaction):\n"; -echo " Visible records: " . $db->count("with-deletions") . "\n"; +echo " Visible records: " . $manual->count("with-deletions") . "\n"; -// Compact to remove null entries +// Compact: rewrite the file without the superseded and deleted lines echo "\n3. After compact:\n"; -$result = $db->compact("with-deletions"); +$result = $manual->compact("with-deletions"); echo " Result: " . json_encode($result) . "\n"; echo " Freed slots: " . $result['freedSlots'] . "\n"; echo " Total records: " . $result['totalRecords'] . "\n"; @@ -143,7 +145,7 @@ // ========================================== echo "\n=== KEY-BASED LOOKUP ===\n\n"; -echo "Key-based lookups are fast because they only read one shard.\n\n"; +echo "Key lookups use the byte-offset index, sharded or not.\n\n"; // Find by key $start = microtime(true); @@ -164,24 +166,14 @@ echo " Found: " . count($result) . " records\n"; // ========================================== -// SHARDING BENEFITS SUMMARY +// WHAT SHARDING CHANGES // ========================================== -echo "\n=== SHARDING BENEFITS ===\n\n"; +echo "\n=== WHAT SHARDING CHANGES ===\n\n"; -echo "Without Sharding (500K records):\n"; -echo " - Every operation reads entire 50MB file\n"; -echo " - find(key) takes ~772ms, uses ~1.1GB RAM\n"; -echo " - insert() takes ~706ms\n\n"; - -echo "With Sharding (500K records, 50 shards):\n"; -echo " - Key lookups read only 1 shard (~1MB)\n"; -echo " - find(key) takes ~16ms, uses ~1MB RAM\n"; -echo " - 50x faster, 1000x less memory\n\n"; - -echo "When to use sharding:\n"; -echo " - < 10K records: Sharding unnecessary\n"; -echo " - 10K - 100K: Beneficial for key lookups\n"; -echo " - 100K - 500K: Highly recommended\n"; -echo " - > 500K: Consider dedicated database\n"; +echo " - The first key lookup of a process loads one shard's index, not the whole database's\n"; +echo " - Auto-compaction rewrites one shard file at a time\n"; +echo " - Field indexes let filtered queries skip shards that can't match\n"; +echo " - Filters on fields without an index scan the shards (limit(), first() and exists() stop early)\n"; +echo " Measurements: docs/PERFORMANCE.md\n"; echo "\n=== Sharding Examples Complete ===\n"; diff --git a/examples/update.php b/examples/update.php index 675385b..4158dd4 100644 --- a/examples/update.php +++ b/examples/update.php @@ -56,12 +56,12 @@ echo " Bob's new data: age=" . $bob[0]['age'] . ", status=" . $bob[0]['status'] . "\n"; // ----------------------------------------- -// 3. Update by Key (Index) +// 3. Update by Key // ----------------------------------------- -echo "\n3. Update by Key (Index):\n"; +echo "\n3. Update by Key:\n"; $result = $db->update("users", [ - ["key" => 2], // Update record at index 2 (Charlie) + ["key" => 2], // Update the record with key 2 (Charlie) ["set" => ["verified" => true]] ]); echo " Updated: " . $result['n'] . " record(s)\n"; diff --git a/migrate/Backup/Backup.php b/migrate/Backup/Backup.php new file mode 100644 index 0000000..14cf5e5 --- /dev/null +++ b/migrate/Backup/Backup.php @@ -0,0 +1,54 @@ + $file, 'size' => $stat['size'], 'mtime' => $stat['mtime'], 'sha256' => Files::copy("$dir/$file", $target, 0600)]; + } + Files::syncDir("$runDir/files"); + $manifest = ['magic' => 'NONEDB4', 'kind' => 'migration-backup'] + $info + ['files' => $entries]; + Files::writeAtomic("$runDir/manifest.json", StoreRecord::sealRecord($manifest), 0600); + return self::verify($runDir); + } + + /** + * The manifest of a backup whose checksum and every file's size and SHA-256 verify + * @throws Stop + */ + public static function verify(string $runDir): array { + $manifest = StoreRecord::openRecord(@file_get_contents("$runDir/manifest.json"), 'migration-backup'); + if($manifest === null || !is_array($manifest['files'] ?? null)){ + throw new Stop('FAILED', "$runDir: no valid manifest.json; the backup is incomplete"); + } + foreach($manifest['files'] as $entry){ + $path = "$runDir/files/" . $entry['name']; + if(!is_file($path) || filesize($path) !== $entry['size'] || hash_file('sha256', $path) !== $entry['sha256']){ + throw new Stop('FAILED', "$runDir: {$entry['name']} does not match the manifest"); + } + } + return $manifest; + } +} diff --git a/migrate/Cli.php b/migrate/Cli.php new file mode 100644 index 0000000..0614c0f --- /dev/null +++ b/migrate/Cli.php @@ -0,0 +1,161 @@ + $e->result, 'message' => $e->getMessage()]); + return Stop::STATUS[$e->result]; + } catch(\Throwable $e){ + // Anything else: the run stays where its journal says, to be continued or aborted + self::output($json, ['result' => 'FAILED', 'message' => get_class($e) . ': ' . $e->getMessage() . ' (' . basename($e->getFile()) . ':' . $e->getLine() . ')']); + return Stop::STATUS['FAILED']; + } + self::output($json, $result); + return isset($result['report']['result']) && $result['report']['result'] !== 'READY' ? Stop::STATUS[$result['report']['result']] : 0; + } + + /** Options, or null for a usage error */ + private static function parse(array $args): ?array { + $o = []; + foreach($args as $arg){ + if(!preg_match('/^--([a-z-]+)(?:=(.*))?$/s', $arg, $m)){ + return null; + } + $flag = in_array($m[1], self::FLAGS, true); + if((!$flag && !in_array($m[1], self::VALUES, true)) || $flag === isset($m[2]) || isset($o[$m[1]])){ + return null; + } + $o[$m[1]] = $m[2] ?? true; + } + $modes = count(array_intersect_key($o, array_flip(['dry-run', 'abort', 'restore', 'cleanup']))); + if($modes > 1 || (isset($o['flushing']) && !in_array($o['flushing'], ['import', 'skip'], true)) + || (isset($o['shard-size']) && !preg_match('/^[1-9]\d*$/', $o['shard-size'])) + || ($modes === 0 && (!isset($o['backup-dir']) || !isset($o['confirm-stopped']))) + || ((isset($o['restore']) || isset($o['cleanup'])) && !isset($o['backup-dir']))){ + return null; + } + return $o; + } + + /** + * @throws Stop + */ + private static function dispatch(array $o): array { + $config = []; + if(isset($o['config'])){ + $config = json_decode((string)@file_get_contents($o['config']), true); + if(!is_array($config)){ + throw new Stop('FAILED', "{$o['config']} is not a readable .nonedb config file"); + } + if(isset($config['dbDir']) && is_string($config['dbDir']) && $config['dbDir'] !== '' && $config['dbDir'][0] !== '/'){ + $config['dbDir'] = dirname(realpath($o['config'])) . '/' . $config['dbDir']; // relative to the config file + } + } + $dir = $o['dir'] ?? $config['dbDir'] ?? null; + if(!is_string($dir) || $dir === ''){ + throw new Stop('FAILED', 'no directory: --dir or a config file with dbDir'); + } + if(isset($o['abort'])){ + return ['aborted' => Migration::abort($dir)]; + } + $target = getenv('NONEDB_SECRET'); + $target = is_string($target) && $target !== '' ? $target : ($config['secretKey'] ?? null); + if(!is_string($target) || $target === ''){ + throw new Stop('FAILED', 'no key: set NONEDB_SECRET or use a config file with secretKey (never a command line argument)'); + } + if(isset($o['restore'])){ + return ['restored' => Restore::restore($dir, $o['backup-dir'], $o['restore'], $target)]; + } + if(isset($o['cleanup'])){ + return ['cleaned' => Restore::cleanup($dir, $o['backup-dir'], $o['cleanup'])]; + } + $source = getenv('NONEDB_SOURCE_SECRET'); + $options = [ + 'db' => isset($o['db']) ? explode(',', $o['db']) : null, + 'rebuildFromData' => isset($o['rebuild-from-data']), + 'flushing' => $o['flushing'] ?? null, + 'shardSize' => (int)($o['shard-size'] ?? $config['shardSize'] ?? 10000), + 'verify' => isset($o['verify']), + ]; + $source = is_string($source) && $source !== '' ? $source : $target; + if(isset($o['dry-run'])){ + return ['report' => Migration::check($dir, $source, $target, $options)]; + } + $real = realpath($dir); + $backup = rtrim($o['backup-dir'], '/') . '/'; + if($real === false || strncmp(self::absolute($backup), $real . '/', strlen($real) + 1) === 0){ + throw new Stop('FAILED', '--backup-dir must be outside --dir'); + } + return ['migrated' => Migration::run($dir, $source, $target, $options + ['backupDir' => $backup])]; + } + + /** A path made absolute without requiring it to exist */ + private static function absolute(string $path): string { + $real = realpath($path); + if($real !== false){ + return $real . '/'; + } + return ($path[0] === '/' ? '' : getcwd() . '/') . $path; + } + + private static function output(bool $json, array $result): void { + if($json){ + echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n"; + return; + } + if(isset($result['message'])){ + fwrite(STDERR, "noneDB-migrate: {$result['result']}: {$result['message']}\n"); + return; + } + if(isset($result['report'])){ + $report = $result['report']; + echo "noneDB-migrate: {$report['dir']}\n"; + foreach($report['databases'] as $name => $db){ + printf(" %-24s %-18s %s%s, %d records, next key %d%s\n", $name, $db['status'], + $db['layout'], $db['layout'] === 'sharded' ? " ({$db['shardSize']} per shard)" : '', $db['records'], $db['nextKey'], + $db['target'] ? '' : ' (not in this run)'); + foreach(array_merge($db['reasons'], array_map(function($reason, $count){ return "not carried over: $count $reason"; }, + array_keys($db['notCarried']), $db['notCarried'])) as $line){ + echo " $line\n"; + } + } + foreach(['leftovers' => 'leftovers of this key', 'otherKeys' => 'files of other keys (untouched)', + 'foreign' => 'other entries (untouched)'] as $list => $label){ + if($report[$list] !== []){ + echo " $label: " . implode(', ', $list === 'leftovers' ? array_keys($report[$list]) : $report[$list]) . "\n"; + } + } + echo " result: {$report['result']}" . ($report['completesKey'] ? '' : ' (the key would not be registered by this run)') . "\n"; + return; + } + echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n"; + } +} diff --git a/migrate/Converter/Writer.php b/migrate/Converter/Writer.php new file mode 100644 index 0000000..049846a --- /dev/null +++ b/migrate/Converter/Writer.php @@ -0,0 +1,118 @@ +name, $key) . '.ndb4/'; + $generation = $home . 'generations/1/'; + Files::makeDir($generation . 'segments'); + Files::writeNew($home . 'LOCK', str_pad('1:' . bin2hex(random_bytes(4)), 128)); + $sharded = $source->layout === 'sharded'; + $last = $source->nextKey - 1; + $lastSegment = $last < 0 ? null : ($sharded ? intdiv($last, $source->shardSize) : 'main'); + $ids = $source->segmentIds(); + if(!$sharded && !in_array('main', $ids, true)){ + $ids[] = 'main'; // a single-layout generation always has its main segment + } + if($lastSegment !== null && !in_array($lastSegment, $ids, true)){ + $ids[] = $lastSegment; + sort($ids); + } + $shards = []; + foreach($ids as $id){ + $path = $generation . 'segments/' . ($id === 'main' ? 'main' : 's' . $id) . '.jsonl'; + $fp = @fopen($path, 'xb'); + if($fp === false){ + throw new Stop('FAILED', "cannot create $path"); + } + @chmod($path, Files::FILE_MODE); + $index = ['kb' => '', 'p' => [], 'l' => []]; // key => offset, key => length of its line + $offset = 0; + $tail = ''; + $max = -1; + $chunk = ''; + foreach($source->lines($id) as $k => $line){ + $index['p'][$k] = $offset; + $index['l'][$k] = strlen($line); + $offset += strlen($line) + 1; + $chunk .= $line . "\n"; + $max = $k; + if(strlen($chunk) >= 1048576){ + self::put($fp, $chunk, $path); + $tail = substr($chunk, -256); + $chunk = ''; + } + } + $tombstones = 0; + if($id === $lastSegment && $max < $last){ + $chunk .= '{"__nonedb_deleted":' . $last . "}\n"; + $offset += strlen('{"__nonedb_deleted":' . $last . "}\n"); + $max = $last; + $tombstones = 1; + } + self::put($fp, $chunk, $path); + if(!fflush($fp) || !Files::syncHandle($fp)){ + fclose($fp); + throw new Stop('FAILED', "cannot sync $path"); + } + fclose($fp); + $tail = substr($tail . $chunk, -256); + $base = $id === 'main' ? 0 : $id * $source->shardSize; + [$layout, $slots, $body, , $live] = KeyIndex::encodeBody($index, $base); + $header = ['kind' => 'kix', 'v' => 1, 'layout' => $layout, 'base' => $base, 'slots' => $slots, 'live' => $live, + 'n' => $max < $base ? 0 : $max - $base + 1, 'd' => 0, 't' => $tombstones, 's' => $offset, + 'h' => sprintf('%08x', crc32($tail)), 'e' => bin2hex(random_bytes(4)), 'created' => time(), 'updated' => time()]; + if($id !== 'main'){ + $header['z'] = $source->shardSize; + $header['sid'] = $id; + } + Files::writeNew(substr($path, 0, -strlen('.jsonl')) . '.kix', KeyIndex::encodeFile($header, $body)); + if($sharded){ + $shards[] = ['id' => $id, 'count' => count($index['p']), 'deleted' => 0]; + } + } + Files::syncDir($generation . 'segments'); + $identity = ['storageFormat' => \noneDB::STORAGE_FORMAT, 'databaseId' => StoreRecord::uuid4(), 'name' => $source->name, + 'shardSize' => $source->shardSize, 'createdAt' => time(), 'formatOptions' => new \stdClass()]; + if($sharded){ + Files::writeNew($generation . 'shards.json', json_encode(['kind' => 'shards', 'v' => 1, 'shardSize' => $source->shardSize, + 'nextKey' => $source->nextKey, 'totalRecords' => array_sum(array_column($shards, 'count')), 'deletedCount' => 0, + 'shards' => $shards], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "\n"); + } + Files::writeNew($generation . 'metadata', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'generation'] + $identity + + ['generation' => 1, 'layout' => $source->layout])); + Files::syncDir($generation); + Files::syncDir($home . 'generations'); + Files::writeNew($home . 'CURRENT', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'current', + 'databaseId' => $identity['databaseId'], 'generation' => 1])); + Files::writeNew($home . 'FORMAT', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'database'] + $identity)); + Files::syncDir($home); + Files::syncDir($storeDir); + return $home; + } + + private static function put($fp, string $bytes, string $path): void { + if($bytes !== '' && fwrite($fp, $bytes) !== strlen($bytes)){ + fclose($fp); + throw new Stop('FAILED', "cannot write $path"); + } + } +} diff --git a/migrate/Files.php b/migrate/Files.php new file mode 100644 index 0000000..3a49c7d --- /dev/null +++ b/migrate/Files.php @@ -0,0 +1,201 @@ + 'NONEDB4', 'kind' => 'journal'] + $journal)); + } + + /** Close the journal: it moves to $to (the run's backup directory) */ + public static function close(string $dir, string $to): void { + $path = self::path($dir); + if(!@rename($path, $to)){ + if(!@copy($path, $to) || !@unlink($path)){ + throw new Stop('FAILED', "cannot close the journal $path"); + } + } + Files::syncDir(dirname($path)); + } +} diff --git a/migrate/Publisher/Migration.php b/migrate/Publisher/Migration.php new file mode 100644 index 0000000..1927d9f --- /dev/null +++ b/migrate/Publisher/Migration.php @@ -0,0 +1,619 @@ +dir = rtrim($dir, '/') . '/'; + $this->sourceKey = $sourceKey; + $this->targetKey = $targetKey; + $this->options = $options + ['db' => null, 'rebuildFromData' => false, 'flushing' => null, 'shardSize' => 10000, 'verify' => false]; + } + + /** + * The pre-check (§7.1): every database of the source key read and judged, nothing written + * @return array the report + */ + public static function check(string $dir, string $sourceKey, string $targetKey, array $options): array { + return (new self($dir, $sourceKey, $targetKey, $options))->precheck()['report']; + } + + /** + * A migration, or the continuation of an interrupted one (§10) + * @return array the report + * @throws Stop + */ + public static function run(string $dir, string $sourceKey, string $targetKey, array $options): array { + $run = new self($dir, $sourceKey, $targetKey, $options); + $journal = Journal::read($run->dir); + if($journal !== null){ + $run->own($journal); + return $run->resume($journal); + } + if(($closed = $run->unverified()) !== null){ + $run->lock([]); + return $run->finish($closed); + } + $check = $run->precheck(); + if($check['report']['result'] !== 'READY'){ + throw new Stop($check['report']['result'], 'the pre-check found databases that cannot be migrated: ' . json_encode($check['report'])); + } + if($check['targets'] === [] && $check['files'] === []){ + // No old file of the key: only a key a partial run left unregistered, with databases here, is registered + $registered = in_array(StoreRecord::keyCheckOf($targetKey), $run->readStore()['keys'] ?? [], true); + if($registered || !$run->hasDatabasesOf($targetKey)){ + return ['run' => null, 'databases' => [], 'keyRegistered' => $registered]; + } + } + $run->checkSpace($check['report']['bytes']); + // Files of the store the run makes where there were none: a restore removes them again (§11). A STORE.LOCK without + // a STORE record is one an earlier run stopped before its journal left (noneDB 4 creates none next to old files) + $noStore = !file_exists($run->dir . 'STORE') && !file_exists($run->dir . 'STORE.2'); + $created = array_values(array_filter(['STORE.LOCK', '.htaccess', 'index.html'], function($file) use ($run, $noStore){ + return !file_exists($run->dir . $file) || ($file === 'STORE.LOCK' && $noStore); + })); + $run->lock(array_keys($check['targets'])); + self::point('locked'); // @test + $runId = date('Ymd-His') . '-' . bin2hex(random_bytes(3)); + $journal = ['op' => 'migrate', 'id' => $runId, 'state' => 'DISCOVERED', 'targets' => array_keys($check['targets']), + 'completesKey' => $check['report']['completesKey'], 'sourceKeyCheck' => StoreRecord::keyCheckOf($sourceKey), + 'targetKeyCheck' => StoreRecord::keyCheckOf($targetKey), 'backup' => rtrim($run->options['backupDir'], '/') . '/' . $runId, + 'options' => array_intersect_key($run->options, array_flip(['rebuildFromData', 'flushing', 'shardSize', 'verify'])), + 'files' => $run->fingerprints($check['files']), 'created' => $created, 'records' => []]; + $run->write($journal); + return $run->resume($journal); + } + + /** + * Roll back an interrupted run that has not committed (§10) + * @return array ['run' => id, 'state' => the state it was in] + * @throws Stop + */ + public static function abort(string $dir): array { + $run = new self($dir, '', '', []); + $journal = Journal::read($run->dir); + if($journal === null || ($journal['op'] ?? null) !== 'migrate'){ + throw new Stop('FAILED', 'no interrupted migration to abort'); + } + $run->lock([]); + $index = array_search($journal['state'], Journal::STATES, true); + if($index === false || $index >= array_search('COMMITTED', Journal::STATES, true)){ + throw new Stop('FAILED', "the run {$journal['id']} is {$journal['state']}: committed runs are completed, not aborted; run the tool again"); + } + if($journal['state'] === 'PUBLISHING'){ + foreach($journal['published'] ?? [] as $locator){ + if(is_dir($run->dir . $locator)){ + Files::removeTree($run->dir . $locator); // still carries m: never visible, the old files are untouched + } + } + Files::syncDir($run->dir); + } + Files::removeTree($run->staging()); + Files::removeTree(Journal::path($run->dir)); + Files::removeStaleTemps(Journal::path($run->dir)); + Files::syncDir(dirname(Journal::path($run->dir))); + return ['run' => $journal['id'], 'state' => $journal['state']]; + } + + // ------------------------------------------------------------------ pre-check + + /** ['report' => array, 'targets' => name => Source, 'files' => the files the run moves] */ + private function precheck(): array { + if(!is_dir($this->dir)){ + throw new Stop('FAILED', "{$this->dir} is not a directory"); + } + try { + new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->staging()]); + } catch(\InvalidArgumentException $e){ + throw new Stop('FAILED', 'noneDB 4 refuses the key or the configuration: ' . $e->getMessage(), $e); + } + $scan = Discovery::scan($this->dir, $this->sourceKey); + $readOptions = array_intersect_key($this->options, array_flip(['rebuildFromData', 'flushing', 'shardSize'])); + $names = $this->options['db'] ?? array_keys($scan['databases']); + foreach($names as $name){ + if(!isset($scan['databases'][$name])){ + throw new Stop('FAILED', "no database named '$name' with this key in {$this->dir}"); + } + } + $report = ['dir' => $this->dir, 'result' => 'READY', 'databases' => [], 'leftovers' => $scan['leftovers'], + 'otherKeys' => $scan['otherKeys'], 'foreign' => $scan['foreign'], 'completesKey' => false, 'bytes' => 0]; + $targets = []; + foreach($scan['databases'] as $name => $files){ + $source = Source::read($this->dir, (string)$name, $this->sourceKey, $files, $readOptions); + if($source->status === 'READY' && is_dir($this->dir . StoreRecord::locatorOf((string)$name, $this->targetKey) . '.ndb4')){ + $source->status = 'AMBIGUOUS_SOURCE'; + $source->reasons[] = 'TARGET_EXISTS: a noneDB 4 database of this name and key already exists; nothing is merged or overwritten'; + } + $target = in_array($name, $names, true); + $report['databases'][$name] = self::describe($source) + ['target' => $target]; + if($target){ + $targets[$name] = $source; + if($source->status !== 'READY' && $report['result'] === 'READY'){ + $report['result'] = $source->status; + } + } + } + // The files the run moves, by database ('': leftovers and files of the directory) + $files = []; + foreach($targets as $name => $source){ + $files[$name] = $source->files; + } + $report['completesKey'] = count($targets) === count($scan['databases']) && $report['result'] === 'READY'; + if($report['completesKey']){ + $own = array_merge($scan['keyFiles'], ...array_values($scan['leftovers'])); + if($scan['otherKeys'] === []){ + $own = array_merge($own, $scan['sharedFiles']); // the 3.0/3.1 hash cache, once no old file of any key is left + } + if($own !== []){ + sort($own); + $files[''] = $own; + } + } + foreach($files as $group){ + foreach($group as $file){ + $report['bytes'] += filesize($this->dir . $file); + } + } + $store = $this->readStore(); + $report['store'] = $store === null ? 'none' : 'valid'; + return ['report' => $report, 'targets' => $targets, 'files' => $files]; + } + + /** True when the directory holds noneDB 4 databases whose directory names are this key's locators */ + private function hasDatabasesOf(string $key): bool { + foreach(glob($this->dir . '*.ndb4', GLOB_ONLYDIR) ?: [] as $home){ + $format = StoreRecord::openRecord(@file_get_contents("$home/FORMAT"), 'database'); + if($format !== null && StoreRecord::locatorOf($format['name'], $key) . '.ndb4' === basename($home)){ + return true; + } + } + return false; + } + + /** + * The backup and the staged databases take about the size of the old files each (§7.1); on one file system, + * the sum + * @throws Stop + */ + private function checkSpace(int $bytes): void { + $backup = rtrim($this->options['backupDir'], '/'); + while(!is_dir($backup) && dirname($backup) !== $backup){ + $backup = dirname($backup); + } + $same = @stat($backup)['dev'] === @stat($this->dir)['dev']; + foreach(array_unique([$this->dir, $backup]) as $path){ + $free = @disk_free_space($path); // @prod + $free = getenv('NONEDB_MIGRATE_FREE') !== false ? (float)getenv('NONEDB_MIGRATE_FREE') : @disk_free_space($path); // @test + $need = $same ? 2 * $bytes : $bytes; + if($free !== false && $free < $need){ + throw new Stop('FAILED', "not enough free space in $path: $need bytes are needed, " . (int)$free . ' are free'); + } + } + } + + /** The report of one database */ + private static function describe(Source $source): array { + return ['status' => $source->status, 'reasons' => $source->reasons, 'layout' => $source->layout, 'shardSize' => $source->shardSize, + 'segments' => $source->kinds, 'records' => $source->count, 'buffered' => $source->buffered, 'nextKey' => $source->nextKey, + 'fields' => $source->fields, 'spatial' => array_keys($source->spatial), 'notCarried' => $source->notCarried]; + } + + // ------------------------------------------------------------------ the run + + /** Continue the run from its journal's state */ + private function resume(array $journal): array { + $this->lock($journal['targets']); + $state = $journal['state']; + if(in_array($state, ['DISCOVERED', 'BACKED_UP', 'STAGING', 'VERIFIED', 'PUBLISHING'], true)){ + $this->verifyFingerprints($journal); + if($state === 'DISCOVERED'){ + $journal = $this->backup($journal); + $state = 'BACKED_UP'; + } elseif($state !== 'PUBLISHING'){ + Backup::verify($journal['backup']); + } + if($state === 'BACKED_UP' || $state === 'STAGING' || $state === 'VERIFIED'){ + $journal = $this->stage($journal); + } + $journal = $this->publish($journal); + $journal = $this->commit($journal); + } + return $this->finish($this->apply($journal)); + } + + private function backup(array $journal): array { + $store = $this->readStore(); + Backup::create(rtrim($this->dir, '/'), $journal['backup'], self::names($journal['files']), ['run' => $journal['id'], + 'dir' => realpath($this->dir), 'created' => time(), 'sourceKeyCheck' => $journal['sourceKeyCheck'], + 'databases' => $journal['targets'], 'store' => $store]); + $journal['state'] = 'BACKED_UP'; + $this->write($journal); + return $journal; + } + + /** Build every target in a staging store and verify it with the engine (§7.4, §7.5) */ + private function stage(array $journal): array { + $journal['state'] = 'STAGING'; + $this->write($journal); + $staging = $this->staging(); + Files::removeTree($staging); + Files::makeDir($staging); + $this->writeStore(rtrim($staging, '/'), ['storeId' => StoreRecord::uuid4(), 'seq' => 1, 'keys' => [StoreRecord::keyCheckOf($this->targetKey)]]); + $sources = $this->sources($journal); + foreach($sources as $source){ + Writer::write($staging, $this->targetKey, $source); + } + $db = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $staging, 'autoCreateDB' => false]); + $scan = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $staging, 'autoCreateDB' => false, 'fieldIndexEnabled' => false, 'spatialIndexEnabled' => false]); + $journal['records'] = []; + foreach($sources as $name => $source){ + // Records first: the engine writes each segment's checkpoint at its first read, one segment at a time, + // before the indexes read every segment in one operation + $samples = Verifier::records($db, $source); + foreach($source->fields as $field){ + $result = $db->createFieldIndex($name, $field); + if(empty($result['success'])){ + throw new Stop('FAILED', "$name: the field index of '$field' cannot be built: " . json_encode($result)); + } + } + foreach($source->spatial as $field => $nodeSize){ + $result = $db->createSpatialIndex($name, $field, $nodeSize === null ? [] : ['nodeSize' => $nodeSize]); + if(empty($result['success'])){ + throw new Stop('FAILED', "$name: the spatial index of '$field' cannot be built: " . json_encode($result)); + } + } + Verifier::totals($db, $scan, $source, $samples); + $journal['records'][$name] = $source->count; + } + unset($db, $scan); + $journal['state'] = 'VERIFIED'; + $this->write($journal); + return $journal; + } + + /** The targets read again from their files (verified unchanged) */ + private function sources(array $journal): array { + $sources = []; + foreach($journal['targets'] as $name){ + $source = Source::read($this->dir, $name, $this->sourceKey, array_keys($journal['files'][$name]), $journal['options']); + if($source->status !== 'READY'){ + throw new Stop($source->status, "$name: " . implode('; ', $source->reasons)); + } + $sources[$name] = $source; + } + return $sources; + } + + /** Move every staged database into the directory, each carrying m (§7.6) */ + private function publish(array $journal): array { + if($journal['state'] !== 'PUBLISHING'){ + $journal['state'] = 'PUBLISHING'; + $journal['published'] = array_map(function($name){ + return StoreRecord::locatorOf($name, $this->targetKey) . '.ndb4'; + }, $journal['targets']); + $this->write($journal); + } + foreach($journal['published'] as $locator){ + $staged = $this->staging() . $locator; + if(is_dir($this->dir . $locator) && !is_dir($staged)){ + continue; // published before the interruption + } + if(!is_dir($staged) || is_dir($this->dir . $locator)){ + throw new Stop('FAILED', "$locator is neither only staged nor only published: the run's state contradicts its files; nothing was changed"); + } + $this->verifyFingerprints($journal, $journal['targets'][array_search($locator, $journal['published'], true)]); + self::writeState($staged . '/LOCK', ['m']); + if(!@rename($staged, $this->dir . $locator)){ + throw new Stop('FAILED', "cannot move $locator into {$this->dir}"); + } + Files::syncDir(rtrim($this->dir, '/')); + Files::syncDir($this->staging()); + self::point('published'); // @test + } + return $journal; + } + + /** The commit point: the journal records COMMITTED with the target STORE record (§7.7) */ + private function commit(array $journal): array { + $this->verifyFingerprints($journal); + $store = $this->readStore(); + $keyCheck = StoreRecord::keyCheckOf($this->targetKey); + if($store === null){ + $store = ['storeId' => StoreRecord::uuid4(), 'seq' => 1, 'keys' => $journal['completesKey'] ? [$keyCheck] : []]; + } elseif($journal['completesKey'] && !in_array($keyCheck, $store['keys'], true)){ + $store['keys'][] = $keyCheck; + sort($store['keys']); + $store['seq']++; + } + $journal['state'] = 'COMMITTED'; + $journal['store'] = $store; + $this->write($journal); + return $journal; + } + + /** After the commit, each step idempotent: STORE copies, retained sources, m cleared, staging removed (§7.7) */ + private function apply(array $journal): array { + $this->writeStore(rtrim($this->dir, '/'), $journal['store']); + self::point('applied:store'); // @test + foreach(['.htaccess' => "Require all denied\nDeny from all\n", 'index.html' => ''] as $file => $content){ + if(!file_exists($this->dir . $file)){ + Files::writeNew($this->dir . $file, $content); // no temp file: one would be a foreign entry of the store + } + } + $retained = $this->dir . '.nonedb-retained/' . $journal['id']; + Files::makeDir($retained . '/files'); + foreach(self::prints($journal['files']) as $file => $fingerprint){ + if(is_file($this->dir . $file)){ + if(!@rename($this->dir . $file, "$retained/files/$file")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "committed, not completed: cannot move $file into $retained; run again"); + } + } elseif(!is_file("$retained/files/$file")){ + throw new Stop('FAILED', "$file is neither in the directory nor retained: the run's state contradicts its files"); + } + } + Files::syncDir("$retained/files"); + Files::syncDir(rtrim($this->dir, '/')); + $entries = []; + foreach(self::prints($journal['files']) as $file => $fingerprint){ + $entries[] = ['name' => $file, 'size' => $fingerprint[0], 'sha256' => $fingerprint[3]]; + } + Files::writeAtomic("$retained/manifest.json", StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'migration-retained', + 'run' => $journal['id'], 'backup' => $journal['backup'], 'files' => $entries])); + self::point('applied:retained'); // @test + foreach($journal['published'] as $locator){ + if(is_file($this->dir . $locator . '/LOCK')){ + self::writeState($this->dir . $locator . '/LOCK', []); + } + } + self::point('applied:unmarked'); // @test + Files::removeTree($this->staging()); + // Closed before the final check: the engine opens no store while a maintenance journal is open + Journal::close(rtrim($this->dir, '/'), $journal['backup'] . '/journal.done.json'); + self::point('closed'); // @test + return $journal; + } + + /** + * The closed journal of a run of these keys in this directory that has no final verification (a crash after the + * close, §10), found through the retained files' manifest; null when there is none + */ + private function unverified(): ?array { + foreach(glob($this->dir . '.nonedb-retained/*/manifest.json') ?: [] as $path){ + $retained = StoreRecord::openRecord(@file_get_contents($path), 'migration-retained'); + $done = $retained === null ? null : StoreRecord::openRecord(@file_get_contents($retained['backup'] . '/journal.done.json'), 'journal'); + if($done !== null && $done['op'] === 'migrate' && $done['state'] === 'COMMITTED' + && $done['sourceKeyCheck'] === StoreRecord::keyCheckOf($this->sourceKey) && $done['targetKeyCheck'] === StoreRecord::keyCheckOf($this->targetKey)){ + return $done; + } + } + return null; + } + + /** + * Final check (§7.8) with a new engine instance: checkIntegrity() and count() of every migrated database, and with + * --verify every record against the live set read from the backup. Recorded as FINAL_VERIFIED in the closed journal. + */ + private function finish(array $journal): array { + $report = ['run' => $journal['id'], 'backup' => $journal['backup'], 'databases' => $journal['records'], + 'keyRegistered' => $journal['completesKey'], 'retained' => '.nonedb-retained/' . $journal['id']]; + if($journal['completesKey']){ + \noneDB::clearStaticCache(); + $db = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->dir, 'autoCreateDB' => false]); + foreach($journal['records'] as $name => $count){ + if(($db->checkIntegrity($name)['ok'] ?? false) !== true || $db->count($name) !== $count){ + throw new Stop('FAILED', "$name: the published database does not read back as verified; the backup is in {$journal['backup']}"); + } + } + if($journal['options']['verify']){ + $scan = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->dir, 'autoCreateDB' => false, 'fieldIndexEnabled' => false, 'spatialIndexEnabled' => false]); + $backup = Discovery::scan($journal['backup'] . '/files', $this->sourceKey); + foreach($journal['targets'] as $name){ + Verifier::verify($db, $scan, Source::read($journal['backup'] . '/files', $name, $this->sourceKey, + $backup['databases'][$name], $journal['options'])); + } + } + unset($db, $scan); + } + // The report first: a closed journal that records FINAL_VERIFIED always has it next to it + Files::writeAtomic($journal['backup'] . '/report.json', json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n", 0600); + self::point('reported'); // @test + $journal['state'] = 'FINAL_VERIFIED'; + Files::writeAtomic($journal['backup'] . '/journal.done.json', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'journal'] + $journal), 0600); + return $report; + } + + // ------------------------------------------------------------------ store, locks, fingerprints + + /** + * STORE.LOCK, then the lock file of every target that has one (3.2, 3.3-dev), all non-blocking (§7.2) + * @throws Stop MAINTENANCE_BUSY, nothing written + */ + private function lock(array $targets): void { + if($this->locks !== []){ + return; + } + $paths = [$this->dir . 'STORE.LOCK']; + foreach($targets as $name){ + $path = $this->dir . Discovery::hashOf($name, $this->sourceKey) . "-$name.nonedb.lock"; + if(is_file($path)){ + $paths[] = $path; + } + } + foreach($paths as $path){ + $fp = Files::openLock($path); + $busy = $fp !== false && !flock($fp, LOCK_EX | LOCK_NB); + if($fp === false || $busy){ + if($fp !== false){ + fclose($fp); + } + foreach($this->locks as $held){ + flock($held, LOCK_UN); + fclose($held); + } + $this->locks = []; + throw $busy ? new Stop('MAINTENANCE_BUSY', basename($path) . ' is locked by another process; nothing was changed') + : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); + } + $this->locks[] = $fp; + } + } + + /** The run belongs to these keys: a journal of another source or target key is never resumed */ + private function own(array $journal): void { + if(($journal['op'] ?? null) !== 'migrate'){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "another maintenance operation ({$journal['op']}) is open in {$this->dir}"); + } + if($journal['sourceKeyCheck'] !== StoreRecord::keyCheckOf($this->sourceKey) || $journal['targetKeyCheck'] !== StoreRecord::keyCheckOf($this->targetKey)){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "the interrupted run {$journal['id']} was started with other keys; run it again with them"); + } + } + + /** group => file => [size, mtime, inode, sha256] */ + private function fingerprints(array $groups): array { + $prints = []; + foreach($groups as $group => $files){ + $prints[$group] = []; + foreach($files as $file){ + $prints[$group][$file] = $this->fingerprint($file); + } + } + return $prints; + } + + private function fingerprint(string $file): ?array { + clearstatcache(true, $this->dir . $file); + $stat = @stat($this->dir . $file); + return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], hash_file('sha256', $this->dir . $file)]; + } + + /** Every file of the groups: file => fingerprint */ + private static function prints(array $groups): array { + return array_merge(...array_values($groups)); + } + + private static function names(array $groups): array { + $names = array_keys(self::prints($groups)); + sort($names); + return $names; + } + + /** + * The source files (of one database, or all) as the journal recorded them (§7.6); a change rolls the run back + * @throws Stop AMBIGUOUS_SOURCE + */ + private function verifyFingerprints(array $journal, ?string $group = null): void { + foreach($group === null ? self::prints($journal['files']) : $journal['files'][$group] as $file => $print){ + if($this->fingerprint($file) !== $print){ + foreach($this->locks as $held){ + flock($held, LOCK_UN); + fclose($held); + } + $this->locks = []; + self::abort(rtrim($this->dir, '/')); + throw new Stop('AMBIGUOUS_SOURCE', "$file changed during the run; the run was rolled back and nothing was migrated"); + } + } + } + + /** + * The STORE record of the directory, null when there is none + * @throws Stop a store whose copies are not valid and identical + */ + private function readStore(): ?array { + $copies = []; + foreach(['STORE', 'STORE.2'] as $name){ + $path = $this->dir . $name; + if(is_file($path)){ + $copies[$name] = StoreRecord::openRecord(@file_get_contents($path), 'store'); + } + } + if($copies === []){ + return null; + } + if(count($copies) !== 2 || in_array(null, $copies, true) || $copies['STORE'] !== $copies['STORE.2']){ + throw new Stop('FAILED', "the STORE record of {$this->dir} is not two valid identical copies: open the store with noneDB once (it repairs one damaged copy) or use repairStore()"); + } + $store = $copies['STORE']; + return ['storeId' => $store['storeId'], 'seq' => $store['seq'], 'keys' => $store['keys']]; + } + + /** Both STORE copies (STORE.2, then STORE), each written only when it differs (STORAGE_FORMAT.md §3.1) */ + private function writeStore(string $dir, array $store): void { + $bytes = StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT] + $store); + foreach(['STORE.2', 'STORE'] as $name){ + if(@file_get_contents("$dir/$name") !== $bytes){ + Files::writeAtomic("$dir/$name", $bytes); + } + } + } + + /** A database's state record with a new epoch and $flags (STORAGE_FORMAT.md §4.2), synced */ + private static function writeState(string $lockPath, array $flags): void { + $record = (string)@file_get_contents($lockPath, false, null, 0, 128); + $gen = preg_match('/^(\d+):/', $record, $m) ? (int)$m[1] : 1; + $fp = @fopen($lockPath, 'c+'); + $state = str_pad(implode(' ', array_merge([$gen . ':' . bin2hex(random_bytes(4))], $flags)), 128); + if($fp === false || fwrite($fp, $state) !== 128 || !fflush($fp) || !Files::syncHandle($fp)){ + throw new Stop('FAILED', "cannot write the state record of $lockPath"); + } + fclose($fp); + } + + private function write(array $journal): void { + Journal::write(rtrim($this->dir, '/'), $journal); + self::point('state:' . $journal['state']); // @test + } + + // @test-begin + /** + * Test seams at a point of the run: NONEDB_MIGRATE_TRACE (a file) gets the name of every point reached; + * NONEDB_MIGRATE_CHANGE ({"at": point, "nth": n, "files": [...], "append": bytes}) appends to source files the n-th + * time the point is reached (another process writing during the run); the process stops with SIGKILL at the point + * NONEDB_MIGRATE_KILL names (a crash there) + */ + private static function point(string $name): void { + static $reached = []; + $reached[$name] = ($reached[$name] ?? 0) + 1; + if(($trace = getenv('NONEDB_MIGRATE_TRACE')) !== false){ + file_put_contents($trace, $name . "\n", FILE_APPEND); + } + $change = json_decode((string)getenv('NONEDB_MIGRATE_CHANGE'), true); + if(is_array($change) && $change['at'] === $name && $change['nth'] === $reached[$name]){ + foreach($change['files'] as $file){ + file_put_contents($file, $change['append'], FILE_APPEND); + } + } + if(getenv('NONEDB_MIGRATE_KILL') === $name){ + posix_kill(getmypid(), 9); + } + } + // @test-end + + private function staging(): string { + return $this->dir . '.nonedb-maintenance/staging/'; + } +} diff --git a/migrate/Readers/Discovery.php b/migrate/Readers/Discovery.php new file mode 100644 index 0000000..6375781 --- /dev/null +++ b/migrate/Readers/Discovery.php @@ -0,0 +1,102 @@ + its files (a data file, a meta, or a write buffer holding records); + * - leftovers: name => files of the key that belong to no database; + * - keyFiles: files of the directory that only this key uses (its 3.2+ hash cache); + * - sharedFiles: the 3.0/3.1 hash cache, which no key owns; + * - otherKeys: files of the old formats that belong to another key; + * - foreign: anything else that is neither a source nor part of a store; + * - store: entries of a noneDB 4 store. + * @return array + */ + public static function scan(string $dir, string $key): array { + $out = ['databases' => [], 'leftovers' => [], 'keyFiles' => [], 'sharedFiles' => [], 'otherKeys' => [], 'foreign' => [], 'store' => []]; + $named = []; + $hashOnly = []; + $fingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', $key), 0, 16); + foreach(scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(preg_match(self::STORE_ENTRY, $entry)){ + $out['store'][] = $entry; + } elseif(preg_match('/^([0-9a-f]{20})-([^_.]*)(?:_s\d+)?\.nonedb/', $entry, $m)){ + $named[$m[2]][$m[1]][] = $entry; + } elseif(preg_match('/^([0-9a-f]{20})(?:_s\d+)?\.nonedb\./', $entry, $m)){ + $hashOnly[$m[1]][] = $entry; + } elseif($entry === '.nonedb_hash_cache'){ + $out['sharedFiles'][] = $entry; + } elseif(preg_match('/^\.nonedb_hash_cache_([0-9a-f]{16})$/', $entry, $m)){ + $out[$m[1] === $fingerprint ? 'keyFiles' : 'otherKeys'][] = $entry; + } else { + $out['foreign'][] = $entry; + } + } + $ownHashes = []; + foreach($named as $name => $byHash){ + $hash = self::hashOf((string)$name, $key); + foreach($byHash as $h => $files){ + if($h === $hash){ + $ownHashes[$h] = (string)$name; + $out['databases'][(string)$name] = $files; + } else { + array_push($out['otherKeys'], ...$files); + } + } + } + foreach($hashOnly as $h => $files){ + if(isset($ownHashes[$h])){ + array_push($out['databases'][$ownHashes[$h]], ...$files); + } else { + array_push($out['otherKeys'], ...$files); + } + } + foreach($out['databases'] as $name => $files){ + sort($files); + if(!self::isDatabase($dir, (string)$name, self::hashOf((string)$name, $key), $files)){ + unset($out['databases'][$name]); + $out['leftovers'][$name] = $files; + } else { + $out['databases'][$name] = $files; + } + } + ksort($out['databases'], SORT_STRING); + ksort($out['leftovers'], SORT_STRING); + foreach(['keyFiles', 'sharedFiles', 'otherKeys', 'foreign', 'store'] as $list){ + sort($out[$list]); + } + return $out; + } + + /** The name hash of every release: PBKDF2-SHA256, the name as password, the key as salt, 20 hex digits */ + public static function hashOf(string $name, string $key): string { + return hash_pbkdf2('sha256', $name, $key, 1000, 20); + } + + /** + * A database has a data file (its own or a shard's), a meta, or a write buffer holding a record + */ + private static function isDatabase(string $dir, string $name, string $hash, array $files): bool { + $base = $hash . '-' . $name; + foreach($files as $file){ + if($file === "$base.nonedb" || $file === "$base.nonedb.meta" || preg_match('/^' . preg_quote($base, '/') . '_s\d+\.nonedb$/', $file)){ + return true; + } + if(preg_match('/\.nonedb\.buffer(\.flushing(\.\d+)?)?$/', $file) && trim((string)@file_get_contents($dir . '/' . $file)) !== ''){ + return true; + } + } + return false; + } +} diff --git a/migrate/Readers/Source.php b/migrate/Readers/Source.php new file mode 100644 index 0000000..7a987f2 --- /dev/null +++ b/migrate/Readers/Source.php @@ -0,0 +1,658 @@ + kind (W, J30, J32, J33, EMPTY) */ + public $kinds = []; + /** @var string single or sharded */ + public $layout = 'single'; + /** @var int */ + public $shardSize; + /** @var int */ + public $nextKey = 0; + /** @var int live records, buffered ones included */ + public $count = 0; + /** @var int records imported from write buffers */ + public $buffered = 0; + /** @var string[] indexed fields */ + public $fields = []; + /** @var array spatial field => node size (null: the engine's default) */ + public $spatial = []; + /** @var int[] what is not carried over: reason => lines or entries */ + public $notCarried = []; + + /** @var string */ + private $dir; + /** @var string "{hash}-{name}" */ + private $base; + /** @var string "{hash}" */ + private $hash; + /** @var array options: rebuildFromData, flushing (null, import, skip), shardSize */ + private $options; + /** @var bool the lock record has an epoch: 3.3-dev wrote the database */ + private $epoch = false; + /** @var bool a 3.2+ lock file exists */ + private $locked = false; + /** + * @var array segment id => [kind, path, jidx (its index file), slots: next local key, count: live records]. The live + * records themselves are read again per segment when written or verified (content()), so memory stays bounded by + * the largest segment + */ + private $segments = []; + /** @var bool reading again (content()): what was checked and reported is neither checked nor reported twice */ + private $quiet = false; + /** @var array [segment id, its content] of the last segment read again */ + private $content = [null, null]; + /** @var array key => record of buffered records, by segment id */ + private $bufferedRecords = []; + + /** Line encoding of the engine (STORAGE_FORMAT.md §2, §6.1) */ + private const JSON = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRESERVE_ZERO_FRACTION; + + /** + * Read the database $name of $dir from its files (as Discovery grouped them) + * @param array $options rebuildFromData (bool), flushing (null|'import'|'skip'), shardSize (int, single layout) + */ + public static function read(string $dir, string $name, string $key, array $files, array $options): self { + $source = new self(); + $source->dir = rtrim($dir, '/') . '/'; + $source->name = $name; + $source->files = $files; + $source->hash = Discovery::hashOf($name, $key); + $source->base = $source->hash . '-' . $name; + $source->options = $options + ['rebuildFromData' => false, 'flushing' => null, 'shardSize' => 10000]; + try { + $source->readAll(); + } catch(Stop $e){ + $source->status = $e->result; + $source->reasons[] = $e->getMessage(); + } + return $source; + } + + private function readAll(): void { + if(preg_match("/^[A-Za-z0-9' -]+$/D", $this->name) !== 1){ + throw new Stop('UNSUPPORTED_SOURCE', "the name '{$this->name}' is not a valid noneDB 4 database name"); + } + $lock = $this->path('.nonedb.lock'); + if(is_file($lock)){ + $this->locked = true; + $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', (string)file_get_contents($lock, false, null, 0, 128)) === 1; + } + if(is_file($this->path('.nonedb.meta'))){ + $this->readSharded(); + } elseif(is_file($this->path('.nonedb'))){ + $this->shardSize = (int)$this->options['shardSize']; + $this->summarize('main', $this->path('.nonedb'), $this->path('.nonedb.jidx')); + } elseif(preg_grep('/^' . preg_quote($this->base, '/') . '_s\d+\.nonedb$/', $this->files)){ + throw new Stop('AMBIGUOUS_SOURCE', 'shard files without a meta: an interrupted drop or a lost meta'); + } else { + $this->shardSize = (int)$this->options['shardSize']; + $this->segments['main'] = ['kind' => 'EMPTY', 'path' => null, 'jidx' => null, 'slots' => 0, 'count' => 0]; + } + foreach($this->segments as $id => $segment){ + $this->kinds[$id] = $segment['kind']; + $this->count += $segment['count']; + } + $this->readBuffers(); + $this->readIndexList(); + $this->count += $this->buffered; + } + + /** Read a segment once to check it and count it (§4.1, §4.2); its records are not kept */ + private function summarize($id, string $path, string $jidxPath): void { + $segment = $this->readSegment($id, $path, $jidxPath); + $this->segments[$id] = ['kind' => $segment['kind'], 'path' => $path, 'jidx' => $jidxPath, 'slots' => $segment['slots'], + 'count' => count($segment['live'] ?? $segment['records'])]; + } + + /** + * A segment's live records, read again: key => [offset, length] of a JSONL line, or key => record (W); [] for a + * segment that holds none + */ + private function content($id): array { + if($this->content[0] !== $id){ + $segment = $this->segments[$id] ?? null; + $this->quiet = true; + try { + $read = $segment === null || $segment['kind'] === 'EMPTY' ? [] : $this->readSegment($id, $segment['path'], $segment['jidx'], $segment['kind']); + } finally { + $this->quiet = false; + } + $this->content = [$id, $read['live'] ?? $read['records'] ?? []]; + } + return $this->content[1]; + } + + // ------------------------------------------------------------------ layout + + private function readSharded(): void { + $this->layout = 'sharded'; + $meta = json_decode((string)@file_get_contents($this->path('.nonedb.meta')), true); + if(!is_array($meta) || !is_array($meta['shards'] ?? null) || !is_int($meta['shardSize'] ?? null) || $meta['shardSize'] < 1){ + throw new Stop('AMBIGUOUS_SOURCE', 'the meta is unreadable'); + } + if(($meta['version'] ?? null) !== 1){ + throw new Stop('UNSUPPORTED_SOURCE', 'the meta has a version no release wrote'); + } + $this->shardSize = $meta['shardSize']; + if(isset($meta['nextKey']) && is_int($meta['nextKey'])){ + $this->raiseNextKey($meta['nextKey']); + } + foreach($meta['shards'] as $shard){ + $id = $shard['id'] ?? null; + if(!is_int($id) || $id < 0 || isset($this->segments[$id])){ + throw new Stop('AMBIGUOUS_SOURCE', 'the meta lists a shard id that is not a distinct non-negative integer'); + } + $this->summarize($id, $this->path("_s$id.nonedb"), $this->path("_s$id.nonedb.jidx")); + } + } + + // ------------------------------------------------------------------ segments + + /** + * A segment's live set by its kind (§4.1); shard keys checked against the shard size (§4.2). $decided: the kind a + * first read decided (read again: the same rule, without deciding anew) + */ + private function readSegment($id, string $path, string $jidxPath, ?string $decided = null): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + if(!is_file($path) || filesize($path) === 0){ + $segment = ['kind' => 'EMPTY', 'path' => null, 'live' => [], 'slots' => 0]; + $index = is_file($jidxPath) ? json_decode((string)file_get_contents($jidxPath), true) : null; + if(is_array($index) && is_int($index['n'] ?? null)){ + $segment['slots'] = $index['n']; + $this->raiseNextKey($base + $index['n']); + } + return $segment; + } + $whole = $decided === null || $decided === 'W' ? $this->wholeJson($path) : null; + if($whole !== null){ + return $this->readWhole($id, $whole); + } + $index = is_file($jidxPath) ? json_decode((string)file_get_contents($jidxPath), true) : null; + if($decided !== null){ + $kind = $decided; + } elseif(is_array($index) && is_array($index['p'] ?? null) && is_array($index['l'] ?? null) && isset($index['s'])){ + $kind = 'J33'; + } elseif(is_array($index) && is_array($index['o'] ?? null) && ($index['v'] ?? null) === 3){ + $kind = isset($index['s']) ? 'J32' : 'J30'; + } elseif($this->epoch){ + $kind = 'J33'; // 3.3-dev: the data file is the database, its checkpoints (written now and then) only a shortcut + } elseif($this->options['rebuildFromData']){ + $kind = 'J33'; + $this->noteNotCarried('index replaced by the data file (--rebuild-from-data)', 1); + } else { + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ': ' . (is_file($jidxPath) ? 'its key index is unreadable' : 'its key index is missing') + . ' (--rebuild-from-data replays the data file; records an older release deleted may come back)'); + } + if($kind === 'J33'){ + $segment = $this->replay($id, $path); + } else { + try { + $segment = $this->indexed($id, $path, $index, $kind); + } catch(Stop $e){ + if(!$this->options['rebuildFromData']){ + throw $e; + } + $kind = 'J33'; // an index that doesn't fit its data, replaced by a replay on request (§6.2) + $this->noteNotCarried('index replaced by the data file (--rebuild-from-data)', 1); + $segment = $this->replay($id, $path); + } + } + if(is_array($index) && is_int($index['n'] ?? null)){ + // The index's counter: the next key of a single segment, the next local key of a shard + $this->raiseNextKey($base + $index['n']); + $segment['slots'] = max($segment['slots'], $index['n']); + } + if($id !== 'main' && $kind !== 'J33'){ + foreach($segment['live'] as $k => $where){ + if(intdiv($k, $this->shardSize) !== $id){ + throw new Stop('AMBIGUOUS_SOURCE', "shard $id holds key $k, outside its range for the meta's shard size {$this->shardSize}"); + } + } + } + $segment['kind'] = $kind; + return $segment; + } + + /** The decoded file when it is one JSON object whose only member is "data", an array; else null */ + private function wholeJson(string $path): ?array { + $fp = fopen($path, 'rb'); + $head = (string)fread($fp, 16); + fclose($fp); + if(preg_match('/^\s*\{\s*"data"\s*:/', $head) !== 1){ + return null; + } + $json = json_decode((string)file_get_contents($path), true); + return is_array($json) && array_keys($json) === ['data'] && is_array($json['data']) ? $json['data'] : null; + } + + /** W: every array entry is a record keyed by its position; other entries only count for the next key */ + private function readWhole($id, array $data): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + $records = []; + $slots = 0; + foreach($data as $i => $entry){ + if(!is_int($i) || $i < 0){ + throw new Stop('AMBIGUOUS_SOURCE', "segment $id: its data has a member that is not a position"); + } + if($id !== 'main' && $i >= $this->shardSize){ + throw new Stop('AMBIGUOUS_SOURCE', "shard $id holds more entries than the meta's shard size {$this->shardSize}"); + } + $slots = max($slots, $i + 1); + if(is_array($entry)){ + $records[$base + $i] = $entry; + } elseif($entry !== null){ + $this->noteNotCarried('an entry that is not a record', 1); + } + } + ksort($records); + $this->raiseNextKey($base + $slots); + return ['kind' => 'W', 'path' => null, 'records' => $records, 'slots' => $slots]; + } + + /** + * J30, J32: the index's map is the live set. Every entry must point at a line that decodes to its key; an entry of + * length 0 is not live; J32 lines after s are not live. + */ + private function indexed($id, string $path, array $index, string $kind): array { + $size = filesize($path); + $fp = fopen($path, 'rb'); + $end = $size; + if($kind === 'J32'){ + $s = $index['s']; + if(!is_int($s) || $s < 0 || $s > $size || ($s > 0 && $this->byteAt($fp, $s - 1) !== "\n")){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ': its key index covers bytes the data file does not have' + . ' (--rebuild-from-data replays the data file)'); + } + $end = $s; + } + $live = []; + foreach($index['o'] as $k => $where){ + $k = (int)$k; + if(!is_array($where) || count($where) !== 2 || !is_int($where[0]) || !is_int($where[1]) || $where[0] < 0 || $where[1] < 0){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ": the key index entry of key $k is malformed"); + } + if($where[1] === 0){ + $this->noteNotCarried('a key index entry of length 0', 1); + continue; + } + if($this->quiet){ + $live[$k] = $where; // read again: checked when first read, and the files are unchanged (fingerprints) + continue; + } + fseek($fp, $where[0]); + $line = (string)fread($fp, $where[1]); + $record = strlen($line) === $where[1] && $where[0] + $where[1] <= $end ? json_decode($line, true) : null; + if(!is_array($record) || ($record['key'] ?? null) !== $k){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ": the key index points key $k at bytes that are not its record" + . ' (--rebuild-from-data replays the data file)'); + } + $live[$k] = $where; + } + fclose($fp); + ksort($live); + if($this->quiet){ + return ['path' => $path, 'live' => $live, 'slots' => 0]; + } + $scan = $this->scanLines($path, $kind === 'J32' ? $end : null); + $this->noteNotCarried('lines that are not live (deleted or superseded records, tombstones)', $scan['before'] - count($live)); + if($scan['after'] > 0){ + $this->noteNotCarried('complete lines after the key index (3.2 never showed them)', $scan['after']); + } + return ['path' => $path, 'live' => $live, 'slots' => 0]; + } + + /** + * J33: the data file is the database. The last line of a key wins, a tombstone removes it; a line that does + * not decode is salvaged (STORAGE_FORMAT.md §6.1); shard lines outside the shard's range are not live. + */ + private function replay($id, string $path): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + $live = []; + $slots = 0; + $fp = fopen($path, 'rb'); + $offset = 0; + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + if($line[$length - 1] !== "\n"){ + $this->noteNotCarried('an unterminated last line', 1); + break; + } + $line = substr($line, 0, -1); + [$type, $k, $at] = $this->classify($line); + if($type !== null){ + $this->raiseNextKey($k + 1); + if($id !== 'main' && intdiv($k, $this->shardSize) !== $id){ + $type = null; + } else { + $slots = max($slots, $k - $base + 1); + } + } + if($type !== null && $at > 0){ + $this->noteNotCarried('bytes glued before a record in its line (the record is salvaged)', 1); + } + if($type === 'record'){ + $live[$k] = [$offset + $at, $length - 1 - $at]; + } elseif($type === 'tombstone'){ + unset($live[$k]); + } else { + $this->noteNotCarried('lines that are neither a record nor a tombstone', 1); + } + $offset += $length; + } + fclose($fp); + ksort($live); + return ['path' => $path, 'live' => $live, 'slots' => $slots]; + } + + /** + * [record|tombstone|null, key, offset of the JSON in the line]: a whole line, else the salvage rule: from right + * to left, the first "{" from which a record or tombstone decodes to the line end + */ + private function classify(string $line): array { + $value = json_decode($line, true); + if(is_array($value)){ + return $this->kindOf($value, 0); + } + for($at = strrpos($line, '{'), $tries = 0; $at !== false && $at > 0 && $tries < 4096; $tries++){ + $value = json_decode(substr($line, $at), true); + if(is_array($value) && ($kind = $this->kindOf($value, $at))[0] !== null){ + return $kind; + } + $at = strrpos($line, '{', $at - strlen($line) - 1); + } + return [null, 0, 0]; + } + + private function kindOf(array $value, int $at): array { + if(count($value) === 1 && is_int($value['__nonedb_deleted'] ?? null) && $value['__nonedb_deleted'] >= 0){ + return ['tombstone', $value['__nonedb_deleted'], $at]; + } + if(is_int($value['key'] ?? null) && $value['key'] >= 0){ + return ['record', $value['key'], $at]; + } + return [null, 0, 0]; + } + + /** + * Every key in a JSONL data file raises the next key (records and tombstones, live or not). Returns the number + * of complete lines before $end (the bytes a 3.2 index covers; null: all) and after it. + */ + private function scanLines(string $path, ?int $end): array { + $fp = fopen($path, 'rb'); + $counts = ['before' => 0, 'after' => 0]; + $offset = 0; + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + if($line[$length - 1] !== "\n"){ + break; + } + [$type, $k] = $this->classify(substr($line, 0, -1)); + if($type !== null){ + $this->raiseNextKey($k + 1); + } + $counts[$end !== null && $offset >= $end ? 'after' : 'before']++; + $offset += $length; + } + fclose($fp); + return $counts; + } + + private function byteAt($fp, int $offset): string { + fseek($fp, $offset); + return (string)fread($fp, 1); + } + + // ------------------------------------------------------------------ buffers (§4.3) + + private function readBuffers(): void { + $current = []; + $flushingK = []; + $plain = []; + foreach($this->files as $file){ + if(!preg_match('/^' . preg_quote($this->base, '/') . '(?:_s(\d+))?\.nonedb\.buffer(\.flushing(?:\.(\d+))?)?$/', $file, $m)){ + continue; + } + $records = $this->bufferRecords($this->dir . $file); + if($records === []){ + continue; + } + $shard = ($m[1] ?? '') === '' ? null : (int)$m[1]; + if(($m[2] ?? '') === ''){ + $current[$shard ?? 'main'] = $records; + } elseif(($m[3] ?? '') === ''){ + $plain[] = [$file, $records]; + } else { + $flushingK[(int)$m[3]] = [$file, $records]; + } + } + // 1. shard buffers of 2.3-3.1: after the shard's slots + foreach($current as $shard => $records){ + if($shard === 'main'){ + continue; + } + if(!isset($this->segments[$shard])){ + throw new Stop('AMBIGUOUS_SOURCE', "a write buffer of shard $shard, which the meta does not list (no release flushed it)"); + } + $next = $shard * $this->shardSize + $this->segments[$shard]['slots']; + foreach($records as $record){ + $this->addBuffered($next++, $record); + } + } + // 2. interrupted flushes of 3.2/3.3-dev: from K on, skipping keys already stored + ksort($flushingK); + foreach($flushingK as $k => [$file, $records]){ + foreach($records as $record){ + if(!$this->isLive($k)){ + $this->addBuffered($k, $record); + } + $k++; + } + } + // 3. the current buffer: from the next key + if(isset($current['main'])){ + if($this->layout === 'sharded' && !$this->locked){ + throw new Stop('AMBIGUOUS_SOURCE', 'a write buffer next to a meta, which releases before 3.2 never flushed'); + } + $next = $this->nextKey; + foreach($current['main'] as $record){ + $this->addBuffered($next++, $record); + } + } + // 4. interrupted flushes of 2.3-3.1: only on request, fresh keys after everything else + if($plain !== []){ + if($this->options['flushing'] === null){ + throw new Stop('AMBIGUOUS_SOURCE', 'an interrupted flush of a release before 3.2 (' . implode(', ', array_column($plain, 0)) + . '): its records may or may not be stored already (--flushing=import or --flushing=skip)'); + } + foreach($plain as [$file, $records]){ + if($this->options['flushing'] === 'skip'){ + $this->noteNotCarried('records of an interrupted flush (--flushing=skip)', count($records)); + continue; + } + $next = $this->nextKey; + foreach($records as $record){ + $this->addBuffered($next++, $record); + } + } + } + } + + /** The records of a buffer file: lines that decode to an array (blank lines ignored, others reported) */ + private function bufferRecords(string $path): array { + $records = []; + foreach(explode("\n", (string)file_get_contents($path)) as $line){ + if(trim($line) === ''){ + continue; + } + $record = json_decode($line, true); + if(is_array($record)){ + $records[] = $record; + } else { + $this->noteNotCarried('buffer lines that are not a record', 1); + } + } + return $records; + } + + private function addBuffered(int $key, array $record): void { + unset($record['key']); + $id = $this->layout === 'single' ? 'main' : intdiv($key, $this->shardSize); + $this->bufferedRecords[$id][$key] = $record; + $this->buffered++; + $this->raiseNextKey($key + 1); + } + + private function isLive(int $key): bool { + $id = $this->layout === 'single' ? 'main' : intdiv($key, $this->shardSize); + return isset($this->bufferedRecords[$id][$key]) || isset($this->content($id)[$key]); + } + + // ------------------------------------------------------------------ indexes + + /** Indexed fields and spatial fields: the release's index list, else the headers of its index files */ + private function readIndexList(): void { + $list = json_decode((string)@file_get_contents($this->path('.nonedb.indexes')), true); + if(is_array($list) && is_array($list['fields'] ?? null) && is_array($list['spatial'] ?? null)){ + $this->fields = array_values(array_filter($list['fields'], 'is_string')); + foreach($list['spatial'] as $field){ + if(is_string($field)){ + $size = $list['spatialOptions'][$field]['nodeSize'] ?? null; + $this->spatial[$field] = is_int($size) ? $size : null; + } + } + } else { + foreach($this->files as $file){ + if(preg_match('/\.nonedb\.(fidx|gfidx|fx|gfx)\./', $file)){ + $field = $this->headerField($this->dir . $file); + if($field !== null && !in_array($field, $this->fields, true)){ + $this->fields[] = $field; + } + } elseif(preg_match('/\.nonedb\.(sidx|gsidx|rx|grx)\./', $file)){ + $header = $this->header($this->dir . $file); + if(is_string($header['field'] ?? null) && !array_key_exists($header['field'], $this->spatial)){ + $size = $header['nodeSize'] ?? $header['ns'] ?? null; + $this->spatial[$header['field']] = is_int($size) ? $size : null; + } + } + } + } + sort($this->fields, SORT_STRING); + ksort($this->spatial, SORT_STRING); + } + + private function headerField(string $path): ?string { + $header = $this->header($path); + return is_string($header['field'] ?? null) ? $header['field'] : null; + } + + /** An index file's header: its first line as JSON (3.1 files are JSON encoded twice) */ + private function header(string $path): array { + $fp = @fopen($path, 'rb'); + $line = $fp === false ? false : fgets($fp); + if($fp !== false){ + fclose($fp); + } + $value = $line === false ? null : json_decode($line, true); + if(is_string($value)){ + $value = json_decode($value, true); + } + return is_array($value) ? $value : []; + } + + // ------------------------------------------------------------------ output + + /** + * The record lines of a segment in key order, without "\n": a JSONL line as stored when it ends with its key, + * else the record encoded as the engine encodes one + * @return \Generator key => line + */ + public function lines($id): \Generator { + $content = $this->content($id); + $kind = $this->segments[$id]['kind'] ?? 'EMPTY'; + $buffered = $this->bufferedRecords[$id] ?? []; + $keys = array_merge(array_keys($content), array_keys($buffered)); + sort($keys); + $fp = $kind !== 'W' && $content !== [] ? fopen($this->segments[$id]['path'], 'rb') : null; + try { + foreach($keys as $k){ + if(isset($buffered[$k])){ + yield $k => self::encode($buffered[$k], $k); + } elseif($fp !== null){ + [$offset, $length] = $content[$k]; + fseek($fp, $offset); + $line = (string)fread($fp, $length); + yield $k => substr($line, -strlen(",\"key\":$k}")) === ",\"key\":$k}" || $line === "{\"key\":$k}" + ? $line : self::encode(json_decode($line, true), $k); + } else { + yield $k => self::encode($content[$k], $k); + } + } + } finally { + if($fp !== null){ + fclose($fp); + } + } + } + + /** Segment ids that hold records (sharded: shard ids; single: main) */ + public function segmentIds(): array { + $ids = []; + foreach($this->segments as $id => $segment){ + if($segment['count'] > 0){ + $ids[$id] = true; + } + } + foreach($this->bufferedRecords as $id => $records){ + $ids[$id] = true; + } + $ids = array_keys($ids); + sort($ids); + return $ids; + } + + /** A record as a data line: its fields, then "key" (STORAGE_FORMAT.md §6.1) */ + public static function encode(array $record, int $key): string { + unset($record['key']); + $record['key'] = $key; + try { + return json_encode($record, self::JSON | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + throw new Stop('FAILED', "record $key cannot be encoded: " . $e->getMessage()); + } + } + + private function path(string $suffix): string { + return $this->dir . $this->base . $suffix; + } + + private function raiseNextKey(int $key): void { + if($key > $this->nextKey){ + $this->nextKey = $key; + } + } + + private function noteNotCarried(string $reason, int $count): void { + if($count > 0 && !$this->quiet){ + $this->notCarried[$reason] = ($this->notCarried[$reason] ?? 0) + $count; + } + } +} diff --git a/migrate/Restore/Restore.php b/migrate/Restore/Restore.php new file mode 100644 index 0000000..caf6ce3 --- /dev/null +++ b/migrate/Restore/Restore.php @@ -0,0 +1,171 @@ + 'restore-old', 'id' => $runId, 'state' => 'RESTORING']); + Files::makeDir("$runDir/v4", 0700); + foreach($done['published'] as $locator){ + if(is_dir($dir . $locator) && !@rename($dir . $locator, "$runDir/v4/$locator")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $locator into the backup; run --restore again"); + } + } + self::removeKey($dir, $runDir, $done['targetKeyCheck']); + $retained = $dir . '.nonedb-retained/' . $runId . '/files/'; + foreach($manifest['files'] as $entry){ + $name = $entry['name']; + if(is_file($dir . $name) && hash_file('sha256', $dir . $name) === $entry['sha256']){ + continue; + } + if(is_file($retained . $name) && hash_file('sha256', $retained . $name) === $entry['sha256']){ + if(!@rename($retained . $name, $dir . $name)){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $name back; run --restore again"); + } + } elseif(Files::copy("$runDir/files/$name", $dir . $name) !== $entry['sha256']){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "$name does not copy back byte for byte; run --restore again"); + } + } + Files::syncDir(rtrim($dir, '/')); + if(is_dir($dir . '.nonedb-retained/' . $runId)){ + Files::removeTree($dir . '.nonedb-retained/' . $runId); + } + Journal::close(rtrim($dir, '/'), "$runDir/restore.done.json"); + if(!is_file($dir . 'STORE') && !is_file($dir . 'STORE.2')){ + // No store is left: the files the run made, and the store's directories when empty, go too + foreach($done['created'] as $file){ + if(is_file($dir . $file)){ + Files::removeTree($dir . $file); + } + } + foreach(['.nonedb-maintenance', '.nonedb-retained'] as $sub){ + if(is_dir($dir . $sub) && (scandir($dir . $sub) ?: []) === ['.', '..']){ + Files::removeTree($dir . $sub); + } + } + Files::syncDir(rtrim($dir, '/')); + } + foreach($locks as $fp){ + flock($fp, LOCK_UN); + fclose($fp); + } + return ['run' => $runId, 'restored' => count($manifest['files']), 'v4' => "$runDir/v4"]; + } + + /** + * Remove .nonedb-retained/$runId after verifying every file against the backup's manifest (§12) + * @throws Stop + */ + public static function cleanup(string $dir, string $backupDir, string $runId): array { + $dir = rtrim($dir, '/') . '/'; + $manifest = Backup::verify(rtrim($backupDir, '/') . '/' . $runId); + $retained = $dir . '.nonedb-retained/' . $runId; + if(!is_dir($retained)){ + return ['run' => $runId, 'removed' => 0]; + } + $done = StoreRecord::openRecord(@file_get_contents(rtrim($backupDir, '/') . "/$runId/journal.done.json"), 'journal'); + if(($done['state'] ?? null) !== 'FINAL_VERIFIED'){ + throw new Stop('FAILED', "the run $runId has no final verification; run the tool again first (it verifies the run); nothing was removed"); + } + foreach($manifest['files'] as $entry){ + $path = "$retained/files/" . $entry['name']; + if(is_file($path) && hash_file('sha256', $path) !== $entry['sha256']){ + throw new Stop('FAILED', "$path differs from the backup; nothing was removed"); + } + } + Files::removeTree($retained); + Files::syncDir($dir . '.nonedb-retained'); + return ['run' => $runId, 'removed' => count($manifest['files'])]; + } + + /** STORE.LOCK, then the LOCK of every database of the run, all non-blocking */ + private static function lock(string $dir, array $published): array { + $locks = []; + $paths = [$dir . 'STORE.LOCK']; + foreach($published as $locator){ + if(is_file($dir . $locator . '/LOCK')){ + $paths[] = $dir . $locator . '/LOCK'; + } + } + foreach($paths as $path){ + $fp = Files::openLock($path); + $busy = $fp !== false && !flock($fp, LOCK_EX | LOCK_NB); + if($fp === false || $busy){ + if($fp !== false){ + fclose($fp); + } + foreach($locks as $held){ + fclose($held); + } + throw $busy ? new Stop('MAINTENANCE_BUSY', basename(dirname($path)) . '/' . basename($path) . ' is locked by another process; nothing was changed') + : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); + } + $locks[] = $fp; + } + return $locks; + } + + /** The run's key leaves the STORE record; when no key is left, the STORE copies move into the backup too */ + private static function removeKey(string $dir, string $runDir, string $keyCheck): void { + $store = StoreRecord::openRecord(@file_get_contents($dir . 'STORE'), 'store'); + if($store === null){ + return; // removed by an earlier attempt + } + $keys = array_values(array_diff($store['keys'], [$keyCheck])); + if($keys === []){ + foreach(['STORE', 'STORE.2'] as $name){ + if(is_file($dir . $name) && !@rename($dir . $name, "$runDir/v4/$name")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $name into the backup; run --restore again"); + } + } + return; + } + if($keys !== $store['keys']){ + $bytes = StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => $store['storageFormat'], + 'storeId' => $store['storeId'], 'seq' => $store['seq'] + 1, 'keys' => $keys]); + Files::writeAtomic($dir . 'STORE.2', $bytes); + Files::writeAtomic($dir . 'STORE', $bytes); + } + } +} diff --git a/migrate/Stop.php b/migrate/Stop.php new file mode 100644 index 0000000..6c098e6 --- /dev/null +++ b/migrate/Stop.php @@ -0,0 +1,20 @@ + 2, 'UNSUPPORTED_SOURCE' => 2, 'MAINTENANCE_BUSY' => 3, + 'MAINTENANCE_IN_PROGRESS' => 4, 'FAILED' => 5]; + + /** @var string one of the keys of STATUS */ + public $result; + + public function __construct(string $result, string $message, ?\Throwable $previous = null){ + parent::__construct($message, 0, $previous); + $this->result = $result; + } +} diff --git a/migrate/Validator/Verifier.php b/migrate/Validator/Verifier.php new file mode 100644 index 0000000..b334e0d --- /dev/null +++ b/migrate/Validator/Verifier.php @@ -0,0 +1,129 @@ +segmentIds() as $id){ + $batch = []; + foreach($source->lines($id) as $key => $line){ + $batch[$key] = json_decode($line, true); + if(count($batch) === self::BATCH){ + self::compare($db, $source->name, $batch); + self::sample($batch, $source, $samples); + $batch = []; + } + } + self::compare($db, $source->name, $batch); + self::sample($batch, $source, $samples); + } + return $samples; + } + + /** + * checkIntegrity(), the count, the next key, the indexed lookups of the sampled values against the keys records() + * found, every spatial index against a scan of the whole world, and an empty recovery log + * @throws Stop + */ + public static function totals(\noneDB $db, \noneDB $scan, Source $source, array $samples): void { + $name = $source->name; + $integrity = $db->checkIntegrity($name); + if(($integrity['ok'] ?? false) !== true){ + throw new Stop('FAILED', "$name: checkIntegrity() reports a problem: " . json_encode($integrity)); + } + if($db->count($name) !== $source->count){ + throw new Stop('FAILED', "$name: count() is " . $db->count($name) . ", {$source->count} records were migrated"); + } + $next = ($db->getShardInfo($name) ?: [])['nextKey'] ?? null; + if($next !== $source->nextKey){ + throw new Stop('FAILED', "$name: the next key is " . var_export($next, true) . ", expected {$source->nextKey}"); + } + foreach($samples as $field => $byValue){ + foreach($byValue as [$value, $keys]){ + if(self::keys($db->find($name, [$field => $value])) !== $keys){ + throw new Stop('FAILED', "$name: the index of '$field' answers " . json_encode($value) . ' unlike the records'); + } + } + } + foreach(array_keys($source->spatial) as $field){ + if(self::keys($db->withinBBox($name, $field, -180, -90, 180, 90)) !== self::keys($scan->withinBBox($name, $field, -180, -90, 180, 90))){ + throw new Stop('FAILED', "$name: the spatial index of '$field' answers unlike a scan"); + } + } + if($db->getRecoveryLog($name) !== []){ + throw new Stop('FAILED', "$name: the engine logged a recovery event while reading it: " . json_encode($db->getRecoveryLog($name))); + } + } + + /** Each expected record (key => record) must be what find() returns for its key */ + private static function compare(\noneDB $db, string $name, array $expected): void { + if($expected === []){ + return; + } + $found = []; + foreach($db->find($name, ['key' => array_keys($expected)]) as $record){ + $found[$record['key']] = $record; + } + foreach($expected as $key => $record){ + if(!array_key_exists($key, $found) || $found[$key] !== $record){ + throw new Stop('FAILED', "$name: record $key differs: expected " . json_encode($record) . ', found ' + . json_encode($found[$key] ?? null)); + } + } + if(count($found) !== count($expected)){ + throw new Stop('FAILED', "$name: find() returns keys that were not migrated"); + } + } + + /** + * field => [serialized value => [value, keys holding it (ascending)]]: the first SAMPLES values of each indexed field + * met, and every key that holds one of them + */ + private static function sample(array $records, Source $source, array &$samples): void { + foreach($records as $key => $record){ + foreach($source->fields as $field){ + if(!array_key_exists($field, $record) || (!is_scalar($record[$field]) && $record[$field] !== null)){ + continue; + } + $id = serialize($record[$field]); + if(isset($samples[$field][$id])){ + $samples[$field][$id][1][] = $key; + } elseif(count($samples[$field] ?? []) < self::SAMPLES){ + $samples[$field][$id] = [$record[$field], [$key]]; + } + } + } + } + + private static function keys(array $records): array { + $keys = array_column($records, 'key'); + sort($keys); + return $keys; + } +} diff --git a/migrate/footer.php b/migrate/footer.php new file mode 100644 index 0000000..00a955b --- /dev/null +++ b/migrate/footer.php @@ -0,0 +1,10 @@ + 'header.php', + 'files' => [ + 'Stop.php', + 'Files.php', + 'Readers/Discovery.php', + 'Readers/Source.php', + 'Converter/Writer.php', + 'Validator/Verifier.php', + 'Backup/Backup.php', + 'Journal/Journal.php', + 'Publisher/Migration.php', + 'Restore/Restore.php', + 'Cli.php', + ], + 'footer' => 'footer.php', +]; diff --git a/noneDB-migrate.php b/noneDB-migrate.php new file mode 100644 index 0000000..4176227 --- /dev/null +++ b/noneDB-migrate.php @@ -0,0 +1,2251 @@ + 2, 'UNSUPPORTED_SOURCE' => 2, 'MAINTENANCE_BUSY' => 3, + 'MAINTENANCE_IN_PROGRESS' => 4, 'FAILED' => 5]; + + /** @var string one of the keys of STATUS */ + public $result; + + public function __construct(string $result, string $message, ?\Throwable $previous = null){ + parent::__construct($message, 0, $previous); + $this->result = $result; + } +} +} +namespace noneDB\Migrate { + +/** + * @internal + */ +final class Files { + /** The permissions of what the tool writes into a store: the engine's defaults (STORAGE_FORMAT.md §3) */ + public const FILE_MODE = 0640; + public const DIR_MODE = 0750; + + /** + * Write a file atomically: a temp file in the same directory, synced, renamed over the target, the directory + * synced; $mode set on it whatever the umask + * @throws Stop + */ + public static function writeAtomic(string $path, string $bytes, int $mode = self::FILE_MODE): void { + self::removeStaleTemps($path); + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + $fp = @fopen($tmp, 'xb'); + if($fp === false){ + throw new Stop('FAILED', "cannot create $tmp"); + } + try { + if(fwrite($fp, $bytes) !== strlen($bytes) || !fflush($fp) || !self::syncHandle($fp)){ + throw new Stop('FAILED', "cannot write $tmp"); + } + } catch(Stop $e){ + fclose($fp); + @unlink($tmp); + throw $e; + } + fclose($fp); + @chmod($tmp, $mode); + if(!@rename($tmp, $path)){ + @unlink($tmp); + throw new Stop('FAILED', "cannot rename $tmp to $path"); + } + self::syncDir(dirname($path)); + } + + /** + * Remove the temps a crashed replacement of $path left ({name}.tmp{8 hex}, STORAGE_FORMAT.md §2: never read); the tool + * writes only while it holds the store's locks, so no other writer is in the middle of one + */ + public static function removeStaleTemps(string $path): void { + $pattern = '/^' . preg_quote(basename($path), '/') . '\.tmp[0-9a-f]{8}$/'; + foreach(@scandir(dirname($path)) ?: [] as $entry){ + if(preg_match($pattern, $entry) === 1){ + @unlink(dirname($path) . '/' . $entry); + } + } + } + + + /** + * Create a file that must not exist yet and write it (synced), for files written once; $mode set on it + * @throws Stop + */ + public static function writeNew(string $path, string $bytes, int $mode = self::FILE_MODE): void { + $fp = @fopen($path, 'xb'); + if($fp !== false){ + @chmod($path, $mode); + } + if($fp === false || fwrite($fp, $bytes) !== strlen($bytes) || !fflush($fp) || !self::syncHandle($fp)){ + if($fp !== false){ + fclose($fp); + } + throw new Stop('FAILED', "cannot write $path"); + } + fclose($fp); + } + + /** + * fsync of a handle: true when done; plain files only + */ + public static function syncHandle($fp): bool { + return @fsync($fp); + } + + /** + * fsync of a directory, so its new, renamed or removed entries survive a power loss (not possible on Windows) + * @throws Stop + */ + public static function syncDir(string $dir): void { + if(PHP_OS_FAMILY === 'Windows'){ + return; + } + $fp = @fopen(rtrim($dir, '/') . '/', 'r'); + $ok = $fp !== false && @fsync($fp); + if($fp !== false){ + fclose($fp); + } + if(!$ok){ + throw new Stop('FAILED', "cannot sync the directory $dir"); + } + } + + /** + * Create a directory and its missing parents, each with $mode whatever the umask + * @throws Stop + */ + public static function makeDir(string $dir, int $mode = self::DIR_MODE): void { + $missing = []; + for($at = rtrim($dir, '/'); $at !== '' && $at !== '.' && !is_dir($at); $at = dirname($at)){ + $missing[] = $at; + } + foreach(array_reverse($missing) as $new){ + if(!@mkdir($new, $mode) && !is_dir($new)){ + throw new Stop('FAILED', "cannot create the directory $new"); + } + @chmod($new, $mode); + } + } + + /** + * Open a lock file, created if missing (with FILE_MODE) + * @return resource|false + */ + public static function openLock(string $path) { + $created = !is_file($path); + $fp = @fopen($path, 'c'); + if($fp !== false && $created){ + @chmod($path, self::FILE_MODE); + } + return $fp; + } + + /** + * Copy a file, synced, and return the SHA-256 of what was copied. $mode: set on the copy (else the umask decides) + * @throws Stop + */ + public static function copy(string $from, string $to, ?int $mode = null): string { + $in = @fopen($from, 'rb'); + $out = $in === false ? false : @fopen($to, 'xb'); + if($in === false || $out === false){ + if($in !== false){ + fclose($in); + } + throw new Stop('FAILED', "cannot copy $from to $to"); + } + $hash = hash_init('sha256'); + $ok = true; + while($ok && !feof($in)){ + $chunk = fread($in, 1048576); + if($chunk === false){ + $ok = false; + break; + } + hash_update($hash, $chunk); + $ok = fwrite($out, $chunk) === strlen($chunk); + } + $ok = $ok && fflush($out) && self::syncHandle($out); + fclose($in); + fclose($out); + if(!$ok){ + throw new Stop('FAILED', "cannot copy $from to $to"); + } + if($mode !== null){ + @chmod($to, $mode); + } + return hash_final($hash); + } + + /** + * Remove a file or directory tree + * @throws Stop + */ + public static function removeTree(string $path): void { + if(is_link($path) || is_file($path)){ + if(!@unlink($path)){ + throw new Stop('FAILED', "cannot remove $path"); + } + return; + } + if(!is_dir($path)){ + return; + } + foreach(scandir($path) ?: [] as $entry){ + if($entry !== '.' && $entry !== '..'){ + self::removeTree($path . '/' . $entry); + } + } + if(!@rmdir($path)){ + throw new Stop('FAILED', "cannot remove $path"); + } + } +} +} +namespace noneDB\Migrate\Readers { + +/** + * @internal + */ +final class Discovery { + /** Entries of a noneDB 4 store (STORAGE_FORMAT.md §3.4): never sources */ + private const STORE_ENTRY = '/^(STORE|STORE\.2|STORE\.LOCK|STORE(\.2)?\.tmp.*|[0-9a-f]{32}\.ndb4|\.nonedb-maintenance|\.nonedb-retained|\.htaccess|index\.html|\.gitkeep|\.gitignore|\.DS_Store)$/'; + + /** + * The entries of $dir for the source key $key: + * - databases: name => its files (a data file, a meta, or a write buffer holding records); + * - leftovers: name => files of the key that belong to no database; + * - keyFiles: files of the directory that only this key uses (its 3.2+ hash cache); + * - sharedFiles: the 3.0/3.1 hash cache, which no key owns; + * - otherKeys: files of the old formats that belong to another key; + * - foreign: anything else that is neither a source nor part of a store; + * - store: entries of a noneDB 4 store. + * @return array + */ + public static function scan(string $dir, string $key): array { + $out = ['databases' => [], 'leftovers' => [], 'keyFiles' => [], 'sharedFiles' => [], 'otherKeys' => [], 'foreign' => [], 'store' => []]; + $named = []; + $hashOnly = []; + $fingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', $key), 0, 16); + foreach(scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(preg_match(self::STORE_ENTRY, $entry)){ + $out['store'][] = $entry; + } elseif(preg_match('/^([0-9a-f]{20})-([^_.]*)(?:_s\d+)?\.nonedb/', $entry, $m)){ + $named[$m[2]][$m[1]][] = $entry; + } elseif(preg_match('/^([0-9a-f]{20})(?:_s\d+)?\.nonedb\./', $entry, $m)){ + $hashOnly[$m[1]][] = $entry; + } elseif($entry === '.nonedb_hash_cache'){ + $out['sharedFiles'][] = $entry; + } elseif(preg_match('/^\.nonedb_hash_cache_([0-9a-f]{16})$/', $entry, $m)){ + $out[$m[1] === $fingerprint ? 'keyFiles' : 'otherKeys'][] = $entry; + } else { + $out['foreign'][] = $entry; + } + } + $ownHashes = []; + foreach($named as $name => $byHash){ + $hash = self::hashOf((string)$name, $key); + foreach($byHash as $h => $files){ + if($h === $hash){ + $ownHashes[$h] = (string)$name; + $out['databases'][(string)$name] = $files; + } else { + array_push($out['otherKeys'], ...$files); + } + } + } + foreach($hashOnly as $h => $files){ + if(isset($ownHashes[$h])){ + array_push($out['databases'][$ownHashes[$h]], ...$files); + } else { + array_push($out['otherKeys'], ...$files); + } + } + foreach($out['databases'] as $name => $files){ + sort($files); + if(!self::isDatabase($dir, (string)$name, self::hashOf((string)$name, $key), $files)){ + unset($out['databases'][$name]); + $out['leftovers'][$name] = $files; + } else { + $out['databases'][$name] = $files; + } + } + ksort($out['databases'], SORT_STRING); + ksort($out['leftovers'], SORT_STRING); + foreach(['keyFiles', 'sharedFiles', 'otherKeys', 'foreign', 'store'] as $list){ + sort($out[$list]); + } + return $out; + } + + /** The name hash of every release: PBKDF2-SHA256, the name as password, the key as salt, 20 hex digits */ + public static function hashOf(string $name, string $key): string { + return hash_pbkdf2('sha256', $name, $key, 1000, 20); + } + + /** + * A database has a data file (its own or a shard's), a meta, or a write buffer holding a record + */ + private static function isDatabase(string $dir, string $name, string $hash, array $files): bool { + $base = $hash . '-' . $name; + foreach($files as $file){ + if($file === "$base.nonedb" || $file === "$base.nonedb.meta" || preg_match('/^' . preg_quote($base, '/') . '_s\d+\.nonedb$/', $file)){ + return true; + } + if(preg_match('/\.nonedb\.buffer(\.flushing(\.\d+)?)?$/', $file) && trim((string)@file_get_contents($dir . '/' . $file)) !== ''){ + return true; + } + } + return false; + } +} +} +namespace noneDB\Migrate\Readers { + +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Source { + /** @var string */ + public $name; + /** @var string[] its files, names in the directory */ + public $files; + /** @var string READY, AMBIGUOUS_SOURCE or UNSUPPORTED_SOURCE */ + public $status = 'READY'; + /** @var string[] why it is not READY */ + public $reasons = []; + /** @var string[] segment id => kind (W, J30, J32, J33, EMPTY) */ + public $kinds = []; + /** @var string single or sharded */ + public $layout = 'single'; + /** @var int */ + public $shardSize; + /** @var int */ + public $nextKey = 0; + /** @var int live records, buffered ones included */ + public $count = 0; + /** @var int records imported from write buffers */ + public $buffered = 0; + /** @var string[] indexed fields */ + public $fields = []; + /** @var array spatial field => node size (null: the engine's default) */ + public $spatial = []; + /** @var int[] what is not carried over: reason => lines or entries */ + public $notCarried = []; + + /** @var string */ + private $dir; + /** @var string "{hash}-{name}" */ + private $base; + /** @var string "{hash}" */ + private $hash; + /** @var array options: rebuildFromData, flushing (null, import, skip), shardSize */ + private $options; + /** @var bool the lock record has an epoch: 3.3-dev wrote the database */ + private $epoch = false; + /** @var bool a 3.2+ lock file exists */ + private $locked = false; + /** + * @var array segment id => [kind, path, jidx (its index file), slots: next local key, count: live records]. The live + * records themselves are read again per segment when written or verified (content()), so memory stays bounded by + * the largest segment + */ + private $segments = []; + /** @var bool reading again (content()): what was checked and reported is neither checked nor reported twice */ + private $quiet = false; + /** @var array [segment id, its content] of the last segment read again */ + private $content = [null, null]; + /** @var array key => record of buffered records, by segment id */ + private $bufferedRecords = []; + + /** Line encoding of the engine (STORAGE_FORMAT.md §2, §6.1) */ + private const JSON = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRESERVE_ZERO_FRACTION; + + /** + * Read the database $name of $dir from its files (as Discovery grouped them) + * @param array $options rebuildFromData (bool), flushing (null|'import'|'skip'), shardSize (int, single layout) + */ + public static function read(string $dir, string $name, string $key, array $files, array $options): self { + $source = new self(); + $source->dir = rtrim($dir, '/') . '/'; + $source->name = $name; + $source->files = $files; + $source->hash = Discovery::hashOf($name, $key); + $source->base = $source->hash . '-' . $name; + $source->options = $options + ['rebuildFromData' => false, 'flushing' => null, 'shardSize' => 10000]; + try { + $source->readAll(); + } catch(Stop $e){ + $source->status = $e->result; + $source->reasons[] = $e->getMessage(); + } + return $source; + } + + private function readAll(): void { + if(preg_match("/^[A-Za-z0-9' -]+$/D", $this->name) !== 1){ + throw new Stop('UNSUPPORTED_SOURCE', "the name '{$this->name}' is not a valid noneDB 4 database name"); + } + $lock = $this->path('.nonedb.lock'); + if(is_file($lock)){ + $this->locked = true; + $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', (string)file_get_contents($lock, false, null, 0, 128)) === 1; + } + if(is_file($this->path('.nonedb.meta'))){ + $this->readSharded(); + } elseif(is_file($this->path('.nonedb'))){ + $this->shardSize = (int)$this->options['shardSize']; + $this->summarize('main', $this->path('.nonedb'), $this->path('.nonedb.jidx')); + } elseif(preg_grep('/^' . preg_quote($this->base, '/') . '_s\d+\.nonedb$/', $this->files)){ + throw new Stop('AMBIGUOUS_SOURCE', 'shard files without a meta: an interrupted drop or a lost meta'); + } else { + $this->shardSize = (int)$this->options['shardSize']; + $this->segments['main'] = ['kind' => 'EMPTY', 'path' => null, 'jidx' => null, 'slots' => 0, 'count' => 0]; + } + foreach($this->segments as $id => $segment){ + $this->kinds[$id] = $segment['kind']; + $this->count += $segment['count']; + } + $this->readBuffers(); + $this->readIndexList(); + $this->count += $this->buffered; + } + + /** Read a segment once to check it and count it (§4.1, §4.2); its records are not kept */ + private function summarize($id, string $path, string $jidxPath): void { + $segment = $this->readSegment($id, $path, $jidxPath); + $this->segments[$id] = ['kind' => $segment['kind'], 'path' => $path, 'jidx' => $jidxPath, 'slots' => $segment['slots'], + 'count' => count($segment['live'] ?? $segment['records'])]; + } + + /** + * A segment's live records, read again: key => [offset, length] of a JSONL line, or key => record (W); [] for a + * segment that holds none + */ + private function content($id): array { + if($this->content[0] !== $id){ + $segment = $this->segments[$id] ?? null; + $this->quiet = true; + try { + $read = $segment === null || $segment['kind'] === 'EMPTY' ? [] : $this->readSegment($id, $segment['path'], $segment['jidx'], $segment['kind']); + } finally { + $this->quiet = false; + } + $this->content = [$id, $read['live'] ?? $read['records'] ?? []]; + } + return $this->content[1]; + } + + // ------------------------------------------------------------------ layout + + private function readSharded(): void { + $this->layout = 'sharded'; + $meta = json_decode((string)@file_get_contents($this->path('.nonedb.meta')), true); + if(!is_array($meta) || !is_array($meta['shards'] ?? null) || !is_int($meta['shardSize'] ?? null) || $meta['shardSize'] < 1){ + throw new Stop('AMBIGUOUS_SOURCE', 'the meta is unreadable'); + } + if(($meta['version'] ?? null) !== 1){ + throw new Stop('UNSUPPORTED_SOURCE', 'the meta has a version no release wrote'); + } + $this->shardSize = $meta['shardSize']; + if(isset($meta['nextKey']) && is_int($meta['nextKey'])){ + $this->raiseNextKey($meta['nextKey']); + } + foreach($meta['shards'] as $shard){ + $id = $shard['id'] ?? null; + if(!is_int($id) || $id < 0 || isset($this->segments[$id])){ + throw new Stop('AMBIGUOUS_SOURCE', 'the meta lists a shard id that is not a distinct non-negative integer'); + } + $this->summarize($id, $this->path("_s$id.nonedb"), $this->path("_s$id.nonedb.jidx")); + } + } + + // ------------------------------------------------------------------ segments + + /** + * A segment's live set by its kind (§4.1); shard keys checked against the shard size (§4.2). $decided: the kind a + * first read decided (read again: the same rule, without deciding anew) + */ + private function readSegment($id, string $path, string $jidxPath, ?string $decided = null): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + if(!is_file($path) || filesize($path) === 0){ + $segment = ['kind' => 'EMPTY', 'path' => null, 'live' => [], 'slots' => 0]; + $index = is_file($jidxPath) ? json_decode((string)file_get_contents($jidxPath), true) : null; + if(is_array($index) && is_int($index['n'] ?? null)){ + $segment['slots'] = $index['n']; + $this->raiseNextKey($base + $index['n']); + } + return $segment; + } + $whole = $decided === null || $decided === 'W' ? $this->wholeJson($path) : null; + if($whole !== null){ + return $this->readWhole($id, $whole); + } + $index = is_file($jidxPath) ? json_decode((string)file_get_contents($jidxPath), true) : null; + if($decided !== null){ + $kind = $decided; + } elseif(is_array($index) && is_array($index['p'] ?? null) && is_array($index['l'] ?? null) && isset($index['s'])){ + $kind = 'J33'; + } elseif(is_array($index) && is_array($index['o'] ?? null) && ($index['v'] ?? null) === 3){ + $kind = isset($index['s']) ? 'J32' : 'J30'; + } elseif($this->epoch){ + $kind = 'J33'; // 3.3-dev: the data file is the database, its checkpoints (written now and then) only a shortcut + } elseif($this->options['rebuildFromData']){ + $kind = 'J33'; + $this->noteNotCarried('index replaced by the data file (--rebuild-from-data)', 1); + } else { + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ': ' . (is_file($jidxPath) ? 'its key index is unreadable' : 'its key index is missing') + . ' (--rebuild-from-data replays the data file; records an older release deleted may come back)'); + } + if($kind === 'J33'){ + $segment = $this->replay($id, $path); + } else { + try { + $segment = $this->indexed($id, $path, $index, $kind); + } catch(Stop $e){ + if(!$this->options['rebuildFromData']){ + throw $e; + } + $kind = 'J33'; // an index that doesn't fit its data, replaced by a replay on request (§6.2) + $this->noteNotCarried('index replaced by the data file (--rebuild-from-data)', 1); + $segment = $this->replay($id, $path); + } + } + if(is_array($index) && is_int($index['n'] ?? null)){ + // The index's counter: the next key of a single segment, the next local key of a shard + $this->raiseNextKey($base + $index['n']); + $segment['slots'] = max($segment['slots'], $index['n']); + } + if($id !== 'main' && $kind !== 'J33'){ + foreach($segment['live'] as $k => $where){ + if(intdiv($k, $this->shardSize) !== $id){ + throw new Stop('AMBIGUOUS_SOURCE', "shard $id holds key $k, outside its range for the meta's shard size {$this->shardSize}"); + } + } + } + $segment['kind'] = $kind; + return $segment; + } + + /** The decoded file when it is one JSON object whose only member is "data", an array; else null */ + private function wholeJson(string $path): ?array { + $fp = fopen($path, 'rb'); + $head = (string)fread($fp, 16); + fclose($fp); + if(preg_match('/^\s*\{\s*"data"\s*:/', $head) !== 1){ + return null; + } + $json = json_decode((string)file_get_contents($path), true); + return is_array($json) && array_keys($json) === ['data'] && is_array($json['data']) ? $json['data'] : null; + } + + /** W: every array entry is a record keyed by its position; other entries only count for the next key */ + private function readWhole($id, array $data): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + $records = []; + $slots = 0; + foreach($data as $i => $entry){ + if(!is_int($i) || $i < 0){ + throw new Stop('AMBIGUOUS_SOURCE', "segment $id: its data has a member that is not a position"); + } + if($id !== 'main' && $i >= $this->shardSize){ + throw new Stop('AMBIGUOUS_SOURCE', "shard $id holds more entries than the meta's shard size {$this->shardSize}"); + } + $slots = max($slots, $i + 1); + if(is_array($entry)){ + $records[$base + $i] = $entry; + } elseif($entry !== null){ + $this->noteNotCarried('an entry that is not a record', 1); + } + } + ksort($records); + $this->raiseNextKey($base + $slots); + return ['kind' => 'W', 'path' => null, 'records' => $records, 'slots' => $slots]; + } + + /** + * J30, J32: the index's map is the live set. Every entry must point at a line that decodes to its key; an entry of + * length 0 is not live; J32 lines after s are not live. + */ + private function indexed($id, string $path, array $index, string $kind): array { + $size = filesize($path); + $fp = fopen($path, 'rb'); + $end = $size; + if($kind === 'J32'){ + $s = $index['s']; + if(!is_int($s) || $s < 0 || $s > $size || ($s > 0 && $this->byteAt($fp, $s - 1) !== "\n")){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ': its key index covers bytes the data file does not have' + . ' (--rebuild-from-data replays the data file)'); + } + $end = $s; + } + $live = []; + foreach($index['o'] as $k => $where){ + $k = (int)$k; + if(!is_array($where) || count($where) !== 2 || !is_int($where[0]) || !is_int($where[1]) || $where[0] < 0 || $where[1] < 0){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ": the key index entry of key $k is malformed"); + } + if($where[1] === 0){ + $this->noteNotCarried('a key index entry of length 0', 1); + continue; + } + if($this->quiet){ + $live[$k] = $where; // read again: checked when first read, and the files are unchanged (fingerprints) + continue; + } + fseek($fp, $where[0]); + $line = (string)fread($fp, $where[1]); + $record = strlen($line) === $where[1] && $where[0] + $where[1] <= $end ? json_decode($line, true) : null; + if(!is_array($record) || ($record['key'] ?? null) !== $k){ + fclose($fp); + throw new Stop('AMBIGUOUS_SOURCE', basename($path) . ": the key index points key $k at bytes that are not its record" + . ' (--rebuild-from-data replays the data file)'); + } + $live[$k] = $where; + } + fclose($fp); + ksort($live); + if($this->quiet){ + return ['path' => $path, 'live' => $live, 'slots' => 0]; + } + $scan = $this->scanLines($path, $kind === 'J32' ? $end : null); + $this->noteNotCarried('lines that are not live (deleted or superseded records, tombstones)', $scan['before'] - count($live)); + if($scan['after'] > 0){ + $this->noteNotCarried('complete lines after the key index (3.2 never showed them)', $scan['after']); + } + return ['path' => $path, 'live' => $live, 'slots' => 0]; + } + + /** + * J33: the data file is the database. The last line of a key wins, a tombstone removes it; a line that does + * not decode is salvaged (STORAGE_FORMAT.md §6.1); shard lines outside the shard's range are not live. + */ + private function replay($id, string $path): array { + $base = $id === 'main' ? 0 : $id * $this->shardSize; + $live = []; + $slots = 0; + $fp = fopen($path, 'rb'); + $offset = 0; + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + if($line[$length - 1] !== "\n"){ + $this->noteNotCarried('an unterminated last line', 1); + break; + } + $line = substr($line, 0, -1); + [$type, $k, $at] = $this->classify($line); + if($type !== null){ + $this->raiseNextKey($k + 1); + if($id !== 'main' && intdiv($k, $this->shardSize) !== $id){ + $type = null; + } else { + $slots = max($slots, $k - $base + 1); + } + } + if($type !== null && $at > 0){ + $this->noteNotCarried('bytes glued before a record in its line (the record is salvaged)', 1); + } + if($type === 'record'){ + $live[$k] = [$offset + $at, $length - 1 - $at]; + } elseif($type === 'tombstone'){ + unset($live[$k]); + } else { + $this->noteNotCarried('lines that are neither a record nor a tombstone', 1); + } + $offset += $length; + } + fclose($fp); + ksort($live); + return ['path' => $path, 'live' => $live, 'slots' => $slots]; + } + + /** + * [record|tombstone|null, key, offset of the JSON in the line]: a whole line, else the salvage rule: from right + * to left, the first "{" from which a record or tombstone decodes to the line end + */ + private function classify(string $line): array { + $value = json_decode($line, true); + if(is_array($value)){ + return $this->kindOf($value, 0); + } + for($at = strrpos($line, '{'), $tries = 0; $at !== false && $at > 0 && $tries < 4096; $tries++){ + $value = json_decode(substr($line, $at), true); + if(is_array($value) && ($kind = $this->kindOf($value, $at))[0] !== null){ + return $kind; + } + $at = strrpos($line, '{', $at - strlen($line) - 1); + } + return [null, 0, 0]; + } + + private function kindOf(array $value, int $at): array { + if(count($value) === 1 && is_int($value['__nonedb_deleted'] ?? null) && $value['__nonedb_deleted'] >= 0){ + return ['tombstone', $value['__nonedb_deleted'], $at]; + } + if(is_int($value['key'] ?? null) && $value['key'] >= 0){ + return ['record', $value['key'], $at]; + } + return [null, 0, 0]; + } + + /** + * Every key in a JSONL data file raises the next key (records and tombstones, live or not). Returns the number + * of complete lines before $end (the bytes a 3.2 index covers; null: all) and after it. + */ + private function scanLines(string $path, ?int $end): array { + $fp = fopen($path, 'rb'); + $counts = ['before' => 0, 'after' => 0]; + $offset = 0; + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + if($line[$length - 1] !== "\n"){ + break; + } + [$type, $k] = $this->classify(substr($line, 0, -1)); + if($type !== null){ + $this->raiseNextKey($k + 1); + } + $counts[$end !== null && $offset >= $end ? 'after' : 'before']++; + $offset += $length; + } + fclose($fp); + return $counts; + } + + private function byteAt($fp, int $offset): string { + fseek($fp, $offset); + return (string)fread($fp, 1); + } + + // ------------------------------------------------------------------ buffers (§4.3) + + private function readBuffers(): void { + $current = []; + $flushingK = []; + $plain = []; + foreach($this->files as $file){ + if(!preg_match('/^' . preg_quote($this->base, '/') . '(?:_s(\d+))?\.nonedb\.buffer(\.flushing(?:\.(\d+))?)?$/', $file, $m)){ + continue; + } + $records = $this->bufferRecords($this->dir . $file); + if($records === []){ + continue; + } + $shard = ($m[1] ?? '') === '' ? null : (int)$m[1]; + if(($m[2] ?? '') === ''){ + $current[$shard ?? 'main'] = $records; + } elseif(($m[3] ?? '') === ''){ + $plain[] = [$file, $records]; + } else { + $flushingK[(int)$m[3]] = [$file, $records]; + } + } + // 1. shard buffers of 2.3-3.1: after the shard's slots + foreach($current as $shard => $records){ + if($shard === 'main'){ + continue; + } + if(!isset($this->segments[$shard])){ + throw new Stop('AMBIGUOUS_SOURCE', "a write buffer of shard $shard, which the meta does not list (no release flushed it)"); + } + $next = $shard * $this->shardSize + $this->segments[$shard]['slots']; + foreach($records as $record){ + $this->addBuffered($next++, $record); + } + } + // 2. interrupted flushes of 3.2/3.3-dev: from K on, skipping keys already stored + ksort($flushingK); + foreach($flushingK as $k => [$file, $records]){ + foreach($records as $record){ + if(!$this->isLive($k)){ + $this->addBuffered($k, $record); + } + $k++; + } + } + // 3. the current buffer: from the next key + if(isset($current['main'])){ + if($this->layout === 'sharded' && !$this->locked){ + throw new Stop('AMBIGUOUS_SOURCE', 'a write buffer next to a meta, which releases before 3.2 never flushed'); + } + $next = $this->nextKey; + foreach($current['main'] as $record){ + $this->addBuffered($next++, $record); + } + } + // 4. interrupted flushes of 2.3-3.1: only on request, fresh keys after everything else + if($plain !== []){ + if($this->options['flushing'] === null){ + throw new Stop('AMBIGUOUS_SOURCE', 'an interrupted flush of a release before 3.2 (' . implode(', ', array_column($plain, 0)) + . '): its records may or may not be stored already (--flushing=import or --flushing=skip)'); + } + foreach($plain as [$file, $records]){ + if($this->options['flushing'] === 'skip'){ + $this->noteNotCarried('records of an interrupted flush (--flushing=skip)', count($records)); + continue; + } + $next = $this->nextKey; + foreach($records as $record){ + $this->addBuffered($next++, $record); + } + } + } + } + + /** The records of a buffer file: lines that decode to an array (blank lines ignored, others reported) */ + private function bufferRecords(string $path): array { + $records = []; + foreach(explode("\n", (string)file_get_contents($path)) as $line){ + if(trim($line) === ''){ + continue; + } + $record = json_decode($line, true); + if(is_array($record)){ + $records[] = $record; + } else { + $this->noteNotCarried('buffer lines that are not a record', 1); + } + } + return $records; + } + + private function addBuffered(int $key, array $record): void { + unset($record['key']); + $id = $this->layout === 'single' ? 'main' : intdiv($key, $this->shardSize); + $this->bufferedRecords[$id][$key] = $record; + $this->buffered++; + $this->raiseNextKey($key + 1); + } + + private function isLive(int $key): bool { + $id = $this->layout === 'single' ? 'main' : intdiv($key, $this->shardSize); + return isset($this->bufferedRecords[$id][$key]) || isset($this->content($id)[$key]); + } + + // ------------------------------------------------------------------ indexes + + /** Indexed fields and spatial fields: the release's index list, else the headers of its index files */ + private function readIndexList(): void { + $list = json_decode((string)@file_get_contents($this->path('.nonedb.indexes')), true); + if(is_array($list) && is_array($list['fields'] ?? null) && is_array($list['spatial'] ?? null)){ + $this->fields = array_values(array_filter($list['fields'], 'is_string')); + foreach($list['spatial'] as $field){ + if(is_string($field)){ + $size = $list['spatialOptions'][$field]['nodeSize'] ?? null; + $this->spatial[$field] = is_int($size) ? $size : null; + } + } + } else { + foreach($this->files as $file){ + if(preg_match('/\.nonedb\.(fidx|gfidx|fx|gfx)\./', $file)){ + $field = $this->headerField($this->dir . $file); + if($field !== null && !in_array($field, $this->fields, true)){ + $this->fields[] = $field; + } + } elseif(preg_match('/\.nonedb\.(sidx|gsidx|rx|grx)\./', $file)){ + $header = $this->header($this->dir . $file); + if(is_string($header['field'] ?? null) && !array_key_exists($header['field'], $this->spatial)){ + $size = $header['nodeSize'] ?? $header['ns'] ?? null; + $this->spatial[$header['field']] = is_int($size) ? $size : null; + } + } + } + } + sort($this->fields, SORT_STRING); + ksort($this->spatial, SORT_STRING); + } + + private function headerField(string $path): ?string { + $header = $this->header($path); + return is_string($header['field'] ?? null) ? $header['field'] : null; + } + + /** An index file's header: its first line as JSON (3.1 files are JSON encoded twice) */ + private function header(string $path): array { + $fp = @fopen($path, 'rb'); + $line = $fp === false ? false : fgets($fp); + if($fp !== false){ + fclose($fp); + } + $value = $line === false ? null : json_decode($line, true); + if(is_string($value)){ + $value = json_decode($value, true); + } + return is_array($value) ? $value : []; + } + + // ------------------------------------------------------------------ output + + /** + * The record lines of a segment in key order, without "\n": a JSONL line as stored when it ends with its key, + * else the record encoded as the engine encodes one + * @return \Generator key => line + */ + public function lines($id): \Generator { + $content = $this->content($id); + $kind = $this->segments[$id]['kind'] ?? 'EMPTY'; + $buffered = $this->bufferedRecords[$id] ?? []; + $keys = array_merge(array_keys($content), array_keys($buffered)); + sort($keys); + $fp = $kind !== 'W' && $content !== [] ? fopen($this->segments[$id]['path'], 'rb') : null; + try { + foreach($keys as $k){ + if(isset($buffered[$k])){ + yield $k => self::encode($buffered[$k], $k); + } elseif($fp !== null){ + [$offset, $length] = $content[$k]; + fseek($fp, $offset); + $line = (string)fread($fp, $length); + yield $k => substr($line, -strlen(",\"key\":$k}")) === ",\"key\":$k}" || $line === "{\"key\":$k}" + ? $line : self::encode(json_decode($line, true), $k); + } else { + yield $k => self::encode($content[$k], $k); + } + } + } finally { + if($fp !== null){ + fclose($fp); + } + } + } + + /** Segment ids that hold records (sharded: shard ids; single: main) */ + public function segmentIds(): array { + $ids = []; + foreach($this->segments as $id => $segment){ + if($segment['count'] > 0){ + $ids[$id] = true; + } + } + foreach($this->bufferedRecords as $id => $records){ + $ids[$id] = true; + } + $ids = array_keys($ids); + sort($ids); + return $ids; + } + + /** A record as a data line: its fields, then "key" (STORAGE_FORMAT.md §6.1) */ + public static function encode(array $record, int $key): string { + unset($record['key']); + $record['key'] = $key; + try { + return json_encode($record, self::JSON | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + throw new Stop('FAILED', "record $key cannot be encoded: " . $e->getMessage()); + } + } + + private function path(string $suffix): string { + return $this->dir . $this->base . $suffix; + } + + private function raiseNextKey(int $key): void { + if($key > $this->nextKey){ + $this->nextKey = $key; + } + } + + private function noteNotCarried(string $reason, int $count): void { + if($count > 0 && !$this->quiet){ + $this->notCarried[$reason] = ($this->notCarried[$reason] ?? 0) + $count; + } + } +} +} +namespace noneDB\Migrate\Converter { + +use noneDB\Internal\Storage\KeyIndex; +use noneDB\Internal\Storage\StoreRecord; +use noneDB\Migrate\Files; +use noneDB\Migrate\Readers\Source; +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Writer { + /** + * Write $source into the store directory $storeDir under $key: the state record, generation 1 (segments with + * their key-index checkpoints, shards.json for the sharded layout, metadata), CURRENT, and FORMAT last + * (STORAGE_FORMAT.md §8.1). The segments hold the live records in key order; the segment of the highest key ever + * used ends with its tombstone when that key is not live (§6.2). The checkpoints are written as compaction writes + * them (§6.2 step 4, §6.4), so the engine never replays a whole migrated segment. Everything is synced. Returns the + * database's directory. + */ + public static function write(string $storeDir, string $key, Source $source): string { + $home = rtrim($storeDir, '/') . '/' . StoreRecord::locatorOf($source->name, $key) . '.ndb4/'; + $generation = $home . 'generations/1/'; + Files::makeDir($generation . 'segments'); + Files::writeNew($home . 'LOCK', str_pad('1:' . bin2hex(random_bytes(4)), 128)); + $sharded = $source->layout === 'sharded'; + $last = $source->nextKey - 1; + $lastSegment = $last < 0 ? null : ($sharded ? intdiv($last, $source->shardSize) : 'main'); + $ids = $source->segmentIds(); + if(!$sharded && !in_array('main', $ids, true)){ + $ids[] = 'main'; // a single-layout generation always has its main segment + } + if($lastSegment !== null && !in_array($lastSegment, $ids, true)){ + $ids[] = $lastSegment; + sort($ids); + } + $shards = []; + foreach($ids as $id){ + $path = $generation . 'segments/' . ($id === 'main' ? 'main' : 's' . $id) . '.jsonl'; + $fp = @fopen($path, 'xb'); + if($fp === false){ + throw new Stop('FAILED', "cannot create $path"); + } + @chmod($path, Files::FILE_MODE); + $index = ['kb' => '', 'p' => [], 'l' => []]; // key => offset, key => length of its line + $offset = 0; + $tail = ''; + $max = -1; + $chunk = ''; + foreach($source->lines($id) as $k => $line){ + $index['p'][$k] = $offset; + $index['l'][$k] = strlen($line); + $offset += strlen($line) + 1; + $chunk .= $line . "\n"; + $max = $k; + if(strlen($chunk) >= 1048576){ + self::put($fp, $chunk, $path); + $tail = substr($chunk, -256); + $chunk = ''; + } + } + $tombstones = 0; + if($id === $lastSegment && $max < $last){ + $chunk .= '{"__nonedb_deleted":' . $last . "}\n"; + $offset += strlen('{"__nonedb_deleted":' . $last . "}\n"); + $max = $last; + $tombstones = 1; + } + self::put($fp, $chunk, $path); + if(!fflush($fp) || !Files::syncHandle($fp)){ + fclose($fp); + throw new Stop('FAILED', "cannot sync $path"); + } + fclose($fp); + $tail = substr($tail . $chunk, -256); + $base = $id === 'main' ? 0 : $id * $source->shardSize; + [$layout, $slots, $body, , $live] = KeyIndex::encodeBody($index, $base); + $header = ['kind' => 'kix', 'v' => 1, 'layout' => $layout, 'base' => $base, 'slots' => $slots, 'live' => $live, + 'n' => $max < $base ? 0 : $max - $base + 1, 'd' => 0, 't' => $tombstones, 's' => $offset, + 'h' => sprintf('%08x', crc32($tail)), 'e' => bin2hex(random_bytes(4)), 'created' => time(), 'updated' => time()]; + if($id !== 'main'){ + $header['z'] = $source->shardSize; + $header['sid'] = $id; + } + Files::writeNew(substr($path, 0, -strlen('.jsonl')) . '.kix', KeyIndex::encodeFile($header, $body)); + if($sharded){ + $shards[] = ['id' => $id, 'count' => count($index['p']), 'deleted' => 0]; + } + } + Files::syncDir($generation . 'segments'); + $identity = ['storageFormat' => \noneDB::STORAGE_FORMAT, 'databaseId' => StoreRecord::uuid4(), 'name' => $source->name, + 'shardSize' => $source->shardSize, 'createdAt' => time(), 'formatOptions' => new \stdClass()]; + if($sharded){ + Files::writeNew($generation . 'shards.json', json_encode(['kind' => 'shards', 'v' => 1, 'shardSize' => $source->shardSize, + 'nextKey' => $source->nextKey, 'totalRecords' => array_sum(array_column($shards, 'count')), 'deletedCount' => 0, + 'shards' => $shards], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "\n"); + } + Files::writeNew($generation . 'metadata', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'generation'] + $identity + + ['generation' => 1, 'layout' => $source->layout])); + Files::syncDir($generation); + Files::syncDir($home . 'generations'); + Files::writeNew($home . 'CURRENT', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'current', + 'databaseId' => $identity['databaseId'], 'generation' => 1])); + Files::writeNew($home . 'FORMAT', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'database'] + $identity)); + Files::syncDir($home); + Files::syncDir($storeDir); + return $home; + } + + private static function put($fp, string $bytes, string $path): void { + if($bytes !== '' && fwrite($fp, $bytes) !== strlen($bytes)){ + fclose($fp); + throw new Stop('FAILED', "cannot write $path"); + } + } +} +} +namespace noneDB\Migrate\Validator { + +use noneDB\Migrate\Readers\Source; +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Verifier { + /** Keys read per find() */ + private const BATCH = 1000; + /** Values of each indexed field compared with a scan */ + private const SAMPLES = 50; + + /** + * records(), then totals(); $scan reads without indexes + * @throws Stop + */ + public static function verify(\noneDB $db, \noneDB $scan, Source $source): void { + self::totals($db, $scan, $source, self::records($db, $source)); + } + + /** + * Every record (===), read in batches of keys: one operation per batch, so the engine reads one segment at a time + * (and frees memory between operations). On the way, the keys that hold each of up to SAMPLES values of every + * indexed field: what a scan answers for them, for totals(). + * @throws Stop + */ + public static function records(\noneDB $db, Source $source): array { + $samples = []; + foreach($source->segmentIds() as $id){ + $batch = []; + foreach($source->lines($id) as $key => $line){ + $batch[$key] = json_decode($line, true); + if(count($batch) === self::BATCH){ + self::compare($db, $source->name, $batch); + self::sample($batch, $source, $samples); + $batch = []; + } + } + self::compare($db, $source->name, $batch); + self::sample($batch, $source, $samples); + } + return $samples; + } + + /** + * checkIntegrity(), the count, the next key, the indexed lookups of the sampled values against the keys records() + * found, every spatial index against a scan of the whole world, and an empty recovery log + * @throws Stop + */ + public static function totals(\noneDB $db, \noneDB $scan, Source $source, array $samples): void { + $name = $source->name; + $integrity = $db->checkIntegrity($name); + if(($integrity['ok'] ?? false) !== true){ + throw new Stop('FAILED', "$name: checkIntegrity() reports a problem: " . json_encode($integrity)); + } + if($db->count($name) !== $source->count){ + throw new Stop('FAILED', "$name: count() is " . $db->count($name) . ", {$source->count} records were migrated"); + } + $next = ($db->getShardInfo($name) ?: [])['nextKey'] ?? null; + if($next !== $source->nextKey){ + throw new Stop('FAILED', "$name: the next key is " . var_export($next, true) . ", expected {$source->nextKey}"); + } + foreach($samples as $field => $byValue){ + foreach($byValue as [$value, $keys]){ + if(self::keys($db->find($name, [$field => $value])) !== $keys){ + throw new Stop('FAILED', "$name: the index of '$field' answers " . json_encode($value) . ' unlike the records'); + } + } + } + foreach(array_keys($source->spatial) as $field){ + if(self::keys($db->withinBBox($name, $field, -180, -90, 180, 90)) !== self::keys($scan->withinBBox($name, $field, -180, -90, 180, 90))){ + throw new Stop('FAILED', "$name: the spatial index of '$field' answers unlike a scan"); + } + } + if($db->getRecoveryLog($name) !== []){ + throw new Stop('FAILED', "$name: the engine logged a recovery event while reading it: " . json_encode($db->getRecoveryLog($name))); + } + } + + /** Each expected record (key => record) must be what find() returns for its key */ + private static function compare(\noneDB $db, string $name, array $expected): void { + if($expected === []){ + return; + } + $found = []; + foreach($db->find($name, ['key' => array_keys($expected)]) as $record){ + $found[$record['key']] = $record; + } + foreach($expected as $key => $record){ + if(!array_key_exists($key, $found) || $found[$key] !== $record){ + throw new Stop('FAILED', "$name: record $key differs: expected " . json_encode($record) . ', found ' + . json_encode($found[$key] ?? null)); + } + } + if(count($found) !== count($expected)){ + throw new Stop('FAILED', "$name: find() returns keys that were not migrated"); + } + } + + /** + * field => [serialized value => [value, keys holding it (ascending)]]: the first SAMPLES values of each indexed field + * met, and every key that holds one of them + */ + private static function sample(array $records, Source $source, array &$samples): void { + foreach($records as $key => $record){ + foreach($source->fields as $field){ + if(!array_key_exists($field, $record) || (!is_scalar($record[$field]) && $record[$field] !== null)){ + continue; + } + $id = serialize($record[$field]); + if(isset($samples[$field][$id])){ + $samples[$field][$id][1][] = $key; + } elseif(count($samples[$field] ?? []) < self::SAMPLES){ + $samples[$field][$id] = [$record[$field], [$key]]; + } + } + } + } + + private static function keys(array $records): array { + $keys = array_column($records, 'key'); + sort($keys); + return $keys; + } +} +} +namespace noneDB\Migrate\Backup { + +use noneDB\Internal\Storage\StoreRecord; +use noneDB\Migrate\Files; +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Backup { + /** + * Copy $files (names in $dir) into {$runDir}/files/ (directories 0700, files 0600), write manifest.json last (a + * sealed record: $info plus every file's name, size, mtime and SHA-256), then read the copy back and compare + * @return array the manifest + * @throws Stop + */ + public static function create(string $dir, string $runDir, array $files, array $info): array { + Files::makeDir($runDir . '/files', 0700); + @chmod($runDir, 0700); + $entries = []; + foreach($files as $file){ + $target = "$runDir/files/$file"; + if(is_file($target)){ + @unlink($target); // left by an interrupted backup: copied again + } + $stat = stat("$dir/$file"); + $entries[] = ['name' => $file, 'size' => $stat['size'], 'mtime' => $stat['mtime'], 'sha256' => Files::copy("$dir/$file", $target, 0600)]; + } + Files::syncDir("$runDir/files"); + $manifest = ['magic' => 'NONEDB4', 'kind' => 'migration-backup'] + $info + ['files' => $entries]; + Files::writeAtomic("$runDir/manifest.json", StoreRecord::sealRecord($manifest), 0600); + return self::verify($runDir); + } + + /** + * The manifest of a backup whose checksum and every file's size and SHA-256 verify + * @throws Stop + */ + public static function verify(string $runDir): array { + $manifest = StoreRecord::openRecord(@file_get_contents("$runDir/manifest.json"), 'migration-backup'); + if($manifest === null || !is_array($manifest['files'] ?? null)){ + throw new Stop('FAILED', "$runDir: no valid manifest.json; the backup is incomplete"); + } + foreach($manifest['files'] as $entry){ + $path = "$runDir/files/" . $entry['name']; + if(!is_file($path) || filesize($path) !== $entry['size'] || hash_file('sha256', $path) !== $entry['sha256']){ + throw new Stop('FAILED', "$runDir: {$entry['name']} does not match the manifest"); + } + } + return $manifest; + } +} +} +namespace noneDB\Migrate\Journal { + +use noneDB\Internal\Storage\StoreRecord; +use noneDB\Migrate\Files; +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Journal { + /** The states of a migration, in order */ + public const STATES = ['DISCOVERED', 'BACKED_UP', 'STAGING', 'VERIFIED', 'PUBLISHING', 'COMMITTED', 'FINAL_VERIFIED']; + + public static function path(string $dir): string { + return rtrim($dir, '/') . '/.nonedb-maintenance/journal.json'; + } + + /** + * The open journal, null when there is none + * @throws Stop an unreadable journal: nothing is decided from it + */ + public static function read(string $dir): ?array { + $path = self::path($dir); + clearstatcache(true, $path); + if(!is_file($path)){ + return null; + } + $journal = StoreRecord::openRecord(@file_get_contents($path), 'journal'); + if($journal === null){ + throw new Stop('FAILED', "$path is unreadable: the state of the interrupted maintenance is unknown; nothing was changed"); + } + return $journal; + } + + /** Write the journal (atomic, synced) */ + public static function write(string $dir, array $journal): void { + Files::makeDir(dirname(self::path($dir))); + Files::writeAtomic(self::path($dir), StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'journal'] + $journal)); + } + + /** Close the journal: it moves to $to (the run's backup directory) */ + public static function close(string $dir, string $to): void { + $path = self::path($dir); + if(!@rename($path, $to)){ + if(!@copy($path, $to) || !@unlink($path)){ + throw new Stop('FAILED', "cannot close the journal $path"); + } + } + Files::syncDir(dirname($path)); + } +} +} +namespace noneDB\Migrate\Publisher { + +use noneDB\Internal\Storage\StoreRecord; +use noneDB\Migrate\Backup\Backup; +use noneDB\Migrate\Converter\Writer; +use noneDB\Migrate\Files; +use noneDB\Migrate\Journal\Journal; +use noneDB\Migrate\Readers\Discovery; +use noneDB\Migrate\Readers\Source; +use noneDB\Migrate\Stop; +use noneDB\Migrate\Validator\Verifier; + +/** + * @internal + */ +final class Migration { + /** @var string the directory, with a trailing slash */ + private $dir; + /** @var string */ + private $sourceKey; + /** @var string */ + private $targetKey; + /** @var array cli options */ + private $options; + /** @var resource[] held until the run ends */ + private $locks = []; + + private function __construct(string $dir, string $sourceKey, string $targetKey, array $options){ + $this->dir = rtrim($dir, '/') . '/'; + $this->sourceKey = $sourceKey; + $this->targetKey = $targetKey; + $this->options = $options + ['db' => null, 'rebuildFromData' => false, 'flushing' => null, 'shardSize' => 10000, 'verify' => false]; + } + + /** + * The pre-check (§7.1): every database of the source key read and judged, nothing written + * @return array the report + */ + public static function check(string $dir, string $sourceKey, string $targetKey, array $options): array { + return (new self($dir, $sourceKey, $targetKey, $options))->precheck()['report']; + } + + /** + * A migration, or the continuation of an interrupted one (§10) + * @return array the report + * @throws Stop + */ + public static function run(string $dir, string $sourceKey, string $targetKey, array $options): array { + $run = new self($dir, $sourceKey, $targetKey, $options); + $journal = Journal::read($run->dir); + if($journal !== null){ + $run->own($journal); + return $run->resume($journal); + } + if(($closed = $run->unverified()) !== null){ + $run->lock([]); + return $run->finish($closed); + } + $check = $run->precheck(); + if($check['report']['result'] !== 'READY'){ + throw new Stop($check['report']['result'], 'the pre-check found databases that cannot be migrated: ' . json_encode($check['report'])); + } + if($check['targets'] === [] && $check['files'] === []){ + // No old file of the key: only a key a partial run left unregistered, with databases here, is registered + $registered = in_array(StoreRecord::keyCheckOf($targetKey), $run->readStore()['keys'] ?? [], true); + if($registered || !$run->hasDatabasesOf($targetKey)){ + return ['run' => null, 'databases' => [], 'keyRegistered' => $registered]; + } + } + $run->checkSpace($check['report']['bytes']); + // Files of the store the run makes where there were none: a restore removes them again (§11). A STORE.LOCK without + // a STORE record is one an earlier run stopped before its journal left (noneDB 4 creates none next to old files) + $noStore = !file_exists($run->dir . 'STORE') && !file_exists($run->dir . 'STORE.2'); + $created = array_values(array_filter(['STORE.LOCK', '.htaccess', 'index.html'], function($file) use ($run, $noStore){ + return !file_exists($run->dir . $file) || ($file === 'STORE.LOCK' && $noStore); + })); + $run->lock(array_keys($check['targets'])); + $runId = date('Ymd-His') . '-' . bin2hex(random_bytes(3)); + $journal = ['op' => 'migrate', 'id' => $runId, 'state' => 'DISCOVERED', 'targets' => array_keys($check['targets']), + 'completesKey' => $check['report']['completesKey'], 'sourceKeyCheck' => StoreRecord::keyCheckOf($sourceKey), + 'targetKeyCheck' => StoreRecord::keyCheckOf($targetKey), 'backup' => rtrim($run->options['backupDir'], '/') . '/' . $runId, + 'options' => array_intersect_key($run->options, array_flip(['rebuildFromData', 'flushing', 'shardSize', 'verify'])), + 'files' => $run->fingerprints($check['files']), 'created' => $created, 'records' => []]; + $run->write($journal); + return $run->resume($journal); + } + + /** + * Roll back an interrupted run that has not committed (§10) + * @return array ['run' => id, 'state' => the state it was in] + * @throws Stop + */ + public static function abort(string $dir): array { + $run = new self($dir, '', '', []); + $journal = Journal::read($run->dir); + if($journal === null || ($journal['op'] ?? null) !== 'migrate'){ + throw new Stop('FAILED', 'no interrupted migration to abort'); + } + $run->lock([]); + $index = array_search($journal['state'], Journal::STATES, true); + if($index === false || $index >= array_search('COMMITTED', Journal::STATES, true)){ + throw new Stop('FAILED', "the run {$journal['id']} is {$journal['state']}: committed runs are completed, not aborted; run the tool again"); + } + if($journal['state'] === 'PUBLISHING'){ + foreach($journal['published'] ?? [] as $locator){ + if(is_dir($run->dir . $locator)){ + Files::removeTree($run->dir . $locator); // still carries m: never visible, the old files are untouched + } + } + Files::syncDir($run->dir); + } + Files::removeTree($run->staging()); + Files::removeTree(Journal::path($run->dir)); + Files::removeStaleTemps(Journal::path($run->dir)); + Files::syncDir(dirname(Journal::path($run->dir))); + return ['run' => $journal['id'], 'state' => $journal['state']]; + } + + // ------------------------------------------------------------------ pre-check + + /** ['report' => array, 'targets' => name => Source, 'files' => the files the run moves] */ + private function precheck(): array { + if(!is_dir($this->dir)){ + throw new Stop('FAILED', "{$this->dir} is not a directory"); + } + try { + new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->staging()]); + } catch(\InvalidArgumentException $e){ + throw new Stop('FAILED', 'noneDB 4 refuses the key or the configuration: ' . $e->getMessage(), $e); + } + $scan = Discovery::scan($this->dir, $this->sourceKey); + $readOptions = array_intersect_key($this->options, array_flip(['rebuildFromData', 'flushing', 'shardSize'])); + $names = $this->options['db'] ?? array_keys($scan['databases']); + foreach($names as $name){ + if(!isset($scan['databases'][$name])){ + throw new Stop('FAILED', "no database named '$name' with this key in {$this->dir}"); + } + } + $report = ['dir' => $this->dir, 'result' => 'READY', 'databases' => [], 'leftovers' => $scan['leftovers'], + 'otherKeys' => $scan['otherKeys'], 'foreign' => $scan['foreign'], 'completesKey' => false, 'bytes' => 0]; + $targets = []; + foreach($scan['databases'] as $name => $files){ + $source = Source::read($this->dir, (string)$name, $this->sourceKey, $files, $readOptions); + if($source->status === 'READY' && is_dir($this->dir . StoreRecord::locatorOf((string)$name, $this->targetKey) . '.ndb4')){ + $source->status = 'AMBIGUOUS_SOURCE'; + $source->reasons[] = 'TARGET_EXISTS: a noneDB 4 database of this name and key already exists; nothing is merged or overwritten'; + } + $target = in_array($name, $names, true); + $report['databases'][$name] = self::describe($source) + ['target' => $target]; + if($target){ + $targets[$name] = $source; + if($source->status !== 'READY' && $report['result'] === 'READY'){ + $report['result'] = $source->status; + } + } + } + // The files the run moves, by database ('': leftovers and files of the directory) + $files = []; + foreach($targets as $name => $source){ + $files[$name] = $source->files; + } + $report['completesKey'] = count($targets) === count($scan['databases']) && $report['result'] === 'READY'; + if($report['completesKey']){ + $own = array_merge($scan['keyFiles'], ...array_values($scan['leftovers'])); + if($scan['otherKeys'] === []){ + $own = array_merge($own, $scan['sharedFiles']); // the 3.0/3.1 hash cache, once no old file of any key is left + } + if($own !== []){ + sort($own); + $files[''] = $own; + } + } + foreach($files as $group){ + foreach($group as $file){ + $report['bytes'] += filesize($this->dir . $file); + } + } + $store = $this->readStore(); + $report['store'] = $store === null ? 'none' : 'valid'; + return ['report' => $report, 'targets' => $targets, 'files' => $files]; + } + + /** True when the directory holds noneDB 4 databases whose directory names are this key's locators */ + private function hasDatabasesOf(string $key): bool { + foreach(glob($this->dir . '*.ndb4', GLOB_ONLYDIR) ?: [] as $home){ + $format = StoreRecord::openRecord(@file_get_contents("$home/FORMAT"), 'database'); + if($format !== null && StoreRecord::locatorOf($format['name'], $key) . '.ndb4' === basename($home)){ + return true; + } + } + return false; + } + + /** + * The backup and the staged databases take about the size of the old files each (§7.1); on one file system, + * the sum + * @throws Stop + */ + private function checkSpace(int $bytes): void { + $backup = rtrim($this->options['backupDir'], '/'); + while(!is_dir($backup) && dirname($backup) !== $backup){ + $backup = dirname($backup); + } + $same = @stat($backup)['dev'] === @stat($this->dir)['dev']; + foreach(array_unique([$this->dir, $backup]) as $path){ + $free = @disk_free_space($path); + $need = $same ? 2 * $bytes : $bytes; + if($free !== false && $free < $need){ + throw new Stop('FAILED', "not enough free space in $path: $need bytes are needed, " . (int)$free . ' are free'); + } + } + } + + /** The report of one database */ + private static function describe(Source $source): array { + return ['status' => $source->status, 'reasons' => $source->reasons, 'layout' => $source->layout, 'shardSize' => $source->shardSize, + 'segments' => $source->kinds, 'records' => $source->count, 'buffered' => $source->buffered, 'nextKey' => $source->nextKey, + 'fields' => $source->fields, 'spatial' => array_keys($source->spatial), 'notCarried' => $source->notCarried]; + } + + // ------------------------------------------------------------------ the run + + /** Continue the run from its journal's state */ + private function resume(array $journal): array { + $this->lock($journal['targets']); + $state = $journal['state']; + if(in_array($state, ['DISCOVERED', 'BACKED_UP', 'STAGING', 'VERIFIED', 'PUBLISHING'], true)){ + $this->verifyFingerprints($journal); + if($state === 'DISCOVERED'){ + $journal = $this->backup($journal); + $state = 'BACKED_UP'; + } elseif($state !== 'PUBLISHING'){ + Backup::verify($journal['backup']); + } + if($state === 'BACKED_UP' || $state === 'STAGING' || $state === 'VERIFIED'){ + $journal = $this->stage($journal); + } + $journal = $this->publish($journal); + $journal = $this->commit($journal); + } + return $this->finish($this->apply($journal)); + } + + private function backup(array $journal): array { + $store = $this->readStore(); + Backup::create(rtrim($this->dir, '/'), $journal['backup'], self::names($journal['files']), ['run' => $journal['id'], + 'dir' => realpath($this->dir), 'created' => time(), 'sourceKeyCheck' => $journal['sourceKeyCheck'], + 'databases' => $journal['targets'], 'store' => $store]); + $journal['state'] = 'BACKED_UP'; + $this->write($journal); + return $journal; + } + + /** Build every target in a staging store and verify it with the engine (§7.4, §7.5) */ + private function stage(array $journal): array { + $journal['state'] = 'STAGING'; + $this->write($journal); + $staging = $this->staging(); + Files::removeTree($staging); + Files::makeDir($staging); + $this->writeStore(rtrim($staging, '/'), ['storeId' => StoreRecord::uuid4(), 'seq' => 1, 'keys' => [StoreRecord::keyCheckOf($this->targetKey)]]); + $sources = $this->sources($journal); + foreach($sources as $source){ + Writer::write($staging, $this->targetKey, $source); + } + $db = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $staging, 'autoCreateDB' => false]); + $scan = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $staging, 'autoCreateDB' => false, 'fieldIndexEnabled' => false, 'spatialIndexEnabled' => false]); + $journal['records'] = []; + foreach($sources as $name => $source){ + // Records first: the engine writes each segment's checkpoint at its first read, one segment at a time, + // before the indexes read every segment in one operation + $samples = Verifier::records($db, $source); + foreach($source->fields as $field){ + $result = $db->createFieldIndex($name, $field); + if(empty($result['success'])){ + throw new Stop('FAILED', "$name: the field index of '$field' cannot be built: " . json_encode($result)); + } + } + foreach($source->spatial as $field => $nodeSize){ + $result = $db->createSpatialIndex($name, $field, $nodeSize === null ? [] : ['nodeSize' => $nodeSize]); + if(empty($result['success'])){ + throw new Stop('FAILED', "$name: the spatial index of '$field' cannot be built: " . json_encode($result)); + } + } + Verifier::totals($db, $scan, $source, $samples); + $journal['records'][$name] = $source->count; + } + unset($db, $scan); + $journal['state'] = 'VERIFIED'; + $this->write($journal); + return $journal; + } + + /** The targets read again from their files (verified unchanged) */ + private function sources(array $journal): array { + $sources = []; + foreach($journal['targets'] as $name){ + $source = Source::read($this->dir, $name, $this->sourceKey, array_keys($journal['files'][$name]), $journal['options']); + if($source->status !== 'READY'){ + throw new Stop($source->status, "$name: " . implode('; ', $source->reasons)); + } + $sources[$name] = $source; + } + return $sources; + } + + /** Move every staged database into the directory, each carrying m (§7.6) */ + private function publish(array $journal): array { + if($journal['state'] !== 'PUBLISHING'){ + $journal['state'] = 'PUBLISHING'; + $journal['published'] = array_map(function($name){ + return StoreRecord::locatorOf($name, $this->targetKey) . '.ndb4'; + }, $journal['targets']); + $this->write($journal); + } + foreach($journal['published'] as $locator){ + $staged = $this->staging() . $locator; + if(is_dir($this->dir . $locator) && !is_dir($staged)){ + continue; // published before the interruption + } + if(!is_dir($staged) || is_dir($this->dir . $locator)){ + throw new Stop('FAILED', "$locator is neither only staged nor only published: the run's state contradicts its files; nothing was changed"); + } + $this->verifyFingerprints($journal, $journal['targets'][array_search($locator, $journal['published'], true)]); + self::writeState($staged . '/LOCK', ['m']); + if(!@rename($staged, $this->dir . $locator)){ + throw new Stop('FAILED', "cannot move $locator into {$this->dir}"); + } + Files::syncDir(rtrim($this->dir, '/')); + Files::syncDir($this->staging()); + } + return $journal; + } + + /** The commit point: the journal records COMMITTED with the target STORE record (§7.7) */ + private function commit(array $journal): array { + $this->verifyFingerprints($journal); + $store = $this->readStore(); + $keyCheck = StoreRecord::keyCheckOf($this->targetKey); + if($store === null){ + $store = ['storeId' => StoreRecord::uuid4(), 'seq' => 1, 'keys' => $journal['completesKey'] ? [$keyCheck] : []]; + } elseif($journal['completesKey'] && !in_array($keyCheck, $store['keys'], true)){ + $store['keys'][] = $keyCheck; + sort($store['keys']); + $store['seq']++; + } + $journal['state'] = 'COMMITTED'; + $journal['store'] = $store; + $this->write($journal); + return $journal; + } + + /** After the commit, each step idempotent: STORE copies, retained sources, m cleared, staging removed (§7.7) */ + private function apply(array $journal): array { + $this->writeStore(rtrim($this->dir, '/'), $journal['store']); + foreach(['.htaccess' => "Require all denied\nDeny from all\n", 'index.html' => ''] as $file => $content){ + if(!file_exists($this->dir . $file)){ + Files::writeNew($this->dir . $file, $content); // no temp file: one would be a foreign entry of the store + } + } + $retained = $this->dir . '.nonedb-retained/' . $journal['id']; + Files::makeDir($retained . '/files'); + foreach(self::prints($journal['files']) as $file => $fingerprint){ + if(is_file($this->dir . $file)){ + if(!@rename($this->dir . $file, "$retained/files/$file")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "committed, not completed: cannot move $file into $retained; run again"); + } + } elseif(!is_file("$retained/files/$file")){ + throw new Stop('FAILED', "$file is neither in the directory nor retained: the run's state contradicts its files"); + } + } + Files::syncDir("$retained/files"); + Files::syncDir(rtrim($this->dir, '/')); + $entries = []; + foreach(self::prints($journal['files']) as $file => $fingerprint){ + $entries[] = ['name' => $file, 'size' => $fingerprint[0], 'sha256' => $fingerprint[3]]; + } + Files::writeAtomic("$retained/manifest.json", StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'migration-retained', + 'run' => $journal['id'], 'backup' => $journal['backup'], 'files' => $entries])); + foreach($journal['published'] as $locator){ + if(is_file($this->dir . $locator . '/LOCK')){ + self::writeState($this->dir . $locator . '/LOCK', []); + } + } + Files::removeTree($this->staging()); + // Closed before the final check: the engine opens no store while a maintenance journal is open + Journal::close(rtrim($this->dir, '/'), $journal['backup'] . '/journal.done.json'); + return $journal; + } + + /** + * The closed journal of a run of these keys in this directory that has no final verification (a crash after the + * close, §10), found through the retained files' manifest; null when there is none + */ + private function unverified(): ?array { + foreach(glob($this->dir . '.nonedb-retained/*/manifest.json') ?: [] as $path){ + $retained = StoreRecord::openRecord(@file_get_contents($path), 'migration-retained'); + $done = $retained === null ? null : StoreRecord::openRecord(@file_get_contents($retained['backup'] . '/journal.done.json'), 'journal'); + if($done !== null && $done['op'] === 'migrate' && $done['state'] === 'COMMITTED' + && $done['sourceKeyCheck'] === StoreRecord::keyCheckOf($this->sourceKey) && $done['targetKeyCheck'] === StoreRecord::keyCheckOf($this->targetKey)){ + return $done; + } + } + return null; + } + + /** + * Final check (§7.8) with a new engine instance: checkIntegrity() and count() of every migrated database, and with + * --verify every record against the live set read from the backup. Recorded as FINAL_VERIFIED in the closed journal. + */ + private function finish(array $journal): array { + $report = ['run' => $journal['id'], 'backup' => $journal['backup'], 'databases' => $journal['records'], + 'keyRegistered' => $journal['completesKey'], 'retained' => '.nonedb-retained/' . $journal['id']]; + if($journal['completesKey']){ + \noneDB::clearStaticCache(); + $db = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->dir, 'autoCreateDB' => false]); + foreach($journal['records'] as $name => $count){ + if(($db->checkIntegrity($name)['ok'] ?? false) !== true || $db->count($name) !== $count){ + throw new Stop('FAILED', "$name: the published database does not read back as verified; the backup is in {$journal['backup']}"); + } + } + if($journal['options']['verify']){ + $scan = new \noneDB(['secretKey' => $this->targetKey, 'dbDir' => $this->dir, 'autoCreateDB' => false, 'fieldIndexEnabled' => false, 'spatialIndexEnabled' => false]); + $backup = Discovery::scan($journal['backup'] . '/files', $this->sourceKey); + foreach($journal['targets'] as $name){ + Verifier::verify($db, $scan, Source::read($journal['backup'] . '/files', $name, $this->sourceKey, + $backup['databases'][$name], $journal['options'])); + } + } + unset($db, $scan); + } + // The report first: a closed journal that records FINAL_VERIFIED always has it next to it + Files::writeAtomic($journal['backup'] . '/report.json', json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n", 0600); + $journal['state'] = 'FINAL_VERIFIED'; + Files::writeAtomic($journal['backup'] . '/journal.done.json', StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'journal'] + $journal), 0600); + return $report; + } + + // ------------------------------------------------------------------ store, locks, fingerprints + + /** + * STORE.LOCK, then the lock file of every target that has one (3.2, 3.3-dev), all non-blocking (§7.2) + * @throws Stop MAINTENANCE_BUSY, nothing written + */ + private function lock(array $targets): void { + if($this->locks !== []){ + return; + } + $paths = [$this->dir . 'STORE.LOCK']; + foreach($targets as $name){ + $path = $this->dir . Discovery::hashOf($name, $this->sourceKey) . "-$name.nonedb.lock"; + if(is_file($path)){ + $paths[] = $path; + } + } + foreach($paths as $path){ + $fp = Files::openLock($path); + $busy = $fp !== false && !flock($fp, LOCK_EX | LOCK_NB); + if($fp === false || $busy){ + if($fp !== false){ + fclose($fp); + } + foreach($this->locks as $held){ + flock($held, LOCK_UN); + fclose($held); + } + $this->locks = []; + throw $busy ? new Stop('MAINTENANCE_BUSY', basename($path) . ' is locked by another process; nothing was changed') + : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); + } + $this->locks[] = $fp; + } + } + + /** The run belongs to these keys: a journal of another source or target key is never resumed */ + private function own(array $journal): void { + if(($journal['op'] ?? null) !== 'migrate'){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "another maintenance operation ({$journal['op']}) is open in {$this->dir}"); + } + if($journal['sourceKeyCheck'] !== StoreRecord::keyCheckOf($this->sourceKey) || $journal['targetKeyCheck'] !== StoreRecord::keyCheckOf($this->targetKey)){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "the interrupted run {$journal['id']} was started with other keys; run it again with them"); + } + } + + /** group => file => [size, mtime, inode, sha256] */ + private function fingerprints(array $groups): array { + $prints = []; + foreach($groups as $group => $files){ + $prints[$group] = []; + foreach($files as $file){ + $prints[$group][$file] = $this->fingerprint($file); + } + } + return $prints; + } + + private function fingerprint(string $file): ?array { + clearstatcache(true, $this->dir . $file); + $stat = @stat($this->dir . $file); + return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], hash_file('sha256', $this->dir . $file)]; + } + + /** Every file of the groups: file => fingerprint */ + private static function prints(array $groups): array { + return array_merge(...array_values($groups)); + } + + private static function names(array $groups): array { + $names = array_keys(self::prints($groups)); + sort($names); + return $names; + } + + /** + * The source files (of one database, or all) as the journal recorded them (§7.6); a change rolls the run back + * @throws Stop AMBIGUOUS_SOURCE + */ + private function verifyFingerprints(array $journal, ?string $group = null): void { + foreach($group === null ? self::prints($journal['files']) : $journal['files'][$group] as $file => $print){ + if($this->fingerprint($file) !== $print){ + foreach($this->locks as $held){ + flock($held, LOCK_UN); + fclose($held); + } + $this->locks = []; + self::abort(rtrim($this->dir, '/')); + throw new Stop('AMBIGUOUS_SOURCE', "$file changed during the run; the run was rolled back and nothing was migrated"); + } + } + } + + /** + * The STORE record of the directory, null when there is none + * @throws Stop a store whose copies are not valid and identical + */ + private function readStore(): ?array { + $copies = []; + foreach(['STORE', 'STORE.2'] as $name){ + $path = $this->dir . $name; + if(is_file($path)){ + $copies[$name] = StoreRecord::openRecord(@file_get_contents($path), 'store'); + } + } + if($copies === []){ + return null; + } + if(count($copies) !== 2 || in_array(null, $copies, true) || $copies['STORE'] !== $copies['STORE.2']){ + throw new Stop('FAILED', "the STORE record of {$this->dir} is not two valid identical copies: open the store with noneDB once (it repairs one damaged copy) or use repairStore()"); + } + $store = $copies['STORE']; + return ['storeId' => $store['storeId'], 'seq' => $store['seq'], 'keys' => $store['keys']]; + } + + /** Both STORE copies (STORE.2, then STORE), each written only when it differs (STORAGE_FORMAT.md §3.1) */ + private function writeStore(string $dir, array $store): void { + $bytes = StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT] + $store); + foreach(['STORE.2', 'STORE'] as $name){ + if(@file_get_contents("$dir/$name") !== $bytes){ + Files::writeAtomic("$dir/$name", $bytes); + } + } + } + + /** A database's state record with a new epoch and $flags (STORAGE_FORMAT.md §4.2), synced */ + private static function writeState(string $lockPath, array $flags): void { + $record = (string)@file_get_contents($lockPath, false, null, 0, 128); + $gen = preg_match('/^(\d+):/', $record, $m) ? (int)$m[1] : 1; + $fp = @fopen($lockPath, 'c+'); + $state = str_pad(implode(' ', array_merge([$gen . ':' . bin2hex(random_bytes(4))], $flags)), 128); + if($fp === false || fwrite($fp, $state) !== 128 || !fflush($fp) || !Files::syncHandle($fp)){ + throw new Stop('FAILED', "cannot write the state record of $lockPath"); + } + fclose($fp); + } + + private function write(array $journal): void { + Journal::write(rtrim($this->dir, '/'), $journal); + } + + + private function staging(): string { + return $this->dir . '.nonedb-maintenance/staging/'; + } +} +} +namespace noneDB\Migrate\Restore { + +use noneDB\Internal\Storage\StoreRecord; +use noneDB\Migrate\Backup\Backup; +use noneDB\Migrate\Files; +use noneDB\Migrate\Journal\Journal; +use noneDB\Migrate\Stop; + +/** + * @internal + */ +final class Restore { + /** + * The directory as before the run $runId (§11): its noneDB 4 databases move into the backup, the key ($key, the + * run's target key) leaves the STORE record, the old files come back byte for byte + * @return array report + * @throws Stop + */ + public static function restore(string $dir, string $backupDir, string $runId, string $key): array { + $dir = rtrim($dir, '/') . '/'; + $runDir = rtrim($backupDir, '/') . '/' . $runId; + $manifest = Backup::verify($runDir); + $done = StoreRecord::openRecord(@file_get_contents("$runDir/journal.done.json"), 'journal'); + if($done === null){ + throw new Stop('FAILED', "$runDir has no closed journal: the run did not complete, run the tool again (or --abort it)"); + } + if(StoreRecord::keyCheckOf($key) !== $done['targetKeyCheck']){ + throw new Stop('FAILED', "the run $runId registered another key: restore it with that key (NONEDB_SECRET)"); + } + $open = Journal::read($dir); + if($open !== null && ($open['op'] !== 'restore-old' || $open['id'] !== $runId)){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "another maintenance operation ({$open['op']}) is open in $dir"); + } + $locks = self::lock($dir, $done['published']); + // A database the key gained after the run would lose its key: refused, nothing changed + foreach(glob($dir . '*.ndb4', GLOB_ONLYDIR) ?: [] as $home){ + $format = StoreRecord::openRecord(@file_get_contents("$home/FORMAT"), 'database'); + if($format !== null && !in_array(basename($home), $done['published'], true) + && StoreRecord::locatorOf($format['name'], $key) . '.ndb4' === basename($home)){ + foreach($locks as $fp){ + fclose($fp); + } + throw new Stop('FAILED', "the key has a database created after the run ({$format['name']}): restoring would make it unreachable; nothing was changed"); + } + } + Journal::write(rtrim($dir, '/'), ['op' => 'restore-old', 'id' => $runId, 'state' => 'RESTORING']); + Files::makeDir("$runDir/v4", 0700); + foreach($done['published'] as $locator){ + if(is_dir($dir . $locator) && !@rename($dir . $locator, "$runDir/v4/$locator")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $locator into the backup; run --restore again"); + } + } + self::removeKey($dir, $runDir, $done['targetKeyCheck']); + $retained = $dir . '.nonedb-retained/' . $runId . '/files/'; + foreach($manifest['files'] as $entry){ + $name = $entry['name']; + if(is_file($dir . $name) && hash_file('sha256', $dir . $name) === $entry['sha256']){ + continue; + } + if(is_file($retained . $name) && hash_file('sha256', $retained . $name) === $entry['sha256']){ + if(!@rename($retained . $name, $dir . $name)){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $name back; run --restore again"); + } + } elseif(Files::copy("$runDir/files/$name", $dir . $name) !== $entry['sha256']){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "$name does not copy back byte for byte; run --restore again"); + } + } + Files::syncDir(rtrim($dir, '/')); + if(is_dir($dir . '.nonedb-retained/' . $runId)){ + Files::removeTree($dir . '.nonedb-retained/' . $runId); + } + Journal::close(rtrim($dir, '/'), "$runDir/restore.done.json"); + if(!is_file($dir . 'STORE') && !is_file($dir . 'STORE.2')){ + // No store is left: the files the run made, and the store's directories when empty, go too + foreach($done['created'] as $file){ + if(is_file($dir . $file)){ + Files::removeTree($dir . $file); + } + } + foreach(['.nonedb-maintenance', '.nonedb-retained'] as $sub){ + if(is_dir($dir . $sub) && (scandir($dir . $sub) ?: []) === ['.', '..']){ + Files::removeTree($dir . $sub); + } + } + Files::syncDir(rtrim($dir, '/')); + } + foreach($locks as $fp){ + flock($fp, LOCK_UN); + fclose($fp); + } + return ['run' => $runId, 'restored' => count($manifest['files']), 'v4' => "$runDir/v4"]; + } + + /** + * Remove .nonedb-retained/$runId after verifying every file against the backup's manifest (§12) + * @throws Stop + */ + public static function cleanup(string $dir, string $backupDir, string $runId): array { + $dir = rtrim($dir, '/') . '/'; + $manifest = Backup::verify(rtrim($backupDir, '/') . '/' . $runId); + $retained = $dir . '.nonedb-retained/' . $runId; + if(!is_dir($retained)){ + return ['run' => $runId, 'removed' => 0]; + } + $done = StoreRecord::openRecord(@file_get_contents(rtrim($backupDir, '/') . "/$runId/journal.done.json"), 'journal'); + if(($done['state'] ?? null) !== 'FINAL_VERIFIED'){ + throw new Stop('FAILED', "the run $runId has no final verification; run the tool again first (it verifies the run); nothing was removed"); + } + foreach($manifest['files'] as $entry){ + $path = "$retained/files/" . $entry['name']; + if(is_file($path) && hash_file('sha256', $path) !== $entry['sha256']){ + throw new Stop('FAILED', "$path differs from the backup; nothing was removed"); + } + } + Files::removeTree($retained); + Files::syncDir($dir . '.nonedb-retained'); + return ['run' => $runId, 'removed' => count($manifest['files'])]; + } + + /** STORE.LOCK, then the LOCK of every database of the run, all non-blocking */ + private static function lock(string $dir, array $published): array { + $locks = []; + $paths = [$dir . 'STORE.LOCK']; + foreach($published as $locator){ + if(is_file($dir . $locator . '/LOCK')){ + $paths[] = $dir . $locator . '/LOCK'; + } + } + foreach($paths as $path){ + $fp = Files::openLock($path); + $busy = $fp !== false && !flock($fp, LOCK_EX | LOCK_NB); + if($fp === false || $busy){ + if($fp !== false){ + fclose($fp); + } + foreach($locks as $held){ + fclose($held); + } + throw $busy ? new Stop('MAINTENANCE_BUSY', basename(dirname($path)) . '/' . basename($path) . ' is locked by another process; nothing was changed') + : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); + } + $locks[] = $fp; + } + return $locks; + } + + /** The run's key leaves the STORE record; when no key is left, the STORE copies move into the backup too */ + private static function removeKey(string $dir, string $runDir, string $keyCheck): void { + $store = StoreRecord::openRecord(@file_get_contents($dir . 'STORE'), 'store'); + if($store === null){ + return; // removed by an earlier attempt + } + $keys = array_values(array_diff($store['keys'], [$keyCheck])); + if($keys === []){ + foreach(['STORE', 'STORE.2'] as $name){ + if(is_file($dir . $name) && !@rename($dir . $name, "$runDir/v4/$name")){ + throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $name into the backup; run --restore again"); + } + } + return; + } + if($keys !== $store['keys']){ + $bytes = StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => $store['storageFormat'], + 'storeId' => $store['storeId'], 'seq' => $store['seq'] + 1, 'keys' => $keys]); + Files::writeAtomic($dir . 'STORE.2', $bytes); + Files::writeAtomic($dir . 'STORE', $bytes); + } + } +} +} +namespace noneDB\Migrate { + +use noneDB\Migrate\Publisher\Migration; +use noneDB\Migrate\Restore\Restore; + +/** + * @internal + */ +final class Cli { + private const USAGE = <<<'TXT' +usage: NONEDB_SECRET=... php noneDB-migrate.php --dir=DIR [--config=FILE] (--dry-run | --backup-dir=DIR --confirm-stopped + [--db=NAME,...] [--verify] [--rebuild-from-data] [--flushing=import|skip] [--shard-size=N] | --abort + | --restore=RUNID --backup-dir=DIR | --cleanup=RUNID --backup-dir=DIR) [--json] + NONEDB_SOURCE_SECRET: the key the old files were written with, when it differs (docs/MIGRATION.md §2) + +TXT; + private const FLAGS = ['dry-run', 'confirm-stopped', 'verify', 'rebuild-from-data', 'abort', 'json']; + private const VALUES = ['dir', 'config', 'backup-dir', 'db', 'flushing', 'shard-size', 'restore', 'cleanup']; + + /** @return int the exit status (§13) */ + public static function main(array $args): int { + ini_set('serialize_precision', '-1'); + $o = self::parse($args); + if($o === null){ + fwrite(STDERR, self::USAGE); + return 64; + } + $json = isset($o['json']); + try { + $result = self::dispatch($o); + } catch(Stop $e){ + self::output($json, ['result' => $e->result, 'message' => $e->getMessage()]); + return Stop::STATUS[$e->result]; + } catch(\Throwable $e){ + // Anything else: the run stays where its journal says, to be continued or aborted + self::output($json, ['result' => 'FAILED', 'message' => get_class($e) . ': ' . $e->getMessage() . ' (' . basename($e->getFile()) . ':' . $e->getLine() . ')']); + return Stop::STATUS['FAILED']; + } + self::output($json, $result); + return isset($result['report']['result']) && $result['report']['result'] !== 'READY' ? Stop::STATUS[$result['report']['result']] : 0; + } + + /** Options, or null for a usage error */ + private static function parse(array $args): ?array { + $o = []; + foreach($args as $arg){ + if(!preg_match('/^--([a-z-]+)(?:=(.*))?$/s', $arg, $m)){ + return null; + } + $flag = in_array($m[1], self::FLAGS, true); + if((!$flag && !in_array($m[1], self::VALUES, true)) || $flag === isset($m[2]) || isset($o[$m[1]])){ + return null; + } + $o[$m[1]] = $m[2] ?? true; + } + $modes = count(array_intersect_key($o, array_flip(['dry-run', 'abort', 'restore', 'cleanup']))); + if($modes > 1 || (isset($o['flushing']) && !in_array($o['flushing'], ['import', 'skip'], true)) + || (isset($o['shard-size']) && !preg_match('/^[1-9]\d*$/', $o['shard-size'])) + || ($modes === 0 && (!isset($o['backup-dir']) || !isset($o['confirm-stopped']))) + || ((isset($o['restore']) || isset($o['cleanup'])) && !isset($o['backup-dir']))){ + return null; + } + return $o; + } + + /** + * @throws Stop + */ + private static function dispatch(array $o): array { + $config = []; + if(isset($o['config'])){ + $config = json_decode((string)@file_get_contents($o['config']), true); + if(!is_array($config)){ + throw new Stop('FAILED', "{$o['config']} is not a readable .nonedb config file"); + } + if(isset($config['dbDir']) && is_string($config['dbDir']) && $config['dbDir'] !== '' && $config['dbDir'][0] !== '/'){ + $config['dbDir'] = dirname(realpath($o['config'])) . '/' . $config['dbDir']; // relative to the config file + } + } + $dir = $o['dir'] ?? $config['dbDir'] ?? null; + if(!is_string($dir) || $dir === ''){ + throw new Stop('FAILED', 'no directory: --dir or a config file with dbDir'); + } + if(isset($o['abort'])){ + return ['aborted' => Migration::abort($dir)]; + } + $target = getenv('NONEDB_SECRET'); + $target = is_string($target) && $target !== '' ? $target : ($config['secretKey'] ?? null); + if(!is_string($target) || $target === ''){ + throw new Stop('FAILED', 'no key: set NONEDB_SECRET or use a config file with secretKey (never a command line argument)'); + } + if(isset($o['restore'])){ + return ['restored' => Restore::restore($dir, $o['backup-dir'], $o['restore'], $target)]; + } + if(isset($o['cleanup'])){ + return ['cleaned' => Restore::cleanup($dir, $o['backup-dir'], $o['cleanup'])]; + } + $source = getenv('NONEDB_SOURCE_SECRET'); + $options = [ + 'db' => isset($o['db']) ? explode(',', $o['db']) : null, + 'rebuildFromData' => isset($o['rebuild-from-data']), + 'flushing' => $o['flushing'] ?? null, + 'shardSize' => (int)($o['shard-size'] ?? $config['shardSize'] ?? 10000), + 'verify' => isset($o['verify']), + ]; + $source = is_string($source) && $source !== '' ? $source : $target; + if(isset($o['dry-run'])){ + return ['report' => Migration::check($dir, $source, $target, $options)]; + } + $real = realpath($dir); + $backup = rtrim($o['backup-dir'], '/') . '/'; + if($real === false || strncmp(self::absolute($backup), $real . '/', strlen($real) + 1) === 0){ + throw new Stop('FAILED', '--backup-dir must be outside --dir'); + } + return ['migrated' => Migration::run($dir, $source, $target, $options + ['backupDir' => $backup])]; + } + + /** A path made absolute without requiring it to exist */ + private static function absolute(string $path): string { + $real = realpath($path); + if($real !== false){ + return $real . '/'; + } + return ($path[0] === '/' ? '' : getcwd() . '/') . $path; + } + + private static function output(bool $json, array $result): void { + if($json){ + echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n"; + return; + } + if(isset($result['message'])){ + fwrite(STDERR, "noneDB-migrate: {$result['result']}: {$result['message']}\n"); + return; + } + if(isset($result['report'])){ + $report = $result['report']; + echo "noneDB-migrate: {$report['dir']}\n"; + foreach($report['databases'] as $name => $db){ + printf(" %-24s %-18s %s%s, %d records, next key %d%s\n", $name, $db['status'], + $db['layout'], $db['layout'] === 'sharded' ? " ({$db['shardSize']} per shard)" : '', $db['records'], $db['nextKey'], + $db['target'] ? '' : ' (not in this run)'); + foreach(array_merge($db['reasons'], array_map(function($reason, $count){ return "not carried over: $count $reason"; }, + array_keys($db['notCarried']), $db['notCarried'])) as $line){ + echo " $line\n"; + } + } + foreach(['leftovers' => 'leftovers of this key', 'otherKeys' => 'files of other keys (untouched)', + 'foreign' => 'other entries (untouched)'] as $list => $label){ + if($report[$list] !== []){ + echo " $label: " . implode(', ', $list === 'leftovers' ? array_keys($report[$list]) : $report[$list]) . "\n"; + } + } + echo " result: {$report['result']}" . ($report['completesKey'] ? '' : ' (the key would not be registered by this run)') . "\n"; + return; + } + echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n"; + } +} +} +namespace { +if (PHP_SAPI === 'cli' && isset($argv[0]) && (realpath($argv[0]) === __FILE__ || isset($GLOBALS['_composer_bin_dir']))) { + if (!class_exists('noneDB', false)) { + require __DIR__ . '/noneDB.php'; + } + exit(\noneDB\Migrate\Cli::main(array_slice($argv, 1))); +} +} diff --git a/noneDB.php b/noneDB.php index d763ce9..fc558ce 100644 --- a/noneDB.php +++ b/noneDB.php @@ -11,4520 +11,6878 @@ * Respect for Labour - Emeğe Saygı */ -class noneDB { +namespace noneDB\Internal\Cache { + +/** + * @internal + */ +final class Runtime { + private static $instance = null; + + public static function get(): self { + return self::$instance ?? (self::$instance = new self()); + } + // Configuration file name - private static $configFile = '.nonedb'; - private static $configLoaded = false; - private static $configData = null; - private static $configFilePath = null; // Path of the loaded config file (null = none / dev mode) - private static $configFileDir = null; // Directory relative dbDir paths in the file resolve against - private static $scriptDirs = []; // SCRIPT_FILENAME + cwd -> resolved entry script directory + public $configFile = '.nonedb'; + public $configLoaded = false; + public $configData = null; + public $configFilePath = null; // Path of the loaded config file (null = none / dev mode) + public $configFileDir = null; // Directory relative dbDir paths in the file resolve against + public $scriptDirs = []; // SCRIPT_FILENAME + cwd -> resolved entry script directory // Placeholder keys shipped in the docs/templates; never accepted as a real secret - private static $placeholderKeys = [ + public $placeholderKeys = [ 'CHANGE_THIS_TO_A_SECURE_RANDOM_STRING', 'YOUR_SECURE_RANDOM_STRING', 'YOUR_SECURE_RANDOM_STRING_HERE', 'GENERATE_UNIQUE_64_CHAR_STRING', 'your_secure_key', 'your_key', + 'nonedb_123', // a default key that was published: as public as the examples ]; // secretKey values live outside the object so print_r/var_export/serialize can't leak them - private static $secretKeys = []; + public $secretKeys = []; + public $bufferCounts=[]; // bufferPath => [size, count] so limits don't need a re-read per insert + public $bufferEnds=[]; // bufferPath => size the buffer had after this process's last append + public $flushingNow=[]; // database base path => true while its buffer is being flushed + + // Static caches for cross-instance sharing + public $staticIndexCache=[]; + public $staticVerifiedIndexes=[]; + public $pendingRecoveryEvents=[]; // lock path -> event id -> event found by a reader, logged by the next writer + public $uncheckpointed=[]; // lock path -> segments loaded without a checkpoint, written by the next writer + public $decodeDue=[]; // lock path -> key index paths decoded at the start of the database's next operation + public $staticShardedCache=[]; + public $staticMetaCache=[]; + public $staticHashCache=[]; + public $staticFileExistsCache=[]; + public $staticSanitizeCache=[]; + public $staticFieldIndexCache=[]; + public $staticSpatialIndexCache=[]; + public $staticGlobalSpatialIndexCache=[]; + public $staticGenerations=[]; // lock path -> generation the caches were validated against + public $staticCacheEnabled=true; // Enable/disable static caching + + // Process-wide lock registry (shared by every instance so two instances never self-deadlock) + public $lockRegistry=[]; // lock path -> ['fp', 'mode' => LOCK_SH|LOCK_EX, 'depth', 'dbname', 'gen', 'flags', 'bumped', 'pending', 'scope', 'rolledBack', 'sid'] while locked + public $lockHandles=[]; // lock path -> ['fp' => resource, 'pid' => int]: unlocked handles kept open for the next operation + public $sessionCounter=0; // id of the latest lock session ('sid' in the registry) + public $sessionLimit=null; // the cache limit in that session, null until it is needed (segments read it, memory_limit once) + public $sessionNear=null; // where the process's peak is near memory_limit in that session, null until it is needed + public $shutdownCleanupRegistered=false; + public $syncStats=['file' => 0, 'dir' => 0, 'failed' => 0]; + // getMetrics() (docs/OPERATIONS.md §1): counters of the process, each counted where its work is done + public $metrics=['hits' => 0, 'misses' => 0, 'evictions' => 0, 'pages' => 0, 'bytes' => 0, 'checkpoints' => 0, + 'checkpointSeconds' => 0.0, 'compactions' => 0, 'compactionSeconds' => 0.0, 'locks' => 0, 'lockWaits' => 0, + 'lockWaitSeconds' => 0.0, 'lockTimeouts' => 0, 'recoveryEvents' => 0, 'recoveryActions' => []]; + public $integrityChecks=[]; // databaseId -> summary of its last checkIntegrity() + public $lastIntegrity=null; // summary of the process's last checkIntegrity() + public $lastSyncError=null; + public $lastOperation=null; // outcome of the last write operation of the process (getDurability()) + public $noWrites=0; // > 0 while checkIntegrity() runs: no recovery, no repair, no file written + public $multiLock=0; // > 0 inside withSnapshot(): no exclusive lock may be waited for + + // Static caches are dropped while the process uses more memory than this: when the last lock is released + // and between the segments of an operation (CacheManager::evictCaches()) + public $cacheMemoryLimit = null; // bytes, 0 = no limit, null = automatic (cacheLimitBytes); the last instance's setting counts + public $memoryLimitSetting = null; // the memory_limit value memoryLimitHalf was worked out from + public $memoryLimitHalf = 0; // half of it in bytes (the automatic cache limit), 0 = no limit + public $cacheTouch = []; // cache key -> tick of its last use + public $cacheTick = 0; + public $cacheGrew = false; // something was loaded into a cache since evictCaches() last looked + public $exactFloats = false; // serialize_precision is -1 for the batch being encoded (withExactFloats) + public $staticDataSizes = []; + public $staticGlobalSkip = []; + public $staticSegWrites = []; + public $staticDataEpochs = []; + public $staticShardFilesUpTo = []; + public $secondaryChecked = []; // data path -> "epoch:bytes:crc" a secondary index or global entry was checked against + public $tailCache = []; // data path -> [from, to, rows] of the last parsed data tail + public $indexTokens = []; // field -> its index file name part (indexToken) + public $staticWritableIndexes = []; + + public $devModeNoticeShown = false; + + public $storeState = []; // dbDir => keyCheck => true: the key is registered (store opened) + public $storeLocks = []; // dbDir => ['fp' => resource, 'mode' => LOCK_SH|LOCK_EX, 'depth' => int] + public $maintenanceDepth = 0; // > 0 while this process runs maintenance (its own journal is open) + public $staticCurrentGenerations = []; + public $staticFormatRecords = []; + + public $parsedLockStates = []; // lock path => [raw record, state, flags, damaged] of its last read + +} +} +namespace noneDB\Internal\Core { + +use function array_fill_keys, get_object_vars, is_object; + +use noneDB\Internal\Cache\Runtime; - private $dbDir = null; // Set via .nonedb config file or constructor - private $autoCreateDB = true; - private $devMode = false; +/** + * @internal + */ +final class Kernel { + /** @var Runtime */ + public $rt; + /** @var array values linked to the process-wide caches (shared by every instance) */ + public $linked = []; + /** @var \noneDB\Internal\Config\Configuration */ + public $config; + /** @var \noneDB\Internal\Cache\CacheManager */ + public $cache; + /** @var \noneDB\Internal\Storage\StoreRecord */ + public $store; + /** @var \noneDB\Internal\Storage\Layout */ + public $layout; + /** @var \noneDB\Internal\Storage\Segments */ + public $segments; + /** @var \noneDB\Internal\Storage\Checkpoints */ + public $checkpoints; + /** @var \noneDB\Internal\Storage\RecordIO */ + public $records; + /** @var \noneDB\Internal\Storage\Writer */ + public $writer; + /** @var \noneDB\Internal\Storage\Buffer */ + public $buffer; + /** @var \noneDB\Internal\Concurrency\LockManager */ + public $locks; + /** @var \noneDB\Internal\Concurrency\Snapshot */ + public $snapshot; + /** @var \noneDB\Internal\Recovery\RollbackScope */ + public $scope; + /** @var \noneDB\Internal\Recovery\RecoveryLog */ + public $recoveryLog; + /** @var \noneDB\Internal\Recovery\Recovery */ + public $recovery; + /** @var \noneDB\Internal\Durability\FileIO */ + public $io; + /** @var \noneDB\Internal\Durability\Sync */ + public $sync; + /** @var \noneDB\Internal\Sharding\ShardMeta */ + public $shardMeta; + /** @var \noneDB\Internal\Sharding\ShardRouter */ + public $shardRouter; + /** @var \noneDB\Internal\Sharding\LayoutSwitch */ + public $layoutSwitch; + /** @var \noneDB\Internal\Sharding\ShardInfo */ + public $shardInfo; + /** @var \noneDB\Internal\Index\SecondaryFiles */ + public $secondary; + /** @var \noneDB\Internal\Index\FieldIndex */ + public $fieldIndex; + /** @var \noneDB\Internal\Index\FieldIndexApi */ + public $fieldIndexApi; + /** @var \noneDB\Internal\Index\Spatial\GeoJson */ + public $geoJson; + /** @var \noneDB\Internal\Index\Spatial\Geometry */ + public $geometry; + /** @var \noneDB\Internal\Index\Spatial\RTree */ + public $rtree; + /** @var \noneDB\Internal\Index\Spatial\SpatialIndexApi */ + public $spatialIndexApi; + /** @var \noneDB\Internal\Query\FindFilter */ + public $findFilter; + /** @var \noneDB\Internal\Query\Values */ + public $values; + /** @var \noneDB\Internal\Query\Planner */ + public $planner; + /** @var \noneDB\Internal\Query\QueryMethods */ + public $queryMethods; + /** @var \noneDB\Internal\Query\Aggregation */ + public $aggregation; + /** @var \noneDB\Internal\Query\Utility */ + public $utility; + /** @var \noneDB\Internal\Query\SpatialQueries */ + public $spatialQueries; + /** @var \noneDB\Internal\Core\Crud */ + public $crud; + /** @var \noneDB\Internal\Core\DatabaseOps */ + public $db; + /** @var \noneDB\Internal\Diagnostics\Integrity */ + public $integrity; + /** @var \noneDB\Internal\Diagnostics\Metrics */ + public $metrics; + /** @var \noneDB\Internal\Maintenance\Maintenance */ + public $maintenance; + /** @var \noneDB\Internal\Maintenance\Compaction */ + public $compaction; + /** @var \noneDB\Internal\Maintenance\KeyMaintenance */ + public $keyMaintenance; + /** @var \noneDB\Internal\Maintenance\BackupRestore */ + public $backupRestore; + + + public $dbDir = null; // Set via .nonedb config file or constructor + public $autoCreateDB = true; + public $devMode = false; // Sharding configuration - private $shardingEnabled=true; // Enable/disable auto-sharding - private $shardSize=10000; // Records per shard for NEW sharded databases (existing ones keep the size stored in their meta) - private $autoMigrate=true; // Auto-migrate legacy DBs to sharded format + public $shardingEnabled=true; // A database switches to the sharded layout once it has assigned shardSize keys + public $shardSize=10000; // Records per shard of databases created from now on (each keeps the size in its FORMAT) // File locking configuration - private $lockTimeout=5; // Max seconds to wait for a database lock (0 = wait forever) - private $lockRetryDelay=10000; // Microseconds between lock attempts (10ms) + public $lockTimeout=5; // Max seconds to wait for a database lock (0 = wait forever) + public $lockRetryDelay=10000; // Microseconds between lock attempts (10ms) - // Write buffer configuration (off by default since v3.2.0: direct writes are faster and always visible) - private $bufferEnabled=false; // Enable/disable write buffering - private $bufferSizeLimit=1048576; // 1MB buffer size limit per buffer - private $bufferCountLimit=10000; // Max records per buffer (safety limit) - private $bufferFlushOnRead=true; // Flush buffer before read operations - private $bufferFlushInterval=30; // Seconds between auto-flush (0 = disabled) - private $bufferAutoFlushOnShutdown=true; // Register shutdown handler for flush + // Write buffer configuration (off by default: direct writes are faster and always visible) + public $bufferEnabled=false; // Enable/disable write buffering + public $bufferSizeLimit=1048576; // 1MB buffer size limit per buffer + public $bufferCountLimit=10000; // Max records per buffer (safety limit) + public $bufferFlushInterval=30; // Seconds between auto-flush (0 = disabled) + public $bufferAutoFlushOnShutdown=true; // Register shutdown handler for flush // Buffer state tracking (runtime) - private $bufferLastFlush=[]; // Track last flush time per DB/shard - private $shutdownHandlerRegistered=false; // Track if shutdown handler is registered - private static $bufferCounts=[]; // bufferPath => [size, count] so limits don't need a re-read per insert - private static $flushingNow=[]; // database base path => true while its buffer is being flushed + public $bufferLastFlush=[]; // Track last flush time per DB/shard + public $shutdownHandlerRegistered=false; // Track if shutdown handler is registered // Performance caches (runtime). All keys are full file paths (or key-fingerprint|dbname for hashes), // so instances with different dbDir/secretKey never see each other's entries. - private $hashCache=[]; // keyFingerprint|dbname -> hash (PBKDF2 is expensive) - private $metaCache=[]; // meta path -> meta data - private $metaCacheTime=[]; // kept for API compatibility (unused since v3.2.0) + public $hashCache=[]; // keyFingerprint|dbname -> locator + public $metaCache=[]; // meta path -> meta data // Index configuration - private $indexEnabled=true; // Kept for API compatibility (enableIndexing / isIndexingEnabled) - private $indexCache=[]; // jidx path -> JSONL index - private $shardedCache=[]; // meta path -> bool + public $indexCache=[]; // checkpoint path -> key index of the segment (Checkpoints, KeyIndex) + public $verifiedIndexes=[]; // checkpoint path -> true: the cached index covers the data file as it is + public $shardedCache=[]; // meta path -> bool // JSONL Storage Engine - private $jsonlFormatCache=[]; // data path -> bool (JSONL vs legacy v2) - private $jsonlGarbageThreshold=0.3; // Trigger compaction when garbage > 30% of live records + public $autoCompactThreshold=0.3; // Trigger compaction when garbage > 30% of live records + public $generations=[]; // per-instance view of $staticGenerations - // Static caches for cross-instance sharing - private static $staticIndexCache=[]; - private static $staticShardedCache=[]; - private static $staticMetaCache=[]; - private static $staticMetaCacheTime=[]; - private static $staticHashCache=[]; - private static $staticFormatCache=[]; - private static $staticFileExistsCache=[]; - private static $staticSanitizeCache=[]; - private static $staticFieldIndexCache=[]; - private static $staticSpatialIndexCache=[]; - private static $staticGlobalSpatialIndexCache=[]; - private static $staticGenerations=[]; // lock path -> generation the caches were validated against - private static $staticHashFilesLoaded=[]; // hash cache file -> true - private static $staticCacheEnabled=true; // Enable/disable static caching + // Durability mode (docs/DURABILITY.md): true = 'durable', every sync of §3 is done and required + public $durable=false; - // Process-wide lock registry (shared by every instance so two instances never self-deadlock) - private static $lockRegistry=[]; // lock path -> ['fp', 'mode' => LOCK_SH|LOCK_EX, 'depth', 'dbname', 'gen', 'flags', 'bumped', 'pending'] while locked - private static $lockHandles=[]; // lock path -> ['fp' => resource, 'pid' => int]: unlocked handles kept open for the next operation - private $generations=[]; // per-instance view of $staticGenerations - private $hashFilesLoaded=[]; + // Permissions of what noneDB creates in dbDir, whatever the umask (docs/STORAGE_FORMAT.md §3) + public $filePermissions = 0640; + public $directoryPermissions = 0750; // Field indexing configuration - private $fieldIndexEnabled = true; // Enable field-based indexing - private $fieldIndexCache = []; // field index path -> index + public $fieldIndexEnabled = true; // Enable field-based indexing + public $fieldIndexCache = []; // field index path -> index // Spatial indexing configuration - private $spatialIndexEnabled = true; // Enable spatial R-tree indexing - private $spatialIndexCache = []; // spatial index path -> index - private $globalSpatialIndexCache = []; // global spatial index path -> index - private $rtreeNodeSize = 32; // Max entries per R-tree node - private $dirtySpatialIndexes = []; // Dirty flags for batch spatial writes (flushed before a write lock is released) - private $distanceCache = []; // Haversine distance memoization cache - private $centroidCache = []; // Geometry centroid cache - - private $keyFingerprint = null; // HMAC of the secret, used to bind caches to the key - private $dbBases = []; // dbname -> file name prefix (depends on dbDir and key) - private $dbBasesDir = null; // dbDir the prefixes were built for + public $spatialIndexEnabled = true; // Enable spatial R-tree indexing + public $spatialIndexCache = []; // spatial index path -> index + public $globalSpatialIndexCache = []; // global spatial index path -> index + public $rtreeNodeSize = 32; // Max entries per R-tree node + public $appendMismatch = false; // appendToFile(): the file didn't have the expected size + public $dataSizes = []; // data path -> size, until another process writes (global index skips) + public $dataEpochs = []; // data path -> segment epoch read from its checkpoint (global index entries) + public $shardFilesUpTo = []; // database file prefix -> highest shard number with files (setOrphanShardsAside) + public $globalSkip = []; // lock path -> global field index path -> shards a lookup checks (globalCheckShards) + public $segWrites = []; // lock path -> ['seq' => n, 'paths' => [data path => seq]]: data files this process wrote + public $writableIndexes = []; // checkpoint path -> lock session id in which it was checked against its data file + public $indexedCandidates = []; // key => true: the last spatialCandidates() found it in an R-tree (valid geometry) + + public $keyFingerprint = null; // HMAC of the secret, used to bind caches to the key + + public $cloneSourceId = null; + // ========================================== + // DATABASE DIRECTORY (docs/STORAGE_FORMAT.md §4, §5, §8) + // ========================================== + // + // {dbDir}{locator}.ndb4/ holds FORMAT (identity), LOCK (lock + state record), CURRENT (active + // generation) and generations/{N}/ (metadata, shards.json, indexes.json, segments/, indexes/, + // spatial/, buffer/). The generation number of a database is read from CURRENT once per lock epoch. + + public $currentGenerations = []; // lock path => active generation number + public $formatRecords = []; // lock path => FORMAT record of a present database + + public $dbHomes = []; // database name => its directory (dbDir and key are fixed per instance) + public $lockPaths = []; // database name => its LOCK file + public $genPaths = []; // database name => paths of its active generation (genDir()) + + /** + * Kernels of the live noneDB instances by the instance's id. The instance holds only that id: a dump of + * it (var_export, json_encode) shows nothing internal, and no reference cycle is reachable from it. + * @var array + */ + public static $instances = []; + + /** The services: kernel property => class (created on first use) */ + public const SERVICES = [ + 'config' => \noneDB\Internal\Config\Configuration::class, + 'cache' => \noneDB\Internal\Cache\CacheManager::class, + 'store' => \noneDB\Internal\Storage\StoreRecord::class, + 'layout' => \noneDB\Internal\Storage\Layout::class, + 'segments' => \noneDB\Internal\Storage\Segments::class, + 'checkpoints' => \noneDB\Internal\Storage\Checkpoints::class, + 'records' => \noneDB\Internal\Storage\RecordIO::class, + 'writer' => \noneDB\Internal\Storage\Writer::class, + 'buffer' => \noneDB\Internal\Storage\Buffer::class, + 'locks' => \noneDB\Internal\Concurrency\LockManager::class, + 'snapshot' => \noneDB\Internal\Concurrency\Snapshot::class, + 'scope' => \noneDB\Internal\Recovery\RollbackScope::class, + 'recoveryLog' => \noneDB\Internal\Recovery\RecoveryLog::class, + 'recovery' => \noneDB\Internal\Recovery\Recovery::class, + 'io' => \noneDB\Internal\Durability\FileIO::class, + 'sync' => \noneDB\Internal\Durability\Sync::class, + 'shardMeta' => \noneDB\Internal\Sharding\ShardMeta::class, + 'shardRouter' => \noneDB\Internal\Sharding\ShardRouter::class, + 'layoutSwitch' => \noneDB\Internal\Sharding\LayoutSwitch::class, + 'shardInfo' => \noneDB\Internal\Sharding\ShardInfo::class, + 'secondary' => \noneDB\Internal\Index\SecondaryFiles::class, + 'fieldIndex' => \noneDB\Internal\Index\FieldIndex::class, + 'fieldIndexApi' => \noneDB\Internal\Index\FieldIndexApi::class, + 'geoJson' => \noneDB\Internal\Index\Spatial\GeoJson::class, + 'geometry' => \noneDB\Internal\Index\Spatial\Geometry::class, + 'rtree' => \noneDB\Internal\Index\Spatial\RTree::class, + 'spatialIndexApi' => \noneDB\Internal\Index\Spatial\SpatialIndexApi::class, + 'findFilter' => \noneDB\Internal\Query\FindFilter::class, + 'values' => \noneDB\Internal\Query\Values::class, + 'planner' => \noneDB\Internal\Query\Planner::class, + 'queryMethods' => \noneDB\Internal\Query\QueryMethods::class, + 'aggregation' => \noneDB\Internal\Query\Aggregation::class, + 'utility' => \noneDB\Internal\Query\Utility::class, + 'spatialQueries' => \noneDB\Internal\Query\SpatialQueries::class, + 'crud' => \noneDB\Internal\Core\Crud::class, + 'db' => \noneDB\Internal\Core\DatabaseOps::class, + 'integrity' => \noneDB\Internal\Diagnostics\Integrity::class, + 'metrics' => \noneDB\Internal\Diagnostics\Metrics::class, + 'maintenance' => \noneDB\Internal\Maintenance\Maintenance::class, + 'compaction' => \noneDB\Internal\Maintenance\Compaction::class, + 'keyMaintenance' => \noneDB\Internal\Maintenance\KeyMaintenance::class, + 'backupRestore' => \noneDB\Internal\Maintenance\BackupRestore::class, + ]; - private static $devModeNoticeShown = false; + public function __construct($config = null){ + $this->rt = Runtime::get(); + unset($this->config, $this->cache, $this->store, $this->layout, $this->segments, $this->checkpoints, $this->records, $this->writer, $this->buffer, $this->locks, $this->snapshot, $this->scope, $this->recoveryLog, $this->recovery, $this->io, $this->sync, $this->shardMeta, $this->shardRouter, $this->layoutSwitch, $this->shardInfo, $this->secondary, $this->fieldIndex, $this->fieldIndexApi, $this->geoJson, $this->geometry, $this->rtree, $this->spatialIndexApi, $this->findFilter, $this->values, $this->planner, $this->queryMethods, $this->aggregation, $this->utility, $this->spatialQueries, $this->crud, $this->db, $this->integrity, $this->metrics, $this->maintenance, $this->compaction, $this->keyMaintenance, $this->backupRestore); + $this->linkCaches(); + $this->config->initialize($config); + } /** - * Constructor - load config and initialize static caches - * @param array|null $config Optional config array; merged over the .nonedb file when one exists - * @throws \RuntimeException If no configuration can be found outside dev mode - * @throws \InvalidArgumentException If secretKey is missing, empty, not a string or a placeholder + * A service, created on first use: it binds the kernel's values by reference (after linkCaches(), so + * its references follow the links) */ - public function __construct($config = null){ - $this->loadConfig($config); - $this->validateSecretKey(); - if(((int)ini_get('mbstring.func_overload')) & 2){ - // Byte offsets are computed with strlen(); an overloaded strlen() would corrupt them - throw new \RuntimeException('noneDB: mbstring.func_overload must be disabled'); + public function __get(string $name) { + $class = self::SERVICES[$name] ?? null; + if($class === null){ + throw new \LogicException('noneDB: no service ' . $name); } + $service = new $class($this); + $this->$name = $service; + $service->bind(); + return $service; + } + /** + * Link the instance's caches to the process-wide ones (static cache sharing): before any service exists + */ + private function linkCaches(): void { // Link instance caches to static caches for cross-instance sharing - if(self::$staticCacheEnabled){ - $this->indexCache = &self::$staticIndexCache; - $this->shardedCache = &self::$staticShardedCache; - $this->metaCache = &self::$staticMetaCache; - $this->metaCacheTime = &self::$staticMetaCacheTime; - $this->hashCache = &self::$staticHashCache; - $this->jsonlFormatCache = &self::$staticFormatCache; - $this->fieldIndexCache = &self::$staticFieldIndexCache; - $this->spatialIndexCache = &self::$staticSpatialIndexCache; - $this->globalSpatialIndexCache = &self::$staticGlobalSpatialIndexCache; - $this->generations = &self::$staticGenerations; - $this->hashFilesLoaded = &self::$staticHashFilesLoaded; + if($this->rt->staticCacheEnabled){ + $this->indexCache = &$this->rt->staticIndexCache; + $this->verifiedIndexes = &$this->rt->staticVerifiedIndexes; + $this->shardedCache = &$this->rt->staticShardedCache; + $this->metaCache = &$this->rt->staticMetaCache; + $this->hashCache = &$this->rt->staticHashCache; + $this->fieldIndexCache = &$this->rt->staticFieldIndexCache; + $this->spatialIndexCache = &$this->rt->staticSpatialIndexCache; + $this->globalSpatialIndexCache = &$this->rt->staticGlobalSpatialIndexCache; + $this->generations = &$this->rt->staticGenerations; + $this->writableIndexes = &$this->rt->staticWritableIndexes; + $this->dataSizes = &$this->rt->staticDataSizes; + $this->globalSkip = &$this->rt->staticGlobalSkip; + $this->segWrites = &$this->rt->staticSegWrites; + $this->dataEpochs = &$this->rt->staticDataEpochs; + $this->shardFilesUpTo = &$this->rt->staticShardFilesUpTo; + $this->currentGenerations = &$this->rt->staticCurrentGenerations; + $this->formatRecords = &$this->rt->staticFormatRecords; + $this->linked = array_fill_keys(['indexCache', 'verifiedIndexes', 'shardedCache', 'metaCache', 'hashCache', 'fieldIndexCache', 'spatialIndexCache', 'globalSpatialIndexCache', 'generations', 'writableIndexes', 'dataSizes', 'globalSkip', 'segWrites', 'dataEpochs', 'shardFilesUpTo', 'currentGenerations', 'formatRecords'], true); + } + } + + private function unsetServices(): void { + unset($this->config, $this->cache, $this->store, $this->layout, $this->segments, $this->checkpoints, $this->records, $this->writer, $this->buffer, $this->locks, $this->snapshot, $this->scope, $this->recoveryLog, $this->recovery, $this->io, $this->sync, $this->shardMeta, $this->shardRouter, $this->layoutSwitch, $this->shardInfo, $this->secondary, $this->fieldIndex, $this->fieldIndexApi, $this->geoJson, $this->geometry, $this->rtree, $this->spatialIndexApi, $this->findFilter, $this->values, $this->planner, $this->queryMethods, $this->aggregation, $this->utility, $this->spatialQueries, $this->crud, $this->db, $this->integrity, $this->metrics, $this->maintenance, $this->compaction, $this->keyMaintenance, $this->backupRestore); + } + + /** + * A copy for a cloned noneDB instance: its own values (settings, per-instance caches), the same links to + * the process-wide caches, its own services + */ + public function cloneKernel(): self { + $copy = clone $this; + // The services hold the kernel's values by reference, so a plain clone would share every value with + // this kernel. Only the caches linked to the process-wide ones are shared; everything else is copied. + foreach (get_object_vars($this) as $name => $value) { + if ($name === 'rt' || is_object($value) || isset($this->linked[$name])) { + continue; + } + unset($copy->$name); + $copy->$name = $value; } + $copy->unsetServices(); + $copy->config->cloned(); + return $copy; } /** - * Never expose the secret in debug output - * @return array + * Drop the handles between the services and this kernel (the noneDB instance is gone): without the + * cycles the memory is freed at once, not by the cycle collector later. Values bound by reference + * (settings, caches, hooks) are not touched; only handles to engine objects are cleared. */ - public function __debugInfo(){ - return [ - 'dbDir' => $this->dbDir, - 'secretKey' => '***', - 'autoCreateDB' => $this->autoCreateDB, - 'shardingEnabled' => $this->shardingEnabled, - 'shardSize' => $this->shardSize, - 'bufferEnabled' => $this->bufferEnabled, - 'devMode' => $this->devMode, - ]; + public function release(): void { + foreach (self::SERVICES as $name => $class) { + if (isset($this->$name)) { + $this->$name->unbind(); + $this->$name = null; + } + } } /** - * Instances hold a secret and open resources; they can't be serialized - * @throws \LogicException + * The kernel of a noneDB instance */ - public function __serialize(): array { - throw new \LogicException('noneDB instances cannot be serialized'); + public static function of(\noneDB $db): self { + static $id = null; + $id = $id ?? \Closure::bind(function (\noneDB $db) { return $db->id; }, null, \noneDB::class); + return self::$instances[$id($db)]; } +} +} +namespace noneDB\Internal\Cache { - public function __unserialize(array $data): void { - throw new \LogicException('noneDB instances cannot be unserialized'); +use function array_keys, clearstatcache, fclose, file_exists, gc_mem_caches, hash_hmac, ini_get, intdiv, is_array, + memory_get_peak_usage, memory_get_usage, min, strlen, strncmp, strtolower, substr, trim, usort; +use const PHP_INT_MAX; + +/** + * @internal + */ +final class CacheManager { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dataEpochs; + private $dataSizes; + private $fieldIndexCache; + private $globalSkip; + private $globalSpatialIndexCache; + private $indexCache; + private $keyFingerprint; + private $metaCache; + private $segWrites; + private $shardFilesUpTo; + private $shardedCache; + private $spatialIndexCache; + private $verifiedIndexes; + private $writableIndexes; + private $bufferCounts; + private $bufferEnds; + private $cacheGrew; + private $cacheTouch; + private $sessionLimit; + private $sessionNear; + private $lockRegistry; + private $metrics; + private $secondaryChecked; + private $staticCacheEnabled; + private $staticFieldIndexCache; + private $staticFileExistsCache; + private $staticGlobalSpatialIndexCache; + private $staticIndexCache; + private $staticSpatialIndexCache; + private $staticVerifiedIndexes; + private $staticWritableIndexes; + private $tailCache; + private $config; + private $layout; + private $locks; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->config, $this->layout, $this->locks); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->keyFingerprint = &$this->k->keyFingerprint; + $this->metaCache = &$this->k->metaCache; + $this->segWrites = &$this->k->segWrites; + $this->shardFilesUpTo = &$this->k->shardFilesUpTo; + $this->shardedCache = &$this->k->shardedCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->bufferCounts = &$this->rt->bufferCounts; + $this->bufferEnds = &$this->rt->bufferEnds; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->sessionNear = &$this->rt->sessionNear; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->staticFieldIndexCache = &$this->rt->staticFieldIndexCache; + $this->staticFileExistsCache = &$this->rt->staticFileExistsCache; + $this->staticGlobalSpatialIndexCache = &$this->rt->staticGlobalSpatialIndexCache; + $this->staticIndexCache = &$this->rt->staticIndexCache; + $this->staticSpatialIndexCache = &$this->rt->staticSpatialIndexCache; + $this->staticVerifiedIndexes = &$this->rt->staticVerifiedIndexes; + $this->staticWritableIndexes = &$this->rt->staticWritableIndexes; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->config = null; + $this->layout = null; + $this->locks = null; + $this->k = null; + $this->rt = null; } - public function __clone(){ - $source = self::$secretKeys[$this->cloneSourceId] ?? null; - $this->cloneSourceId = spl_object_id($this); - if($source !== null){ - self::$secretKeys[$this->cloneSourceId] = $source; + /** + * HMAC fingerprint of the secret; binds caches to the key without revealing it + */ + public function getKeyFingerprint(){ + if($this->keyFingerprint === null){ + $this->keyFingerprint = substr(hash_hmac('sha256', 'nonedb4/cache-key', (string)$this->config->getSecretKey()), 0, 16); } + return $this->keyFingerprint; } - private $cloneSourceId = null; - - public function __destruct(){ - // Anything still dirty belongs to a lock we no longer hold; it was flushed on release. - unset(self::$secretKeys[spl_object_id($this)]); + /** + * Clear all static caches (useful for testing or memory management) + * @return void + */ + public static function clearStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticVerifiedIndexes = []; + \noneDB\Internal\Cache\Runtime::get()->uncheckpointed = []; + \noneDB\Internal\Cache\Runtime::get()->decodeDue = []; + \noneDB\Internal\Cache\Runtime::get()->staticShardedCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticMetaCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticHashCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticFileExistsCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticSanitizeCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticFieldIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticSpatialIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticGlobalSpatialIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticGenerations = []; + \noneDB\Internal\Cache\Runtime::get()->staticCurrentGenerations = []; + \noneDB\Internal\Cache\Runtime::get()->staticFormatRecords = []; + \noneDB\Internal\Cache\Runtime::get()->storeState = []; + \noneDB\Internal\Cache\Runtime::get()->parsedLockStates = []; + \noneDB\Internal\Cache\Runtime::get()->bufferCounts = []; + \noneDB\Internal\Cache\Runtime::get()->bufferEnds = []; + \noneDB\Internal\Cache\Runtime::get()->staticWritableIndexes = []; + \noneDB\Internal\Cache\Runtime::get()->staticDataSizes = []; + \noneDB\Internal\Cache\Runtime::get()->staticGlobalSkip = []; + \noneDB\Internal\Cache\Runtime::get()->staticSegWrites = []; + \noneDB\Internal\Cache\Runtime::get()->staticDataEpochs = []; + \noneDB\Internal\Cache\Runtime::get()->staticShardFilesUpTo = []; + \noneDB\Internal\Cache\Runtime::get()->secondaryChecked = []; + \noneDB\Internal\Cache\Runtime::get()->cacheTouch = []; + \noneDB\Internal\Cache\Runtime::get()->tailCache = []; + foreach(\noneDB\Internal\Cache\Runtime::get()->lockHandles as $handle){ + @fclose($handle['fp']); + } + \noneDB\Internal\Cache\Runtime::get()->lockHandles = []; } - private function getSecretKey(){ - return self::$secretKeys[spl_object_id($this)] ?? null; + /** + * Disable static caching (each instance uses its own cache) + * @return void + */ + public static function disableStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticCacheEnabled = false; } - private function setSecretKey($key){ - self::$secretKeys[spl_object_id($this)] = $key; - $this->cloneSourceId = spl_object_id($this); - $this->keyFingerprint = null; - $this->dbBases = []; + /** + * Enable static caching (default) + * @return void + */ + public static function enableStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticCacheEnabled = true; } /** - * Load configuration. - * - * The .nonedb file is searched in: the entry script's directory, noneDB.php's directory, - * the Composer project root (when installed under vendor/) and the current working directory. - * A config array passed to the constructor is merged over the file (file is optional when the - * array already contains secretKey and dbDir). + * Cached file_exists check. Entries are dropped whenever another process writes the database + * (see withLock), so the cache can't go stale across processes. * - * @param array|null $config Optional config array - * @throws \RuntimeException If config is missing and not in dev mode + * @param string $path File path to check + * @return bool True if file exists */ - private function loadConfig($config = null){ - $arrayConfig = is_array($config) ? $config : []; - // A secretKey given in the array is used (and validated) as given, also null/false/'': - // never replaced by the file's key or the dev mode key - $keyGiven = array_key_exists('secretKey', $arrayConfig); - $arrayIsComplete = $keyGiven && isset($arrayConfig['dbDir']); - - // Load (and cache) the file config once per process - if(!self::$configLoaded){ - self::$configData = null; - self::$configFilePath = null; - self::$configFileDir = null; - - $configPath = self::findConfigFile(); - if($configPath !== null){ - $content = @file_get_contents($configPath); - if($content === false){ - if(!$arrayIsComplete){ - throw new \RuntimeException("noneDB: Cannot read config file: {$configPath}"); - } - } else { - $data = json_decode($content, true); - if(!is_array($data)){ - if(!$arrayIsComplete){ - throw new \RuntimeException("noneDB: Invalid JSON in config file: {$configPath}"); - } - } else { - self::$configData = $data; - self::$configFilePath = $configPath; - self::$configFileDir = dirname($configPath); - } - } - } - self::$configLoaded = true; - } - - if(self::$configData !== null){ - $fileConfig = self::$configData; - if(isset($arrayConfig['dbDir'])){ - unset($fileConfig['dbDir']); // overridden below; no need to resolve the file's one - } - $this->applyConfig($fileConfig, self::$configFileDir, true); - } - if(!empty($arrayConfig)){ - $this->applyConfig($arrayConfig, null, false); + public function cachedFileExists($path){ + if(!$this->staticCacheEnabled){ + clearstatcache(true, $path); + return file_exists($path); } - if(self::$configData !== null || $arrayIsComplete || ($keyGiven && $this->dbDir !== null)){ - return; + if(isset($this->staticFileExistsCache[$path])){ + return $this->staticFileExistsCache[$path]; } + clearstatcache(true, $path); + $exists = file_exists($path); + $this->staticFileExistsCache[$path] = $exists; + return $exists; + } - if(self::isDevModeEnabled()){ - $this->devMode = true; - $projectRoot = self::projectRoot(); - if($this->dbDir === null){ - $this->dbDir = $projectRoot . '/db/'; - } - if(!$keyGiven){ - // Deterministic for backward compatibility with databases created in dev mode before v3.2.0. - $this->setSecretKey('nonedb_dev_mode_key_' . md5(__DIR__)); - } - if(PHP_SAPI !== 'cli' && !self::$devModeNoticeShown){ - self::$devModeNoticeShown = true; - trigger_error('noneDB: development mode is active (predictable secretKey). Never use it in production.', E_USER_NOTICE); - } - return; + /** + * Mark file as existing in cache (call after creating file) + * @param string $path File path + */ + public function markFileExists($path){ + if($this->staticCacheEnabled){ + $this->staticFileExistsCache[$path] = true; } - - throw new \RuntimeException( - "noneDB: Configuration file not found!\n" . - "Create a '.nonedb' config file in your project root.\n" . - "See '.nonedb.example' for reference.\n" . - "For development, set NONEDB_DEV_MODE=true environment variable or define('NONEDB_DEV_MODE', true);" - ); } /** - * Candidate config file locations, in priority order - * @return array + * Mark file as not existing in cache (call after deleting file) + * @param string $path File path */ - private static function configSearchPaths(): array { - $dirs = [self::scriptDir(), __DIR__]; - $root = self::projectRoot(); - if($root !== __DIR__){ - $dirs[] = $root; + public function markFileNotExists($path){ + if($this->staticCacheEnabled){ + $this->staticFileExistsCache[$path] = false; } - $cwd = getcwd(); - if($cwd !== false){ - $dirs[] = $cwd; - } - $paths = []; - foreach(array_unique($dirs) as $dir){ - $paths[] = rtrim($dir, '/\\') . '/' . self::$configFile; - } - return $paths; } - private static function findConfigFile(): ?string { - foreach(self::configSearchPaths() as $path){ - if(is_file($path)){ - return $path; - } + /** + * The cache limit in bytes, 0 = none: the cacheMemoryLimit setting, or by default half of PHP's + * memory_limit (no limit without one): the cache doesn't make a process run out of + * memory, and doesn't slow down one that may use all the memory it needs + */ + public static function cacheLimitBytes(){ + $rt = \noneDB\Internal\Cache\Runtime::get(); + return $rt->cacheMemoryLimit ?? self::memoryLimitHalf(); + } + + /** + * Half of PHP's memory_limit in bytes, 0 without one + */ + private static function memoryLimitHalf(){ + $rt = \noneDB\Internal\Cache\Runtime::get(); + $setting = (string)ini_get('memory_limit'); + if($setting !== $rt->memoryLimitSetting){ + // Worked out once per value of memory_limit: this runs between the segments of every operation + $ini = strtolower(trim($setting)); + $bytes = (int)$ini; + $unit = substr($ini, -1); + $bytes *= $unit === 'g' ? 1073741824 : ($unit === 'm' ? 1048576 : ($unit === 'k' ? 1024 : 1)); + $rt->memoryLimitSetting = $setting; + $rt->memoryLimitHalf = $bytes <= 0 ? 0 : intdiv($bytes, 2); } - return null; + return $rt->memoryLimitHalf; } /** - * Absolute directory of the entry script (falls back to the working directory) + * Keep the process under cacheMemoryLimit by dropping cached indexes: the process's memory + * (memory_get_usage()) is what counts, the caches are what can be dropped. Runs when the last lock of the + * process is released and between the segments of an operation (evictBetweenSegments()); while the process + * uses less memory than the limit it does nothing. Least recently used entries go first, until the process + * is under the limit (three quarters of it between segments) or nothing is left to drop. Every evicted + * entry is loaded again when needed. A global index a write session changed stays: it is written when the + * session ends (FieldIndex::deferGlobalWrite()). + * @param bool $within Between the segments of an operation (evictBetweenSegments()) */ - private static function scriptDir(): string { - $script = $_SERVER['SCRIPT_FILENAME'] ?? ''; - $cwd = getcwd(); - // A relative SCRIPT_FILENAME depends on the working directory: both are part of the memo key - $memoKey = $script . "\0" . $cwd; - if(isset(self::$scriptDirs[$memoKey])){ - return self::$scriptDirs[$memoKey]; + public function evictCaches($within = false){ + $limit = self::cacheLimitBytes(); + $target = $within ? intdiv(3 * $limit, 4) : $limit; + if(memory_get_usage() <= $target){ + return; } - $dir = null; - if($script !== ''){ - $real = realpath($script); - if($real !== false){ - $dir = dirname($real); - } else { - $realDir = realpath(dirname($script)); - if($realDir !== false){ - $dir = $realDir; + $pending = []; + foreach($this->lockRegistry as $entry){ + $pending += $entry['globals']; + } + $entries = []; + foreach(['staticIndexCache', 'staticFieldIndexCache', 'staticSpatialIndexCache', 'staticGlobalSpatialIndexCache'] as $cache => $property){ + foreach(array_keys($this->$property) as $key){ + if(!isset($pending[$key])){ + $entries[] = [$this->cacheTouch[$key] ?? 0, $cache, $key]; } } } - if($dir === null){ - $dir = $cwd !== false ? $cwd : __DIR__; - } - if(count(self::$scriptDirs) > 16){ - self::$scriptDirs = []; + usort($entries, function($a, $b){ return $a[0] <=> $b[0]; }); + foreach($entries as [, $cache, $key]){ + if($cache === 0){ + unset($this->staticIndexCache[$key], $this->staticVerifiedIndexes[$key], $this->staticWritableIndexes[$key]); + } elseif($cache === 1){ + unset($this->staticFieldIndexCache[$key]); + } elseif($cache === 2){ + unset($this->staticSpatialIndexCache[$key]); + } else { + unset($this->staticGlobalSpatialIndexCache[$key]); + } + unset($this->cacheTouch[$key]); + $this->metrics['evictions']++; + if(memory_get_usage() <= $target){ + break; + } } - return self::$scriptDirs[$memoKey] = $dir; } /** - * Project root: the directory that contains vendor/ when installed with Composer, else noneDB.php's directory + * Bytes read from an index file, and the 4 KiB pages they are (getMetrics(), docs/OPERATIONS.md §1.1) */ - private static function projectRoot(): string { - $dir = str_replace('\\', '/', __DIR__); - $pos = strpos($dir, '/vendor/'); - if($pos !== false){ - return substr(__DIR__, 0, $pos); - } - return __DIR__; + public function noteIndexRead($bytes, $pages = null){ + $this->metrics['bytes'] += $bytes; + $this->metrics['pages'] += $pages ?? intdiv($bytes + 4095, 4096); } - private static function isDevModeEnabled(): bool { - return getenv('NONEDB_DEV_MODE') === 'true' - || getenv('NONEDB_DEV_MODE') === '1' - || (defined('NONEDB_DEV_MODE') && NONEDB_DEV_MODE === true); + /** + * True when an index may be decoded into a larger in-memory form of about $bytes: no limit, or the + * process stays under half of cacheMemoryLimit with it (room left before eviction starts) + */ + public static function decodeFits($bytes){ + $limit = self::cacheLimitBytes(); + return $limit <= 0 || memory_get_usage() + $bytes <= intdiv($limit, 2); } - private static function isAbsolutePath(string $path): bool { - return $path !== '' && ($path[0] === '/' || $path[0] === '\\' || preg_match('/^[A-Za-z]:[\\\\\/]/', $path) === 1); - } + /** Memory of the process after the last eviction between segments */ + private $betweenFloor = 0; + + /** Memory PHP held for the process after it */ + private $heldFloor = 0; /** - * Apply configuration values to instance properties - * @param array $config Configuration array - * @param string|null $baseDir Directory that relative dbDir values resolve against (null: the entry script's) - * @param bool $fromFile True when the values come from the .nonedb file + * Between the segments of an operation over many segments (a read, an aggregation, a write by filter, an + * index build, compaction), once a segment is done: when the process uses more memory than cacheMemoryLimit, + * least recently used entries go until the estimate is under three quarters of the limit; the next eviction waits + * until the process has grown by another quarter (counted from where it is now if it has freed memory since). A + * process whose peak came within an eighth of memory_limit (what PHP held for it, what memory_limit counts: a + * segment's batches leave more of it behind than the process uses) measures that too, and gives back what it + * freed. Not sooner: a warm query loses what it would load again. One operation over a large database holds the + * cache and the segment it works on (the segments it finished go first), without a scan of the cache at every + * segment. Under the exclusive lock too: what a write session changed is in its files, except the + * global indexes it defers (they stay); a segment index it changed that goes is checkpointed by the + * next reader that loads it (Checkpoints::checkpointTouchedSegments()). */ - private function applyConfig(array $config, ?string $baseDir, bool $fromFile){ - if(array_key_exists('secretKey', $config)){ - $this->setSecretKey($config['secretKey']); - } - if(isset($config['dbDir'])){ - $this->dbDir = $this->resolveDbDir((string)$config['dbDir'], $baseDir, $fromFile); - } - if(isset($config['autoCreateDB'])){ - $this->autoCreateDB = (bool)$config['autoCreateDB']; + public function evictBetweenSegments(){ + if(!$this->staticCacheEnabled){ + return; } - - // Sharding settings - if(isset($config['shardingEnabled'])){ - $this->shardingEnabled = (bool)$config['shardingEnabled']; + $limit = $this->sessionLimit ?? ($this->sessionLimit = self::cacheLimitBytes()); + if($limit <= 0){ + return; } - if(isset($config['shardSize'])){ - $shardSize = (int)$config['shardSize']; - if($shardSize < 1){ - throw new \InvalidArgumentException("noneDB: 'shardSize' must be a positive integer"); - } - $this->shardSize = $shardSize; + $used = memory_get_usage(); + $this->betweenFloor = min($this->betweenFloor, $used); + $quarter = intdiv($limit, 4); + $over = $used > $limit && $used > $this->betweenFloor + $quarter; + $near = memory_get_peak_usage(true) > ($this->sessionNear ?? ($this->sessionNear = self::nearMemoryLimit())); + if($near){ + $held = memory_get_usage(true); + $this->heldFloor = min($this->heldFloor, $held); + $over = $over || ($held > $limit && $held > $this->heldFloor + $quarter); + } + if(!$over){ + return; } - if(isset($config['autoMigrate'])){ - $this->autoMigrate = (bool)$config['autoMigrate']; + $this->cacheGrew = false; + $this->evictCaches(true); + $this->betweenFloor = memory_get_usage(); + if($near){ + gc_mem_caches(); + $this->heldFloor = memory_get_usage(true); } + } - // Compaction settings - if(isset($config['autoCompactThreshold'])){ - $threshold = (float)$config['autoCompactThreshold']; - if($threshold <= 0){ - throw new \InvalidArgumentException("noneDB: 'autoCompactThreshold' must be greater than 0"); - } - $this->jsonlGarbageThreshold = $threshold; - } + /** + * Where a process's peak is near memory_limit: seven eighths of it. Never without one + */ + private static function nearMemoryLimit(){ + $half = self::memoryLimitHalf(); + return $half > 0 ? intdiv(7 * $half, 4) : PHP_INT_MAX; + } - // Lock settings - if(isset($config['lockTimeout'])){ - $timeout = (int)$config['lockTimeout']; - if($timeout < 0){ - throw new \InvalidArgumentException("noneDB: 'lockTimeout' must be 0 (wait forever) or a positive number of seconds"); + /** + * Drop every cached entry that belongs to a database (index, meta, field/spatial indexes, file-exists) + */ + public function invalidateDbCaches($dbname){ + $prefix = $this->layout->dbHome($dbname); + $len = strlen($prefix); + $caches = [ + &$this->indexCache, &$this->verifiedIndexes, &$this->shardedCache, &$this->metaCache, + &$this->fieldIndexCache, &$this->spatialIndexCache, + &$this->globalSpatialIndexCache, &$this->dataSizes, &$this->dataEpochs, &$this->shardFilesUpTo, + &$this->secondaryChecked, &$this->tailCache, + ]; + foreach($caches as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($cache[$key]); + } } - $this->lockTimeout = $timeout; } - if(isset($config['lockRetryDelay'])){ - $delay = (int)$config['lockRetryDelay']; - if($delay < 1){ - throw new \InvalidArgumentException("noneDB: 'lockRetryDelay' must be a positive number of microseconds"); + unset($cache); + foreach(array_keys($this->staticFileExistsCache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->staticFileExistsCache[$key]); } - $this->lockRetryDelay = $delay; - } - - // Write buffer - if(isset($config['bufferEnabled'])){ - $this->bufferEnabled = (bool)$config['bufferEnabled']; } - - // Use indexes for queries (indexes are always kept up to date) - if(isset($config['fieldIndexEnabled'])){ - $this->fieldIndexEnabled = (bool)$config['fieldIndexEnabled']; + $lockPath = $this->locks->getLockPath($dbname); + unset($this->globalSkip[$lockPath], $this->segWrites[$lockPath]); + foreach(array_keys($this->bufferCounts) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->bufferCounts[$key], $this->bufferEnds[$key]); + } } - if(isset($config['spatialIndexEnabled'])){ - $this->spatialIndexEnabled = (bool)$config['spatialIndexEnabled']; + foreach(array_keys($this->writableIndexes) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->writableIndexes[$key]); + } } } /** - * Resolve a configured dbDir to an absolute path ending with a slash. - * Relative paths (with or without a leading ./) are relative to the config file's directory - * (file config) or to the entry script's directory (array config). + * Caches of a database after another process wrote it without replacing a data file (same epoch): + * everything is dropped but the key indexes, which are checked again (the data only grew: the new lines + * are replayed on the next read) */ - private function resolveDbDir(string $dbDir, ?string $baseDir, bool $fromFile): string { - if(!self::isAbsolutePath($dbDir)){ - $dotted = substr($dbDir, 0, 2) === './' || substr($dbDir, 0, 3) === '../' - || $dbDir === '.' || $dbDir === '..'; - $relative = preg_replace('#^(\./)+#', '', $dbDir); - $baseDir = $baseDir ?? self::scriptDir(); - $resolved = rtrim($baseDir, '/\\') . '/' . $relative; - - // Before v3.2.0 "./x" in the file was resolved against the entry script and a plain "x" - // against the working directory. Keep using such a directory (with a deprecation notice) - // until the data is moved. - $legacyBase = is_dir($resolved) ? null : ($dotted ? ($fromFile ? self::scriptDir() : null) : getcwd()); - if(is_string($legacyBase)){ - $legacy = rtrim($legacyBase, '/\\') . '/' . $relative; - if(is_dir($legacy) && realpath($legacy) !== realpath($resolved)){ - trigger_error( - "noneDB: dbDir '{$dbDir}' is now resolved relative to " . ($fromFile ? 'the .nonedb file' : 'the entry script') - . " ({$baseDir}); using the old location {$legacy} for now. Move the directory or use an absolute path.", - E_USER_DEPRECATED - ); - $resolved = $legacy; + public function refreshDbCaches($dbname){ + $prefix = $this->layout->dbHome($dbname); + $len = strlen($prefix); + // Kept (they describe data bytes that didn't change, the new lines are replayed when used): key + // indexes, field indexes and R-trees, what was checked against the data, parsed tails, epochs + $keep = []; + foreach(['indexCache' => &$this->indexCache, 'fieldIndexCache' => &$this->fieldIndexCache, + 'spatialIndexCache' => &$this->spatialIndexCache, 'dataEpochs' => &$this->dataEpochs, + 'secondaryChecked' => &$this->secondaryChecked, 'tailCache' => &$this->tailCache] as $name => &$cache){ + foreach($cache as $key => $value){ + if(strncmp((string)$key, $prefix, $len) === 0 + && ($name !== 'fieldIndexCache' && $name !== 'spatialIndexCache' || (is_array($value) && isset($value['kind'])))){ + $keep[$name][$key] = $value; } } - $dbDir = $resolved; } - - if(substr($dbDir, -1) !== '/' && substr($dbDir, -1) !== '\\'){ - $dbDir .= '/'; + unset($cache); + $this->invalidateDbCaches($dbname); + foreach($keep['indexCache'] ?? [] as $key => $value){ + $this->indexCache[$key] = $value; } - return $dbDir; - } - - /** - * Reject missing, empty, non-string and placeholder secret keys - * @throws \InvalidArgumentException - */ - private function validateSecretKey(){ - $key = $this->getSecretKey(); - if(!is_string($key) || trim($key) === ''){ - throw new \InvalidArgumentException( - "noneDB: 'secretKey' must be a non-empty string. Set it in your .nonedb file or constructor config " - . "(generate one with: php -r \"echo bin2hex(random_bytes(32));\")." - ); + foreach($keep['fieldIndexCache'] ?? [] as $key => $value){ + $this->fieldIndexCache[$key] = $value; } - if(in_array($key, self::$placeholderKeys, true)){ - throw new \InvalidArgumentException( - "noneDB: 'secretKey' is still the example placeholder. Replace it with a random secret " - . "(php -r \"echo bin2hex(random_bytes(32));\"). If you already have data, see noneDB::migrateKey()." - ); + foreach($keep['spatialIndexCache'] ?? [] as $key => $value){ + $this->spatialIndexCache[$key] = $value; } - if($this->dbDir === null){ - throw new \RuntimeException("noneDB: 'dbDir' is not configured"); + foreach($keep['dataEpochs'] ?? [] as $key => $value){ + $this->dataEpochs[$key] = $value; + } + foreach($keep['secondaryChecked'] ?? [] as $key => $value){ + $this->secondaryChecked[$key] = $value; + } + foreach($keep['tailCache'] ?? [] as $key => $value){ + $this->tailCache[$key] = $value; } } +} +} +namespace noneDB\Internal\Concurrency { - /** - * Check if config file exists - * @return bool - */ - public static function configExists(): bool { - return self::findConfigFile() !== null; - } - - /** - * Get the config file template path - * @return string|null - */ - public static function getConfigTemplate(): ?string { - $templatePath = __DIR__ . '/.nonedb.example'; - return file_exists($templatePath) ? $templatePath : null; - } - - /** - * Clear config cache (useful for testing) - * @return void - */ - public static function clearConfigCache(): void { - self::$configLoaded = false; - self::$configData = null; - self::$configFilePath = null; - self::$configFileDir = null; - } - - /** - * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) - * @return bool - */ - public static function isDevMode(): bool { - return self::isDevModeEnabled(); - } +use function array_keys, array_shift, basename, bin2hex, clearstatcache, count, dirname, explode, fclose, + fflush, flock, fopen, fread, fseek, fstat, fwrite, getmypid, implode, in_array, is_dir, is_file, is_link, key, max, + microtime, mkdir, preg_match, random_bytes, reset, stat, str_pad, strlen, strncmp, strpos, substr, trim, + usleep; +use const LOCK_EX, LOCK_NB, LOCK_SH, LOCK_UN; - /** - * Set development mode programmatically - * Useful for testing or when env vars are not available - * @param bool $enabled - * @return void - */ - public static function setDevMode(bool $enabled): void { - if($enabled && !defined('NONEDB_DEV_MODE')){ - define('NONEDB_DEV_MODE', true); - } +/** + * @internal + */ +final class LockManager { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dataEpochs; + private $dataSizes; + private $dbDir; + private $fieldIndexCache; + private $formatRecords; + private $durable; + private $generations; + private $globalSkip; + private $globalSpatialIndexCache; + private $lockPaths; + private $lockRetryDelay; + private $lockTimeout; + private $segWrites; + private $cacheGrew; + private $decodeDue; + private $lockHandles; + private $lockRegistry; + private $maintenanceDepth; + private $metrics; + private $multiLock; + private $noWrites; + private $parsedLockStates; + private $secondaryChecked; + private $sessionCounter; + private $sessionLimit; + private $sessionNear; + private $staticCacheEnabled; + private $storeLocks; + private $storeState; + private $tailCache; + private $buffer; + private $cache; + private $checkpoints; + private $fieldIndex; + private $io; + private $layout; + private $recovery; + private $rtree; + private $scope; + private $store; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->recovery, $this->rtree, $this->scope, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->dbDir = &$this->k->dbDir; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->formatRecords = &$this->k->formatRecords; + $this->durable = &$this->k->durable; + $this->generations = &$this->k->generations; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->lockPaths = &$this->k->lockPaths; + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->segWrites = &$this->k->segWrites; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->decodeDue = &$this->rt->decodeDue; + $this->lockHandles = &$this->rt->lockHandles; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->parsedLockStates = &$this->rt->parsedLockStates; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->sessionNear = &$this->rt->sessionNear; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->storeLocks = &$this->rt->storeLocks; + $this->storeState = &$this->rt->storeState; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->recovery = null; + $this->rtree = null; + $this->scope = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + // Rollback scope. The outermost exclusive withLock() call of a database opens one scope; nested + // exclusive calls on the same database join it. There are no nested scopes: cutting a file back to an + // earlier size can't drop an earlier outer append and keep a later inner one. Every append to a data or + // buffer file records the size the file had before the scope's first append to it (a "mark"). When an + // exception leaves the outermost call, every marked file is cut back to its mark. + // Work that must not depend on the caller's success (buffer flush, the switch to shards) runs in place only + // while the scope has no appends yet, committing its own appends step by step (runIndependent); + // requested later, it is deferred until the scope committed, like compaction, which replaces whole files. + // A deferred job that fails doesn't fail the operation that committed before it: it is logged and runs + // again on a later operation. + + public function getLockPath($dbname){ + if(!isset($this->lockPaths[$dbname])){ + $this->layout->dbHome($dbname); + } + return $this->lockPaths[$dbname]; } /** - * Re-key existing databases: rename every file created with $oldKey to the name $newKey produces. + * Run $fn while holding the database lock: shared for reads, exclusive for writes. * - * File names are {hash}-{dbname}... where hash depends on the secretKey, so changing the key - * without this makes existing data invisible. Run it once, with no other process using the - * databases, then update secretKey in your config. Each database is locked while it is renamed. - * The old per-key hash cache and the pre-3.2 `.nonedb_hash_cache` file are removed. + * The lock is process-wide and re-entrant. A nested exclusive request upgrades a shared lock. * - * @param string $dbDir Database directory - * @param string $oldKey Key the databases were created with (the old placeholder is accepted here) - * @param string $newKey New secret key - * @return array ['success' => bool, 'databases' => string[], 'files' => int, 'error' => ?string] + * @param string $dbname Database name (already normalized) + * @param bool $exclusive + * @param callable $fn + * @param string $op The operation an exclusive call is (getDurability()['lastOperation']) + * @return mixed Whatever $fn returns */ - public static function migrateKey(string $dbDir, string $oldKey, string $newKey): array { - $result = ['success' => false, 'databases' => [], 'files' => 0, 'error' => null]; - if(trim($oldKey) === '' || trim($newKey) === ''){ - $result['error'] = 'Both keys must be non-empty strings'; - return $result; - } - if(in_array($newKey, self::$placeholderKeys, true)){ - $result['error'] = 'The new key is a placeholder; generate a random one'; - return $result; - } - if($oldKey === $newKey){ - $result['error'] = 'The new key is the same as the old key'; - return $result; - } - $dbDir = rtrim($dbDir, '/\\') . '/'; - if(!is_dir($dbDir)){ - $result['error'] = "Directory not found: {$dbDir}"; - return $result; + public function withLock($dbname, $exclusive, callable $fn, $op = 'write'){ + $path = $this->getLockPath($dbname); + if($exclusive && $this->multiLock > 0){ + // A snapshot holds shared locks of several databases: waiting for an exclusive one here could + // deadlock with another snapshot. withSnapshot() does that work alone first and tries again. + throw new \RuntimeException("noneDB: database '{$dbname}' needs maintenance before the snapshot", self::SNAPSHOT_RETRY); + } + + if(isset($this->lockRegistry[$path])){ + if(!empty($this->lockRegistry[$path]['lost'])){ + throw new \RuntimeException("noneDB: the lock on database '{$dbname}' was lost (timed out while upgrading)", \noneDB::LOCK_FAILED); + } + if($exclusive && $this->lockRegistry[$path]['mode'] !== LOCK_EX){ + // flock can't upgrade atomically. On BSD/macOS a failed non-blocking upgrade keeps the shared + // lock, so two processes upgrading at once would wait for each other until lockTimeout: release + // first, then wait for the exclusive lock. Another writer may run in between, so the state is + // read again and whatever it left half-done is recovered. + $this->flushPendingGlobals($dbname, $path); // while what they describe still holds + flock($this->lockRegistry[$path]['fp'], LOCK_UN); + try { + $this->acquireFlock($this->lockRegistry[$path]['fp'], LOCK_EX, $dbname); + } catch(\Throwable $e){ + $this->lockRegistry[$path]['lost'] = true; // never continue reading without a lock + throw $e; + } + $this->lockRegistry[$path]['mode'] = LOCK_EX; + $this->loadLockState($dbname, $path); + $this->prepareDb($dbname, $path); + } + $this->lockRegistry[$path]['depth']++; + try { + if($exclusive && $this->lockRegistry[$path]['scope'] === null){ + return $this->scope->runWriteScope($dbname, $path, $fn, $op); + } + return $fn(); + } finally { + $this->lockRegistry[$path]['depth']--; + } } - // Databases created with the old key - $databases = []; - foreach(scandir($dbDir) ?: [] as $file){ - if(preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+)\\.nonedb(info|\\.meta)?$/", $file, $m) - && hash_pbkdf2('sha256', $m[2], $oldKey, 1000, 20) === $m[1]){ - $databases[$m[2]] = $m[1]; + // Outermost call. A database this process doesn't know to be present takes the missing-database + // path (STORAGE_FORMAT.md §3.3): store re-validated, dbDir listed for foreign entries, STORE.LOCK + // held shared, so no maintenance can run while it is created or answered as missing. + if(!isset($this->formatRecords[$path]) && !isset($this->storeLocks[$this->dbDir])){ + $this->store->openStore(false); + clearstatcache(true, dirname($path) . '/FORMAT'); + if(!is_file(dirname($path) . '/FORMAT')){ + return $this->withMissingDatabase($dbname, $exclusive, $fn, $op); + } + } + try { + return $this->lockSession($dbname, $exclusive, $fn, !isset($this->storeLocks[$this->dbDir]), $op); + } catch(\RuntimeException $e){ + if($e->getCode() !== self::DB_GONE && $e->getCode() !== self::DB_ABSENT){ + throw $e; } + return $this->withMissingDatabase($dbname, $exclusive, $fn, $op); } + } - // Plan every rename first, refuse on any conflict - $plan = []; - $files = scandir($dbDir) ?: []; - foreach($databases as $dbname => $oldHash){ - $newHash = hash_pbkdf2('sha256', $dbname, $newKey, 1000, 20); - foreach($files as $file){ - // {hash}-{name}.*, {hash}-{name}_sN.*, and spatial files {hash}.nonedb.* / {hash}_sN.nonedb.* - $named = strpos($file, $oldHash . '-' . $dbname . '.') === 0 || strpos($file, $oldHash . '-' . $dbname . '_s') === 0; - $spatial = strpos($file, $oldHash . '.nonedb.') === 0 || preg_match('/^' . $oldHash . '_s\d+\.nonedb\./', $file) === 1; - if(!$named && !$spatial){ - continue; + /** + * The missing-database path: under STORE.LOCK (shared) after the store was re-read and dbDir checked + * for foreign entries. A read of a database without a directory runs $fn without any lock (nothing is + * created); a write creates the directory and locks it. + */ + private function withMissingDatabase($dbname, $exclusive, callable $fn, $op){ + $path = $this->getLockPath($dbname); + return $this->store->withStoreRevalidated($exclusive, function() use ($dbname, $exclusive, $fn, $path, $op){ + clearstatcache(true, dirname($path)); + if(!is_dir(dirname($path))){ + if(!$exclusive || !$this->store->storeExists()){ + return $fn(); // absent: reads create nothing } - $target = $newHash . substr($file, strlen($oldHash)); - if(file_exists($dbDir . $target)){ - $result['error'] = "Target file already exists: {$target}"; - return $result; + if(!@mkdir(dirname($path), 0750) && !is_dir(dirname($path))){ + throw new \RuntimeException("noneDB: cannot create the directory of database '{$dbname}'"); } - $plan[$dbname][] = [$file, $target]; + $this->io->setPermissions(dirname($path), true); } - } + return $this->lockSession($dbname, $exclusive, $fn, false, $op); + }); + } - foreach($plan as $dbname => $renames){ - $oldHash = $databases[$dbname]; - $lockPath = $dbDir . $oldHash . '-' . $dbname . '.nonedb.lock'; - // Databases from before v3.2 have no lock file: the one opened here is removed afterwards - clearstatcache(true, $lockPath); - $hadLock = file_exists($lockPath); - $fp = @fopen($lockPath, 'c+b'); - if($fp === false || !flock($fp, LOCK_EX)){ - if($fp !== false){ - fclose($fp); - } - if(!$hadLock){ - @unlink($lockPath); + /** + * One lock session of a database (outermost withLock call). $recheck: the session didn't come through + * the missing-database path, so finding the database absent means going there (DB_ABSENT). + */ + private function lockSession($dbname, $exclusive, callable $fn, $recheck, $op){ + $path = $this->getLockPath($dbname); + $mode = $exclusive ? LOCK_EX : LOCK_SH; + $fp = $this->lockFile($path, $mode, $dbname); + $this->sessionLimit = null; + $this->sessionNear = null; + $this->lockRegistry[$path] = [ + 'fp' => $fp, 'mode' => $mode, 'depth' => 1, 'dbname' => $dbname, + 'gen' => 0, 'epoch' => null, 'flags' => [], 'bumped' => false, 'pending' => [], + 'scope' => null, 'rolledBack' => false, 'sid' => ++$this->sessionCounter, 'globals' => [], 'wrote' => false, + ]; + + $ok = false; + try { + $this->loadLockState($dbname, $path); + try { + $this->prepareDb($dbname, $path); + } catch(\RuntimeException $e){ + // restore() is the way out of a damaged database (STORAGE_FORMAT.md §10.3 step 2): what recovery + // can't resolve is handed to it instead of failing + if($op !== 'restore' || $e->getCode() !== \noneDB::FORMAT_DAMAGED){ + throw $e; } - $result['error'] = "Could not lock database '{$dbname}'"; - return $result; } + if(isset($this->decodeDue[$path]) && $op !== 'rebuildIndex'){ + // §6.4: loaded by an earlier operation, decoded now (not by one that makes them again) + $this->checkpoints->decodeDueIndexes($path); + } + if($recheck && !isset($this->formatRecords[$path]) && !$this->layout->dbPresent($dbname)){ // a cached FORMAT: present + throw new \RuntimeException("noneDB: database '{$dbname}' is absent", self::DB_ABSENT); + } + $result = $exclusive ? $this->scope->runWriteScope($dbname, $path, $fn, $op) : $fn(); + $ok = true; + return $result; + } finally { try { - // Rename the lock file last so writers keep blocking on it until we are done - usort($renames, function($a, $b){ - return (substr($a[0], -12) === '.nonedb.lock') <=> (substr($b[0], -12) === '.nonedb.lock'); - }); - foreach($renames as $pair){ - if(!@rename($dbDir . $pair[0], $dbDir . $pair[1])){ - $result['error'] = "Failed to rename {$pair[0]}"; - return $result; - } - $result['files']++; + if($this->lockRegistry[$path]['mode'] === LOCK_EX){ + $this->finishWriteSession($dbname, $path, $ok); } + $this->flushPendingGlobals($dbname, $path, $ok); } finally { + $fp = $this->lockRegistry[$path]['fp']; + unset($this->lockRegistry[$path]); flock($fp, LOCK_UN); - fclose($fp); - if(!$hadLock){ - @unlink($lockPath); + $this->keepLockHandle($path, $fp); + // The cache size is looked at only after something new was cached + if($this->cacheGrew && empty($this->lockRegistry)){ + $this->cacheGrew = false; + if($this->staticCacheEnabled && \noneDB\Internal\Cache\CacheManager::cacheLimitBytes() > 0){ + $this->cache->evictCaches(); + } } } - $result['databases'][] = $dbname; } - - // Old caches can't be used with the new key (and the legacy one leaks names) - $oldFingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', $oldKey), 0, 16); - @unlink($dbDir . '.nonedb_hash_cache_' . $oldFingerprint); - @unlink($dbDir . '.nonedb_hash_cache'); - self::clearStaticCache(); - - $result['success'] = true; - return $result; } /** - * HMAC fingerprint of the secret; binds caches to the key without revealing it + * End of an exclusive session: write pending spatial indexes, close the journal entries of this + * session, and make sure the generation moved so other processes drop their caches */ - private function getKeyFingerprint(){ - if($this->keyFingerprint === null){ - $this->keyFingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', (string)$this->getSecretKey()), 0, 16); + private function finishWriteSession($dbname, $path, $ok){ + if($ok){ + if(empty($this->lockRegistry[$path]['rolledBack'])){ + $this->checkpoints->checkpointTouchedSegments($dbname, $path); + } + // After a rollback (of a deferred job) the journal stays: the next lock holder rebuilds + if(!empty($this->lockRegistry[$path]['flags']['j']) && empty($this->lockRegistry[$path]['rolledBack'])){ + // Data, indexes and meta of every journaled segment are written + unset($this->lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + } else { + // Partial failure: forget in-memory state, the files are the source of truth. + // A journal entry stays, so the next lock holder rebuilds what may be half-written. + $this->cache->invalidateDbCaches($dbname); } - return $this->keyFingerprint; } /** - * Path of the persistent hash cache for this dbDir + secretKey. - * The file name is derived from the key, so different keys never share (or poison) a cache, - * and the file can't be guessed without the key. + * Open (or reuse) a lock file and lock it. + * A kept handle is only used in the process that opened it (a forked child would share the parent's + * locks), and any handle is only trusted while the path still names the same file: if the file was + * deleted or replaced meanwhile, the file the path names now is locked instead. + * @return resource */ - private function getHashCacheFile(){ - return $this->dbDir . '.nonedb_hash_cache_' . $this->getKeyFingerprint(); + private function lockFile($path, $mode, $dbname){ + $pid = getmypid(); + for($attempt = 0; ; $attempt++){ + $kept = $this->lockHandles[$path] ?? null; + unset($this->lockHandles[$path]); + if($kept !== null && $kept['pid'] === $pid){ + $fp = $kept['fp']; + } else { + if($kept !== null){ + @fclose($kept['fp']); + } + // A database directory that is a link is no database: no link is followed (§3.4) + if(is_link(dirname($path))){ + throw new \RuntimeException("noneDB: the directory of database '{$dbname}' is a symbolic link, not a database (docs/STORAGE_FORMAT.md §3.4)", \noneDB::UNSUPPORTED_FORMAT); + } + // "e": close-on-exec. A kept handle must not leak into programs this process starts: + // an inherited copy would keep a lock alive after this process died holding it. + $created = !is_file($path); + $fp = is_dir(dirname($path)) ? @fopen($path, 'c+be') : false; + if($fp !== false && $created){ + $this->io->setPermissions($path); + } + if($fp === false){ + clearstatcache(true, dirname($path)); + if(!is_dir(dirname($path))){ + // The database directory is gone (dropped store, rotated key): the caller takes + // the missing-database path, which re-validates the store + throw new \RuntimeException("noneDB: database '{$dbname}' has no directory", self::DB_GONE); + } + throw new \RuntimeException("noneDB: cannot open lock file for database '{$dbname}'", \noneDB::LOCK_FAILED); + } + } + try { + $this->acquireFlock($fp, $mode, $dbname); + } catch(\Throwable $e){ + fclose($fp); + throw $e; + } + $own = fstat($fp); + clearstatcache(true, $path); + $named = @stat($path); + if($named !== false && ($own === false || !$own['ino'] || !$named['ino'] + || ($own['ino'] === $named['ino'] && $own['dev'] === $named['dev']))){ + return $fp; + } + flock($fp, LOCK_UN); + fclose($fp); + if($attempt >= 10){ + throw new \RuntimeException("noneDB: the lock file of database '{$dbname}' keeps being replaced", \noneDB::LOCK_FAILED); + } + } } /** - * Load the persistent hash cache for this dbDir + key (once per process) - * @return void + * Keep an unlocked lock file handle open for the next operation (the oldest is closed past the limit) */ - private function loadPersistentHashCache(){ - $file = $this->getHashCacheFile(); - if(isset($this->hashFilesLoaded[$file])){ - return; - } - $this->hashFilesLoaded[$file] = true; - - $fp = @fopen($file, 'rb'); - if($fp === false){ - return; - } - $content = ''; - if(flock($fp, LOCK_SH)){ - $content = stream_get_contents($fp); - flock($fp, LOCK_UN); - } - fclose($fp); - - $loaded = json_decode((string)$content, true); - if(!is_array($loaded)){ - return; - } - $prefix = $this->getKeyFingerprint() . '|'; - foreach($loaded as $dbname => $hash){ - if(is_string($dbname) && is_string($hash) && preg_match('/^[0-9a-f]{20}$/', $hash)){ - $this->hashCache[$prefix . $dbname] = $hash; - } + private function keepLockHandle($path, $fp){ + $this->lockHandles[$path] = ['fp' => $fp, 'pid' => getmypid()]; + if(count($this->lockHandles) > self::LOCK_HANDLE_LIMIT){ + reset($this->lockHandles); + $oldest = key($this->lockHandles); + @fclose($this->lockHandles[$oldest]['fp']); + unset($this->lockHandles[$oldest]); } } /** - * Add one name -> hash entry to the persistent cache (locked read-merge-write). - * Entries whose database no longer exists are dropped. - * @return void + * Parse a state record by the grammar of STORAGE_FORMAT.md §4.2: [state, flags, damaged]. An empty + * record is a new one (state ''); anything that doesn't match the grammar is damaged and read as J. */ - private function savePersistentHashCacheEntry($dbname, $hash){ - try { - $this->ensureDbDir(); - } catch(\RuntimeException $e){ - return; // the cache is optional + private static function parseLockState($raw){ + $tokens = explode(' ', trim($raw, " \0")); + $state = (string)array_shift($tokens); // "{generation}:{epoch}" + $flags = []; + $damaged = false; + if($state === ''){ + $tokens = []; // a new record: written with an epoch by the next exclusive session + } elseif(preg_match('/^[0-9]{1,18}:[0-9a-f]{8}$/', $state) !== 1){ + $damaged = true; } - $file = $this->getHashCacheFile(); - $isNew = !file_exists($file); - $fp = @fopen($file, 'c+b'); - if($fp === false){ - return; - } - if($isNew){ - @chmod($file, 0600); - } - if(!flock($fp, LOCK_EX)){ - fclose($fp); - return; - } - try { - $entries = json_decode((string)stream_get_contents($fp), true); - if(!is_array($entries)){ - $entries = []; - } - $entries[$dbname] = $hash; - foreach($entries as $name => $h){ - if(!is_string($h)){ - unset($entries[$name]); - continue; - } - $base = $this->dbDir . $h . '-' . $name; - if($name !== $dbname && !file_exists($base . '.nonedb') && !file_exists($base . '.nonedb.meta') && !file_exists($base . '.nonedbinfo')){ - unset($entries[$name]); - } + foreach($tokens as $token){ + if($token === ''){ + continue; } - $json = json_encode($entries, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); - if($json !== false){ - ftruncate($fp, 0); - rewind($fp); - fwrite($fp, $json); - fflush($fp); + if($token === 'b' || $token === 'c' || $token === 'd' || $token === 'm'){ + $flags[$token] = true; + } elseif($token === 'J'){ + $flags['j']['*'] = true; + } elseif(strncmp($token, 'j=', 2) === 0 && preg_match('/^j=(main|s[0-9]{1,9})(,(main|s[0-9]{1,9}))*$/', $token) === 1){ + foreach(explode(',', substr($token, 2)) as $segment){ + $flags['j'][$segment] = true; + } + } elseif(preg_match('/^g=([0-9]{1,9})>([0-9]{1,9})$/', $token, $m) === 1){ + $flags['g'] = [(int)$m[1], (int)$m[2]]; + } else { + $damaged = true; } - } finally { - flock($fp, LOCK_UN); - fclose($fp); } + if($damaged){ + // Not a record of this format: everything it could have said is recovered from the data (J); + // a generation that isn't CURRENT is never removed on this path (STORAGE_FORMAT.md §4.2) + $state = 'damaged'; + $flags = ['j' => ['*' => true], 'damaged' => true]; + } + return [$state, $flags, $damaged]; } /** - * Clear all static caches (useful for testing or memory management) - * @return void + * Read the state record (caller holds the lock). If another process wrote the database since this + * process last looked, all cached state of the database is dropped. */ - public static function clearStaticCache(){ - self::$staticIndexCache = []; - self::$staticShardedCache = []; - self::$staticMetaCache = []; - self::$staticMetaCacheTime = []; - self::$staticHashCache = []; - self::$staticFormatCache = []; - self::$staticFileExistsCache = []; - self::$staticSanitizeCache = []; - self::$staticFieldIndexCache = []; - self::$staticSpatialIndexCache = []; - self::$staticGlobalSpatialIndexCache = []; - self::$staticGenerations = []; - self::$staticHashFilesLoaded = []; - self::$bufferCounts = []; - foreach(self::$lockHandles as $handle){ - @fclose($handle['fp']); + private function loadLockState($dbname, $path){ + $fp = $this->lockRegistry[$path]['fp']; + fseek($fp, 0); + $raw = (string)fread($fp, self::LOCK_STATE_SIZE); + // The last record parsed per lock file: an unchanged record (the usual case) is not parsed again + $parsed = $this->parsedLockStates[$path] ?? null; + if($parsed !== null && $parsed[0] === $raw){ + [, $state, $flags, $damaged] = $parsed; + } else { + [$state, $flags, $damaged] = self::parseLockState($raw); + if($parsed === null && count($this->parsedLockStates) >= 4096){ + $this->parsedLockStates = []; + } + $this->parsedLockStates[$path] = [$raw, $state, $flags, $damaged]; + } + // The caches were checked against this exact state: nothing to do (the usual case, one comparison; + // the epoch is parsed when needed: false = the valid one of the state) + $seen = $this->generations[$path] ?? null; + $epoch = false; + if($seen !== $state){ + $colon = strpos($state, ':'); + $epoch = $colon === false ? null : substr($state, $colon + 1); + $seenEpoch = $seen === null || strpos($seen, ':') === false ? null : substr($seen, strpos($seen, ':') + 1); + if($seen === null || $epoch === null || $seenEpoch !== $epoch){ + $this->cache->invalidateDbCaches($dbname); + $this->layout->forgetDatabaseLayout($dbname); + // Another process replaced files or ran maintenance: the store is read again (§3.3) + unset($this->storeState[$this->dbDir]); + if($seen !== null && $this->maintenanceDepth === 0){ + $this->checkStoreAfterEpochChange(); + } + } else { + $this->cache->refreshDbCaches($dbname); // same epoch, another generation: the files only grew + } + $this->generations[$path] = $epoch === null ? null : $state; + } + $this->lockRegistry[$path]['gen'] = $damaged ? 0 : (int)$state; + $this->lockRegistry[$path]['state'] = $state; + $this->lockRegistry[$path]['epoch'] = $epoch; + $this->lockRegistry[$path]['flags'] = $flags; + if($this->durable){ + // Durable mode: flags read from the record are synced again with this session's first record + // (another process may have written them unsynced); the epoch protects nothing of this session + $current = $this->registryEpoch($path); + $this->lockRegistry[$path]['synced'] = $current === null ? [] : ['e' . $current => true]; } - self::$lockHandles = []; } /** - * Disable static caching (each instance uses its own cache) - * @return void + * A database's epoch changed under its lock: re-read the store (read only; no repair, which would + * take STORE.LOCK after a database lock) and re-check the key and the maintenance journal */ - public static function disableStaticCache(){ - self::$staticCacheEnabled = false; + private function checkStoreAfterEpochChange(){ + $this->store->checkMaintenanceJournal(); + [, $copy1] = $this->store->readStoreCopy('STORE'); + [, $copy2] = $this->store->readStoreCopy('STORE.2'); + $record = $copy1 ?? $copy2; + if($record === null || ($copy1 !== null && $copy2 !== null && $copy1 !== $copy2)){ + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged", \noneDB::FORMAT_DAMAGED); + } + if(!in_array($this->store->keyCheck(), $record['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + $this->storeState[$this->dbDir][$this->store->keyCheck()] = true; } /** - * Enable static caching (default) - * @return void + * Write the state record of a database held exclusively. $bump: a file is about to be written, so the + * generation moves (once per session; noteWrite()): even a writer that dies half-way makes other + * processes check their caches. A record without epoch gets one. + * Durable mode (DURABILITY.md §3): a flag or an epoch that protects the change that follows is synced + * before the record returns (clearing one needs no sync); $sync forces a sync. + * @throws \RuntimeException DURABILITY_FAILED when that sync fails */ - public static function enableStaticCache(){ - self::$staticCacheEnabled = true; + public function saveLockState($path, $bump = false, $sync = false){ + if($bump && !$this->lockRegistry[$path]['bumped']){ + $this->lockRegistry[$path]['gen']++; + $this->lockRegistry[$path]['bumped'] = true; + } + $this->lockRegistry[$path]['wrote'] = true; + $epoch = $this->registryEpoch($path); + if($epoch === null){ + $epoch = $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + } + $gen = max(1, (int)$this->lockRegistry[$path]['gen']); + $this->lockRegistry[$path]['gen'] = $gen; + $state = $gen . ':' . $epoch; + $this->generations[$path] = $state; + $this->lockRegistry[$path]['state'] = $state; + $flags = $this->lockRegistry[$path]['flags']; + unset($this->lockRegistry[$path]['flags']['damaged']); + foreach(['b', 'c', 'd', 'm'] as $flag){ + if(!empty($flags[$flag])){ + $state .= ' ' . $flag; + } + } + if(!empty($flags['g'])){ + $state .= ' g=' . $flags['g'][0] . '>' . $flags['g'][1]; + } + if(!empty($flags['j'])){ + $segments = ' j=' . implode(',', array_keys($flags['j'])); + $state .= (isset($flags['j']['*']) || strlen($state . $segments) > self::LOCK_STATE_SIZE) ? ' J' : $segments; + } + $fp = $this->lockRegistry[$path]['fp']; + $raw = str_pad($state, self::LOCK_STATE_SIZE); + fseek($fp, 0); + if(fwrite($fp, $raw) !== self::LOCK_STATE_SIZE || !fflush($fp)){ + unset($this->parsedLockStates[$path]); + throw new \RuntimeException("noneDB: cannot write the lock file of database '" . $this->lockRegistry[$path]['dbname'] . "'"); + } + // The next session of this process reads this record: parsed already (what parseLockState() would + // return; "J" alone stands for all segments, so such a record is parsed again) + if(strpos($state, ' J') === false){ + unset($flags['damaged']); + $this->parsedLockStates[$path] = [$raw, $gen . ':' . $epoch, $flags, false]; + } else { + unset($this->parsedLockStates[$path]); + } + if($this->durable){ + // What this record protects: its epoch and flags; what is durable already needs no new sync + $tokens = ['e' . $epoch => true]; + foreach(explode(' ', $state) as $i => $token){ + if($i > 0){ + $tokens[$token] = true; + } + } + $synced = $this->lockRegistry[$path]['synced'] ?? []; + if(!$sync){ + foreach($tokens as $token => $unused){ + if(!isset($synced[$token]) && !($token[0] === 'j' && isset($synced['J']))){ + $sync = true; + break; + } + } + } + if($sync){ + if(!$this->sync->trySync($fp, $path)){ + throw $this->sync->failure('file', $path, false); + } + $this->lockRegistry[$path]['synced'] = $tokens; + } + } } /** - * Cached file_exists check. Entries are dropped whenever another process writes the database - * (see withLock), so the cache can't go stale across processes. - * - * @param string $path File path to check - * @return bool True if file exists + * Change the state flag 'b' (write buffer may hold records); caller holds the exclusive lock */ - private function cachedFileExists($path){ - if(!self::$staticCacheEnabled){ - clearstatcache(true, $path); - return file_exists($path); + public function setBufferFlag($dbname, $on){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]) || !empty($this->lockRegistry[$path]['flags']['b']) === $on){ + return; } - if(isset(self::$staticFileExistsCache[$path])){ - return self::$staticFileExistsCache[$path]; + if($on){ + $this->lockRegistry[$path]['flags']['b'] = true; + } else { + unset($this->lockRegistry[$path]['flags']['b']); } - clearstatcache(true, $path); - $exists = file_exists($path); - self::$staticFileExistsCache[$path] = $exists; - return $exists; + $this->saveLockState($path); } /** - * Mark file as existing in cache (call after creating file) - * @param string $path File path + * After taking the lock: maintenance, an interrupted generation switch, creation or drop, a damaged + * directory and a crashed writer's journal are handled before any data is returned (STORAGE_FORMAT.md + * §4.2, §5.2, §8.3). The usual case (a present database, no flag but b) costs one isset. */ - private function markFileExists($path){ - if(self::$staticCacheEnabled){ - self::$staticFileExistsCache[$path] = true; + private function prepareDb($dbname, $path){ + $flags = $this->lockRegistry[$path]['flags']; + if(!empty($flags['m']) && $this->maintenanceDepth === 0){ + throw new \RuntimeException("noneDB: database '{$dbname}' is under maintenance", \noneDB::MAINTENANCE_IN_PROGRESS); + } + $quiet = empty($flags['g']) && empty($flags['c']) && empty($flags['d']) && empty($flags['j']) + && $this->lockRegistry[$path]['epoch'] !== null; + if($quiet && isset($this->formatRecords[$path])){ + return; + } + if($this->noWrites > 0){ + // checkIntegrity(), explain(): nothing is recovered or written; a state that can't be read is reported + if(!empty($flags['g'])){ + throw new \RuntimeException("noneDB: database '{$dbname}' has an interrupted generation switch (recovered by the next write)", \noneDB::FORMAT_DAMAGED); + } + $this->layout->resolveDatabaseState($dbname, $path); + return; } + if($quiet){ + $this->layout->resolveDatabaseState($dbname, $path); + return; + } + $this->withLock($dbname, true, function() use ($dbname, $path){ + // Re-read under the exclusive lock: another process may have done it meanwhile + if(!empty($this->lockRegistry[$path]['flags']['g'])){ + $this->layout->recoverGenerationSwitch($dbname, $path); + } + $this->layout->resolveDatabaseState($dbname, $path); + if($this->lockRegistry[$path]['epoch'] === null){ + // A new lock record: whatever the lost one said is recovered from the data + if($this->layout->dbPresent($dbname)){ + $this->lockRegistry[$path]['flags']['j']['*'] = true; + } + $this->saveLockState($path, false, true); + // 1:{new epoch} (§4.2): no cache can match a new epoch, so this session's writes need no + // further generation + $this->lockRegistry[$path]['bumped'] = true; + } + if(!empty($this->lockRegistry[$path]['flags']['j']) && $this->layout->dbPresent($dbname)){ + $this->recovery->recoverPendingIndexes($dbname, $path); + } elseif(!empty($this->lockRegistry[$path]['flags']['j'])){ + unset($this->lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + if(!empty($this->lockRegistry[$path]['flags']['damaged'])){ + $this->saveLockState($path, false, true); + } + }, 'recovery'); } /** - * Mark file as not existing in cache (call after deleting file) - * @param string $path File path + * Called before a segment's data changes. The journal is needed only for shard meta counters (sharded) + * and switches to shards ($always): field and spatial indexes follow the data by themselves. */ - private function markFileNotExists($path){ - if(self::$staticCacheEnabled){ - self::$staticFileExistsCache[$path] = false; + public function markIndexesPending($dbname, $shardId, $always = false){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path])){ + return; + } + if(!$always && $shardId === null){ + return; + } + $segment = \noneDB\Internal\Storage\Layout::segmentName($shardId); + if(isset($this->lockRegistry[$path]['pending'][$segment])){ + return; } + $this->lockRegistry[$path]['pending'][$segment] = true; + $this->lockRegistry[$path]['flags']['j'][$segment] = true; + $this->saveLockState($path, true); // the data write follows } /** - * Validate a database name. - * Allowed characters: A-Z a-z 0-9, space, apostrophe and hyphen. Anything else is rejected - * (before v3.2.0 such characters were silently stripped, which let different names collide). - * - * @param mixed $dbname Database name - * @return string The (unchanged) database name - * @throws \InvalidArgumentException + * flock with timeout (lockTimeout seconds; 0 waits forever) + * @throws \RuntimeException on timeout */ - private function normalizeDbName($dbname){ - if(is_int($dbname)){ - $dbname = (string)$dbname; - } - if(!is_string($dbname)){ - throw new \InvalidArgumentException('noneDB: database name must be a string'); - } - if(isset(self::$staticSanitizeCache[$dbname])){ - return $dbname; - } - if($dbname === '' || preg_match("/^[A-Za-z0-9' -]+$/", $dbname) !== 1){ - throw new \InvalidArgumentException( - "noneDB: invalid database name '{$dbname}'. Only letters A-Z/a-z, digits, space, apostrophe (') and hyphen (-) are allowed." - ); + public function acquireFlock($fp, $mode, $dbname){ + $this->metrics['locks']++; + if(flock($fp, $mode | LOCK_NB)){ + return; } - if(self::$staticCacheEnabled){ - self::$staticSanitizeCache[$dbname] = true; + // Held by another process: wait, up to lockTimeout (0: as long as it takes) + $this->metrics['lockWaits']++; + $start = microtime(true); + try { + if($this->lockTimeout <= 0){ + if(!flock($fp, $mode)){ + throw new \RuntimeException("noneDB: could not lock database '{$dbname}'", \noneDB::LOCK_FAILED); + } + return; + } + while(!flock($fp, $mode | LOCK_NB)){ + if((microtime(true) - $start) >= $this->lockTimeout){ + $this->metrics['lockTimeouts']++; + throw new \RuntimeException("noneDB: timed out after {$this->lockTimeout}s waiting for the lock on database '{$dbname}'", \noneDB::LOCK_FAILED); + } + usleep($this->lockRetryDelay); + } + } finally { + $this->metrics['lockWaitSeconds'] += microtime(true) - $start; } - return $dbname; } /** - * Hash a database name for its file names (PBKDF2-SHA256 keyed with secretKey). - * Cached per key fingerprint in memory and in a per-key file, so the secret is always honoured. + * A file is about to be written: the first write of an exclusive session bumps the generation of the + * database the file belongs to (see saveLockState). A database file written without its database's + * exclusive lock is a bug: LogicException. Files of no database (dbDir housekeeping) pass. + * @throws \LogicException */ - private function hashDBName($dbname){ - $cacheKey = $this->getKeyFingerprint() . '|' . $dbname; - if(isset($this->hashCache[$cacheKey])){ - return $this->hashCache[$cacheKey]; + public function noteWrite($file){ + $file = (string)$file; + unset($this->dataSizes[$file]); // a size cached for global index checks + $owner = null; + foreach($this->lockRegistry as $path => $entry){ + // Files of a database are inside its directory: the lock path without "LOCK" + $prefix = substr($path, 0, -4); + if(strncmp($file, $prefix, strlen($prefix)) === 0){ + $owner = $path; + break; + } } - - $this->loadPersistentHashCache(); - if(isset($this->hashCache[$cacheKey])){ - return $this->hashCache[$cacheKey]; + if($owner !== null){ + if(substr($file, -6) === '.jsonl' && strpos($file, '/segments/') !== false){ + // A data file: global index lookups check its shard's newer lines (globalCheckShards) + $writes = &$this->segWrites[$owner]; + $writes['seq'] = ($writes['seq'] ?? 0) + 1; + $writes['paths'][$file] = $writes['seq']; + unset($writes); + } + if($this->lockRegistry[$owner]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: write to " . basename($file) . " under a shared lock of database '" . $this->lockRegistry[$owner]['dbname'] . "'"); + } + $this->lockRegistry[$owner]['wrote'] = true; + if(!$this->lockRegistry[$owner]['bumped']){ + $this->saveLockState($owner, true); + } + } elseif(preg_match('#/[0-9a-f]{32}\.ndb4/#', $file) === 1){ + throw new \LogicException("noneDB: write to " . basename($file) . " without the lock of its database"); } - - $hash = hash_pbkdf2("sha256", (string)$dbname, (string)$this->getSecretKey(), 1000, 20); - $this->hashCache[$cacheKey] = $hash; - $this->savePersistentHashCacheEntry((string)$dbname, $hash); - - return $hash; } /** - * Common file-name prefix of every file belonging to a database: {dbDir}{hash}-{dbname} + * Epoch of a locked database's state record (null: none or invalid) */ - private function dbBase($dbname){ - if($this->dbBasesDir !== $this->dbDir){ - $this->dbBases = []; - $this->dbBasesDir = $this->dbDir; + public function registryEpoch($path){ + $epoch = $this->lockRegistry[$path]['epoch'] ?? null; + if($epoch === false){ + $state = $this->lockRegistry[$path]['state']; + $epoch = substr($state, strpos($state, ':') + 1); // a state with a valid epoch (see loadLockState) + $this->lockRegistry[$path]['epoch'] = $epoch; } - return $this->dbBases[$dbname] ?? ($this->dbBases[$dbname] = $this->dbDir . $this->hashDBName($dbname) . '-' . $dbname); + return $epoch; } /** - * Path of the main (non-sharded) data file + * A data file of the database is about to be replaced or removed (compaction, the switch to shards, drop): new + * epoch, so every other process drops all its caches of the database (caller holds the exclusive lock) */ - private function getDataPath($dbname){ - return $this->dbBase($dbname) . '.nonedb'; - } - /** - * Create the database directory (and protect it from direct web access) if needed + * Global field and spatial index files changed in memory during this lock session (a checkpoint per + * shard updates them): written once here, at the end of the session, at an epoch change or before the + * lock is upgraded, instead of once per shard (O(shards²) for an operation that checkpoints every + * shard). Not written when the session failed or the epoch changed since: a missing entry only means + * the shard is never skipped. */ - private function ensureDbDir(){ - if(is_dir($this->dbDir)){ + private function flushPendingGlobals($dbname, $path, $ok = true){ + $pending = $this->lockRegistry[$path]['globals'] ?? []; + if(empty($pending)){ return; } - if(!@mkdir($this->dbDir, 0755, true) && !is_dir($this->dbDir)){ - throw new \RuntimeException("noneDB: cannot create database directory {$this->dbDir}"); + $this->lockRegistry[$path]['globals'] = []; + $epoch = $this->registryEpoch($path); + foreach($pending as $file => [$kind, $field, $deferredAt]){ + $global = $kind === 'gfx' ? ($this->fieldIndexCache[$file] ?? null) : ($this->globalSpatialIndexCache[$file] ?? null); + if(!$ok || !empty($this->lockRegistry[$path]['rolledBack']) || $deferredAt !== $epoch || $global === null){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + continue; + } + try { + if($kind === 'gfx'){ + $this->fieldIndex->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->rtree->writeGlobalSpatialIndex($dbname, $field, $global); + } + } catch(\Throwable $e){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: a shard without a fitting entry is never skipped + } + } + } + + public function bumpEpoch($dbname){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]) || $this->lockRegistry[$path]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: new epoch of database '{$dbname}' without its exclusive lock"); + } + $this->flushPendingGlobals($dbname, $path); // under the epoch their entries belong to + $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + $this->saveLockState($path, true); + unset($this->globalSkip[$path], $this->segWrites[$path]); + // What this process checked against the data files no longer holds for the one being replaced + $prefix = $this->layout->dbHome($dbname); + foreach([&$this->dataSizes, &$this->dataEpochs, &$this->secondaryChecked, &$this->tailCache] as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, strlen($prefix)) === 0){ + unset($cache[$key]); + } + } } - // Deny direct HTTP access on Apache; harmless elsewhere - @file_put_contents($this->dbDir . '.htaccess', "Require all denied\nDeny from all\n"); - @file_put_contents($this->dbDir . 'index.html', ''); + unset($cache); } + /** + * Run a read operation: shared lock while the database exists, no lock (and no files created) + * when it doesn't (withLock's missing-database path). Buffered records are flushed first. + */ + public function withReadLock($dbname, callable $fn){ + return $this->withLock($dbname, false, function() use ($dbname, $fn){ + if(isset($this->lockRegistry[$this->getLockPath($dbname)])){ + $this->buffer->flushPendingBuffers($dbname); + } + return $fn(); + }); + } // ========================================== - // DATABASE LOCKING AND STATE (v3.2.0) + // DATABASE LOCKING AND STATE // ========================================== // - // Every database has a lock file {base}.nonedb.lock. Besides being the flock target it holds a small - // state record. It is read on every lock acquisition anyway, so the checks below cost no extra I/O: + // Every database has a lock file {locator}.ndb4/LOCK. Besides being the flock target it holds a small + // state record (docs/STORAGE_FORMAT.md §4.2). It is read on every lock acquisition anyway, so the + // checks below cost no extra I/O: // - // "{generation}[ v][ b][ j=main,0,3 | J]" space padded to LOCK_STATE_SIZE bytes + // "{generation}:{epoch}[ b][ c][ d][ m][ g=N>X][ j=main,s0,s3 | J]" space padded to LOCK_STATE_SIZE bytes // - // generation bumped by the first write of every write session. A value other than the one this - // process last saw means another process changed the database: its cached state is dropped. - // v verified: the one-time checks for pre-3.2 leftovers (old buffer files, no index list) - // are done, so the flags below are authoritative and nothing is re-checked per operation. + // generation bumped by the first file write of a write session (sessions that write nothing keep + // it). A value other than the one this process last saw means another process changed + // the database: its caches are checked again (key indexes: only the new data). + // epoch 8 hex digits, new before a file a process may have cached is replaced or removed + // (compaction, generation switch, drop, maintenance) and for a new lock record: another + // epoch drops every cache of the database and makes the process read the store again. // b the write buffer may hold records; they are flushed before the next read or write. - // j=... / J index journal: field/spatial indexes (and shard meta counters) of these segments (J: all) - // may lag behind the data because a writer died; the next lock holder rebuilds them. + // c / d creation / drop in progress (completed or undone by the next exclusive session). + // m maintenance in progress: MAINTENANCE_IN_PROGRESS. + // g=N>X generation switch from N to X in progress: recovered before any data is returned. + // j=... / J journal: shard counters of these segments (J: all) may lag behind the data because a + // writer died; the next lock holder recounts them. Key, field and spatial indexes need + // none: they replay the data after their checkpoint. // // The record is overwritten in place with a fixed size (one small write: no truncate, no rename). // Lock files are never deleted (dropDB keeps them), and their handles stay open between operations. - private const LOCK_STATE_SIZE = 128; - private const LOCK_HANDLE_LIMIT = 64; + public const LOCK_STATE_SIZE = 128; + public const LOCK_HANDLE_LIMIT = 64; + /** Exception code (internal): an exclusive lock was needed while a snapshot holds several shared ones */ + public const SNAPSHOT_RETRY = 3303; + public const DB_GONE = 3399; // internal: the database directory vanished under a kept path + public const DB_ABSENT = 3398; // internal: found absent without the missing-database path +} +} +namespace noneDB\Internal\Concurrency { + +use function array_values, basename, count, dirname, implode, ksort, microtime, realpath, usleep; +use const SORT_STRING; + +/** + * @internal + */ +final class Snapshot { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; - private function getLockPath($dbname){ - return $this->dbBase($dbname) . '.nonedb.lock'; + private $lockRetryDelay; + private $lockTimeout; + private $multiLock; + private $cache; + private $checkpoints; + private $layout; + private $locks; + private $recovery; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->layout, $this->locks, $this->recovery, $this->segments); } /** - * Run $fn while holding the database lock: shared for reads, exclusive for writes. - * - * The lock is process-wide and re-entrant. A nested exclusive request upgrades a shared lock. - * - * @param string $dbname Database name (already normalized) - * @param bool $exclusive - * @param callable $fn - * @return mixed Whatever $fn returns + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function withLock($dbname, $exclusive, callable $fn){ - $path = $this->getLockPath($dbname); + public function bind(): void { + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->multiLock = &$this->rt->multiLock; + } - if(isset(self::$lockRegistry[$path])){ - if($exclusive && self::$lockRegistry[$path]['mode'] !== LOCK_EX){ - // flock upgrades are not atomic: another writer may run in between, so re-read the state - $this->acquireFlock(self::$lockRegistry[$path]['fp'], LOCK_EX, $dbname); - self::$lockRegistry[$path]['mode'] = LOCK_EX; - $this->loadLockState($dbname, $path); + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Run $fn with shared locks on several databases at once (joins): every read sees the same + * state of all of them. Locks are taken in one canonical order (by lock file path), so two snapshots + * never wait for each other in a circle, and no exclusive lock is ever waited for while several are + * held: work that needs one (recovery, an index repair) is done for each database alone first. + */ + public function withSnapshot(array $dbnames, callable $fn){ + $names = []; + foreach($dbnames as $name){ + $name = $this->layout->normalizeDbName($name); + $lock = $this->locks->getLockPath($name); + $dir = realpath(dirname($lock)); + $names[($dir === false ? dirname($lock) : $dir) . '/' . basename($lock)] = $name; + } + ksort($names, SORT_STRING); + $names = array_values($names); + if(count($names) < 2){ + return $fn(); + } + $started = microtime(true); + while(true){ + foreach($names as $name){ + // Alone: recovery, buffer flush and key index repairs happen here, not under the snapshot + $this->locks->withReadLock($name, function() use ($name){ + $done = false; // a segment was read: the cache may drop what it loaded before the next one + foreach($this->segments->listSegments($name) as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $this->checkpoints->readJsonlIndex($name, $segment); + } + }); } - self::$lockRegistry[$path]['depth']++; try { - return $fn(); - } finally { - self::$lockRegistry[$path]['depth']--; + return $this->withSharedLocks($names, $fn); + } catch(\RuntimeException $e){ + if($e->getCode() !== \noneDB\Internal\Concurrency\LockManager::SNAPSHOT_RETRY){ + throw $e; + } + if($this->lockTimeout > 0 && microtime(true) - $started >= $this->lockTimeout){ + throw new \RuntimeException("noneDB: timed out waiting for a consistent snapshot of " . implode(', ', $names), \noneDB::LOCK_FAILED, $e); + } + usleep($this->lockRetryDelay); } } + } - $mode = $exclusive ? LOCK_EX : LOCK_SH; - $fp = $this->lockFile($path, $mode, $dbname); - self::$lockRegistry[$path] = [ - 'fp' => $fp, 'mode' => $mode, 'depth' => 1, 'dbname' => $dbname, - 'gen' => 0, 'flags' => [], 'bumped' => false, 'pending' => [], - ]; - - $ok = false; - try { - $this->loadLockState($dbname, $path); - $this->prepareDb($dbname, $path); - $result = $fn(); - $ok = true; - return $result; - } finally { + private function withSharedLocks(array $names, callable $fn, $i = 0){ + if($i >= count($names)){ + $this->multiLock++; try { - if(self::$lockRegistry[$path]['mode'] === LOCK_EX){ - $this->finishWriteSession($dbname, $path, $ok); - } + return $fn(); } finally { - $fp = self::$lockRegistry[$path]['fp']; - unset(self::$lockRegistry[$path]); - flock($fp, LOCK_UN); - $this->keepLockHandle($path, $fp); + $this->multiLock--; } } + return $this->locks->withReadLock($names[$i], function() use ($names, $fn, $i){ + return $this->withSharedLocks($names, $fn, $i + 1); + }); + } +} +} +namespace noneDB\Internal\Config { + +use function array_key_exists, array_keys, array_reverse, array_unique, basename, count, ctype_digit, define, defined, + dirname, fclose, file_exists, file_get_contents, flock, floor, getcwd, getenv, implode, in_array, is_array, is_bool, + is_file, is_finite, is_float, is_int, is_link, is_numeric, is_string, json_decode, levenshtein, md5, octdec, + preg_match, preg_replace, realpath, register_shutdown_function, rtrim, spl_object_id, str_replace, strpos, + strtolower, substr, trigger_error, trim; +use const DIRECTORY_SEPARATOR; +use const E_USER_NOTICE, LOCK_UN, NAN, PHP_OS_FAMILY, PHP_SAPI; + +/** + * @internal + */ +final class Configuration { + /** The settings (docs/CONFIGURATION.md "Settings"), in the order the documentation lists them */ + public const SETTINGS = ['secretKey', 'dbDir', 'autoCreateDB', 'shardingEnabled', 'shardSize', 'autoCompactThreshold', 'lockTimeout', + 'lockRetryDelay', 'bufferEnabled', 'fieldIndexEnabled', 'spatialIndexEnabled', 'durability', 'cacheMemoryLimit', 'filePermissions', + 'directoryPermissions']; + + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $bufferEnabled; + private $cloneSourceId; + private $dbDir; + private $devMode; + private $fieldIndexEnabled; + private $directoryPermissions; + private $durable; + private $filePermissions; + private $autoCompactThreshold; + private $keyFingerprint; + private $lockRetryDelay; + private $lockTimeout; + private $shardSize; + private $shardingEnabled; + private $spatialIndexEnabled; + private $cacheMemoryLimit; + private $configData; + private $configFileDir; + private $configFilePath; + private $configLoaded; + private $devModeNoticeShown; + private $placeholderKeys; + private $secretKeys; + private $shutdownCleanupRegistered; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferEnabled = &$this->k->bufferEnabled; + $this->cloneSourceId = &$this->k->cloneSourceId; + $this->dbDir = &$this->k->dbDir; + $this->devMode = &$this->k->devMode; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->directoryPermissions = &$this->k->directoryPermissions; + $this->durable = &$this->k->durable; + $this->filePermissions = &$this->k->filePermissions; + $this->autoCompactThreshold = &$this->k->autoCompactThreshold; + $this->keyFingerprint = &$this->k->keyFingerprint; + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->shardSize = &$this->k->shardSize; + $this->shardingEnabled = &$this->k->shardingEnabled; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + $this->cacheMemoryLimit = &$this->rt->cacheMemoryLimit; + $this->configData = &$this->rt->configData; + $this->configFileDir = &$this->rt->configFileDir; + $this->configFilePath = &$this->rt->configFilePath; + $this->configLoaded = &$this->rt->configLoaded; + $this->devModeNoticeShown = &$this->rt->devModeNoticeShown; + $this->placeholderKeys = &$this->rt->placeholderKeys; + $this->secretKeys = &$this->rt->secretKeys; + $this->shutdownCleanupRegistered = &$this->rt->shutdownCleanupRegistered; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + + /** + * Load the configuration of the instance (Kernel::__construct, after the caches are linked) + * @param array|null $config Optional config array; merged over the .nonedb file when one exists + * @throws \RuntimeException If no configuration can be found outside dev mode + * @throws \InvalidArgumentException If secretKey is missing, empty, not a string or a placeholder + */ + public function initialize($config = null){ + $this->loadConfig($config); + $this->validateSecretKey(); + + if(!$this->shutdownCleanupRegistered){ + $this->shutdownCleanupRegistered = true; + register_shutdown_function(static function(){ + self::releaseInterruptedSessions(); + }); + } } /** - * End of an exclusive session: write pending spatial indexes, close the journal entries of this - * session, and make sure the generation moved so other processes drop their caches + * Shutdown: a fatal error (memory_limit, max_execution_time) or exit() inside an operation skips + * its finally blocks, so its locks would stay registered and later shutdown code would continue the + * half-done session. Release them and drop every cache instead: the next operation takes the lock + * again and recovers from the files like after a crash. */ - private function finishWriteSession($dbname, $path, $ok){ - if($ok){ - $this->flushSpatialIndexes(); - if(!empty(self::$lockRegistry[$path]['flags']['j'])){ - // Data, indexes and meta of every journaled segment are written - unset(self::$lockRegistry[$path]['flags']['j']); - $this->saveLockState($path); - } - } else { - // Partial failure: forget in-memory state, the files are the source of truth. - // A journal entry stays, so the next lock holder rebuilds what may be half-written. - $this->dirtySpatialIndexes = []; - $this->invalidateDbCaches($dbname); + private static function releaseInterruptedSessions(){ + if(empty(\noneDB\Internal\Cache\Runtime::get()->lockRegistry)){ + return; } - if(!self::$lockRegistry[$path]['bumped']){ - $this->saveLockState($path); + foreach(\noneDB\Internal\Cache\Runtime::get()->lockRegistry as $entry){ + @flock($entry['fp'], LOCK_UN); + @fclose($entry['fp']); } + \noneDB\Internal\Cache\Runtime::get()->lockRegistry = []; + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); } /** - * Open (or reuse) a lock file and lock it. - * A kept handle is only used in the process that opened it (a forked child would share the parent's - * locks), and any handle is only trusted while the path still names the same file: if the file was - * deleted or replaced meanwhile, the file the path names now is locked instead. - * @return resource + * Never expose the secret in debug output + * @return array */ - private function lockFile($path, $mode, $dbname){ - $pid = getmypid(); - for($attempt = 0; ; $attempt++){ - $kept = self::$lockHandles[$path] ?? null; - unset(self::$lockHandles[$path]); - if($kept !== null && $kept['pid'] === $pid){ - $fp = $kept['fp']; - } else { - if($kept !== null){ - @fclose($kept['fp']); - } - $this->ensureDbDir(); - // "e": close-on-exec. A kept handle must not leak into programs this process starts: - // an inherited copy would keep a lock alive after this process died holding it. - $fp = @fopen($path, 'c+be'); - if($fp === false){ - throw new \RuntimeException("noneDB: cannot open lock file for database '{$dbname}'"); - } - } - try { - $this->acquireFlock($fp, $mode, $dbname); - } catch(\Throwable $e){ - fclose($fp); - throw $e; - } - $own = fstat($fp); - clearstatcache(true, $path); - $named = @stat($path); - if($named !== false && ($own === false || !$own['ino'] || !$named['ino'] - || ($own['ino'] === $named['ino'] && $own['dev'] === $named['dev']))){ - return $fp; - } - flock($fp, LOCK_UN); - fclose($fp); - if($attempt >= 10){ - throw new \RuntimeException("noneDB: the lock file of database '{$dbname}' keeps being replaced"); - } - } + public function debugInfo(){ + return [ + 'dbDir' => $this->dbDir, + 'secretKey' => '***', + 'autoCreateDB' => $this->autoCreateDB, + 'shardingEnabled' => $this->shardingEnabled, + 'shardSize' => $this->shardSize, + 'bufferEnabled' => $this->bufferEnabled, + 'durability' => $this->durable ? 'durable' : 'performance', + 'devMode' => $this->devMode, + ]; } - /** - * Keep an unlocked lock file handle open for the next operation (the oldest is closed past the limit) - */ - private function keepLockHandle($path, $fp){ - self::$lockHandles[$path] = ['fp' => $fp, 'pid' => getmypid()]; - if(count(self::$lockHandles) > self::LOCK_HANDLE_LIMIT){ - reset(self::$lockHandles); - $oldest = key(self::$lockHandles); - @fclose(self::$lockHandles[$oldest]['fp']); - unset(self::$lockHandles[$oldest]); + public function cloned(){ + $source = $this->secretKeys[$this->cloneSourceId] ?? null; + $this->cloneSourceId = spl_object_id($this->k); + if($source !== null){ + $this->secretKeys[$this->cloneSourceId] = $source; } } - /** - * Read the state record (caller holds the lock). If another process wrote the database since this - * process last looked, all cached state of the database is dropped. - */ - private function loadLockState($dbname, $path){ - $fp = self::$lockRegistry[$path]['fp']; - fseek($fp, 0); - $tokens = explode(' ', trim((string)fread($fp, self::LOCK_STATE_SIZE))); - $gen = (int)array_shift($tokens); - $flags = []; - foreach($tokens as $token){ - if($token === 'v' || $token === 'b'){ - $flags[$token] = true; - } elseif($token === 'J'){ - $flags['j']['*'] = true; - } elseif(strncmp($token, 'j=', 2) === 0){ - foreach(explode(',', substr($token, 2)) as $segment){ - if($segment !== ''){ - $flags['j'][$segment] = true; - } - } - } - } - if(!array_key_exists($path, $this->generations) || $this->generations[$path] !== $gen){ - $this->invalidateDbCaches($dbname); - $this->generations[$path] = $gen; - } - self::$lockRegistry[$path]['gen'] = $gen; - self::$lockRegistry[$path]['flags'] = $flags; + public function destroyed(){ + // Anything still dirty belongs to a lock we no longer hold; it was flushed on release. + unset($this->secretKeys[spl_object_id($this->k)]); } - /** - * Write the state record of a database held exclusively. The first write of a session also bumps - * the generation, so even a writer that dies half-way makes other processes drop their caches. - */ - private function saveLockState($path){ - if(!self::$lockRegistry[$path]['bumped']){ - self::$lockRegistry[$path]['gen']++; - self::$lockRegistry[$path]['bumped'] = true; - $this->generations[$path] = self::$lockRegistry[$path]['gen']; - } - $flags = self::$lockRegistry[$path]['flags']; - $state = (string)self::$lockRegistry[$path]['gen']; - if(!empty($flags['v'])){ - $state .= ' v'; - } - if(!empty($flags['b'])){ - $state .= ' b'; - } - if(!empty($flags['j'])){ - $segments = ' j=' . implode(',', array_keys($flags['j'])); - $state .= (isset($flags['j']['*']) || strlen($state . $segments) > self::LOCK_STATE_SIZE) ? ' J' : $segments; - } - $fp = self::$lockRegistry[$path]['fp']; - fseek($fp, 0); - if(fwrite($fp, str_pad($state, self::LOCK_STATE_SIZE)) !== self::LOCK_STATE_SIZE || !fflush($fp)){ - throw new \RuntimeException("noneDB: cannot write the lock file of database '" . self::$lockRegistry[$path]['dbname'] . "'"); - } + public function getSecretKey(){ + return $this->secretKeys[spl_object_id($this->k)] ?? null; } - /** - * Change the state flag 'b' (write buffer may hold records); caller holds the exclusive lock - */ - private function setBufferFlag($dbname, $on){ - $path = $this->getLockPath($dbname); - if(!isset(self::$lockRegistry[$path]) || !empty(self::$lockRegistry[$path]['flags']['b']) === $on){ - return; - } - if($on){ - self::$lockRegistry[$path]['flags']['b'] = true; - } else { - unset(self::$lockRegistry[$path]['flags']['b']); - } - $this->saveLockState($path); + private function setSecretKey($key){ + $this->secretKeys[spl_object_id($this->k)] = $key; + $this->cloneSourceId = spl_object_id($this->k); + $this->keyFingerprint = null; } /** - * True when the state record of a locked database is authoritative (see 'v' above) + * Load configuration. + * + * The .nonedb file is searched in: the entry script's directory, noneDB.php's directory, + * the Composer project root (when installed under vendor/) and the current working directory. + * A config array passed to the constructor is merged over the file (file is optional when the + * array already contains secretKey and dbDir). + * + * @param array|null $config Optional config array + * @throws \RuntimeException If config is missing and not in dev mode */ - private function isVerified($dbname){ - $path = $this->getLockPath($dbname); - return !empty(self::$lockRegistry[$path]['flags']['v']); - } + private function loadConfig($config = null){ + $arrayConfig = is_array($config) ? $config : []; + // A secretKey given in the array is used (and validated) as given, also null/false/'': + // never replaced by the file's key or the dev mode key + $keyGiven = array_key_exists('secretKey', $arrayConfig); + $arrayIsComplete = $keyGiven && isset($arrayConfig['dbDir']); - /** - * After taking the lock: rebuild what a crashed writer left half-updated, and once per database - * look for pre-3.2 leftovers. Both need the exclusive lock and are rare. - */ - private function prepareDb($dbname, $path){ - $flags = self::$lockRegistry[$path]['flags']; - if(!empty($flags['v']) && empty($flags['j'])){ - return; - } - $this->withLock($dbname, true, function() use ($dbname, $path){ - // Re-read under the exclusive lock: another process may have done it meanwhile - if(empty(self::$lockRegistry[$path]['flags']['v'])){ - $this->verifyDb($dbname, $path); - } - if(!empty(self::$lockRegistry[$path]['flags']['j'])){ - $this->recoverPendingIndexes($dbname, $path); - } - }); - } + // Load (and cache) the file config once per process + if(!$this->configLoaded){ + $this->configData = null; + $this->configFilePath = null; + $this->configFileDir = null; - /** - * One-time check of a database whose state record isn't verified yet (created before v3.2, or its - * lock file was deleted): pick up pre-3.2 write buffers and store the index list, so neither has to - * be looked for again - */ - private function verifyDb($dbname, $path){ - if($this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)) || !empty($this->legacyShardBufferFiles($dbname))){ - self::$lockRegistry[$path]['flags']['b'] = true; - } - $manifestPath = $this->getIndexManifestPath($dbname); - clearstatcache(true, $manifestPath); - if(!is_file($manifestPath)){ - unset($this->fieldIndexCache[$manifestPath]); - $manifest = $this->deriveIndexManifest($dbname); - if(!empty($manifest['fields']) || !empty($manifest['spatial'])){ - $this->saveIndexManifest($dbname, $manifest); + $configPath = self::findConfigFile(); + if($configPath !== null){ + $content = @file_get_contents($configPath); + if($content === false){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Cannot read config file: {$configPath}"); + } + } else { + $data = json_decode($content, true); + if(!is_array($data)){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Invalid JSON in config file: {$configPath}"); + } + } else { + $this->configData = $data; + $this->configFilePath = $configPath; + $this->configFileDir = dirname($configPath); + } + } } + $this->configLoaded = true; } - self::$lockRegistry[$path]['flags']['v'] = true; - $this->saveLockState($path); - } - /** - * Index journal: called before a segment's data changes. Needed when the database has secondary - * indexes, or meta counters (sharded) that follow the data. - */ - private function markIndexesPending($dbname, $shardId){ - $path = $this->getLockPath($dbname); - if(!isset(self::$lockRegistry[$path])){ - return; + if($this->configData !== null){ + self::checkNames($this->configData, self::SETTINGS, 'setting', " in {$this->configFilePath}"); + $fileConfig = $this->configData; + if(isset($arrayConfig['dbDir'])){ + unset($fileConfig['dbDir']); // overridden below; no need to resolve the file's one + } + $this->applyConfig($fileConfig, $this->configFileDir); } - $manifest = $this->getIndexManifest($dbname); - if($shardId === null && empty($manifest['fields']) && empty($manifest['spatial'])){ - return; + self::checkNames($arrayConfig, self::SETTINGS, 'setting'); + if(!empty($arrayConfig)){ + $this->applyConfig($arrayConfig, null); } - $segment = $shardId === null ? 'main' : (string)$shardId; - if(isset(self::$lockRegistry[$path]['pending'][$segment])){ + if($this->configData !== null || $arrayIsComplete || ($keyGiven && $this->dbDir !== null)){ return; } - self::$lockRegistry[$path]['pending'][$segment] = true; - self::$lockRegistry[$path]['flags']['j'][$segment] = true; - $this->saveLockState($path); - } - /** - * Rebuild field and spatial indexes of the segments a crashed writer left half-updated - * (caller holds the exclusive lock) - */ - private function recoverPendingIndexes($dbname, $path){ - $pending = self::$lockRegistry[$path]['flags']['j'] ?? []; - $known = $this->listSegments($dbname); - $segments = isset($pending['*']) ? $known : array_map(function($segment){ - return $segment === 'main' ? null : (int)$segment; - }, array_keys($pending)); - $manifest = $this->getIndexManifest($dbname); - foreach($segments as $shardId){ - if(!in_array($shardId, $known, true)){ - continue; - } - $this->repairSegmentIndex($dbname, $shardId); - foreach($manifest['fields'] as $field){ - $this->writeFieldIndex($dbname, $field, $this->buildSegmentFieldIndex($dbname, $shardId, $field), $shardId); - } - foreach($manifest['spatial'] as $field){ - $index = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, - isset($manifest['spatialOptions'][$field]['nodeSize']) ? ['nodeSize' => $manifest['spatialOptions'][$field]['nodeSize']] : []); - $this->writeSpatialIndex($dbname, $field, $index, $shardId); - if($shardId !== null){ - $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); - } + if(self::isDevModeEnabled()){ + $this->devMode = true; + $projectRoot = self::projectRoot(); + if($this->dbDir === null){ + $this->dbDir = $projectRoot . '/db/'; } - } - if($this->isSharded($dbname)){ - foreach($manifest['fields'] as $field){ - $this->rebuildGlobalFieldIndex($dbname, $field); + if(!$keyGiven){ + // Deterministic: a development database stays readable from one run to the next + $this->setSecretKey('nonedb_dev_mode_key_' . md5(__DIR__)); } - // A write may have died between its data and its meta update - $this->recountMeta($dbname); - // A migration may have died after switching to shards but before removing the old file - $legacy = $this->getDataPath($dbname); - clearstatcache(true, $legacy); - if(file_exists($legacy)){ - @unlink($legacy . '.jidx'); - @unlink($legacy); - $this->markFileNotExists($legacy); - unset($this->indexCache[$legacy . '.jidx'], $this->jsonlFormatCache[$legacy]); - foreach($manifest['fields'] as $field){ - $this->deleteFieldIndexFile($dbname, $field, null); - } - foreach($manifest['spatial'] as $field){ - @unlink($this->getSpatialIndexPath($dbname, $field, null)); - } + if(PHP_SAPI !== 'cli' && !$this->devModeNoticeShown){ + $this->devModeNoticeShown = true; + trigger_error('noneDB: development mode is active (predictable secretKey). Never use it in production.', E_USER_NOTICE); } + return; } - unset(self::$lockRegistry[$path]['flags']['j']); - $this->saveLockState($path); + + throw new \RuntimeException( + "noneDB: Configuration file not found!\n" . + "Create a '.nonedb' config file in your project root.\n" . + "See '.nonedb.example' for reference.\n" . + "For development, set NONEDB_DEV_MODE=true environment variable or define('NONEDB_DEV_MODE', true);" + ); } /** - * flock with timeout (lockTimeout seconds; 0 waits forever) - * @throws \RuntimeException on timeout + * Candidate config file locations, in priority order + * @return array */ - private function acquireFlock($fp, $mode, $dbname){ - if($this->lockTimeout <= 0){ - if(!flock($fp, $mode)){ - throw new \RuntimeException("noneDB: could not lock database '{$dbname}'"); - } - return; + private static function configSearchPaths(): array { + $dirs = [self::scriptDir(), __DIR__]; + $root = self::projectRoot(); + if($root !== __DIR__){ + $dirs[] = $root; } - $start = microtime(true); - while(!flock($fp, $mode | LOCK_NB)){ - if((microtime(true) - $start) >= $this->lockTimeout){ - throw new \RuntimeException("noneDB: timed out after {$this->lockTimeout}s waiting for the lock on database '{$dbname}'"); - } - usleep($this->lockRetryDelay); + $cwd = getcwd(); + if($cwd !== false){ + $dirs[] = $cwd; + } + $paths = []; + foreach(array_unique($dirs) as $dir){ + $paths[] = rtrim($dir, '/\\') . '/' . \noneDB\Internal\Cache\Runtime::get()->configFile; } + return $paths; } - /** - * First write of an exclusive session: bump the generation right away (see saveLockState) - */ - private function noteWrite(){ - foreach(self::$lockRegistry as $path => $entry){ - if($entry['mode'] === LOCK_EX && !$entry['bumped']){ - $this->saveLockState($path); + private static function findConfigFile(): ?string { + foreach(self::configSearchPaths() as $path){ + if(is_file($path)){ + return $path; } } + return null; } /** - * Drop every cached entry that belongs to a database (index, meta, field/spatial indexes, file-exists) + * Absolute directory of the entry script (falls back to the working directory) */ - private function invalidateDbCaches($dbname){ - $prefix = $this->dbDir . $this->hashDBName($dbname); - $len = strlen($prefix); - $caches = [ - &$this->indexCache, &$this->shardedCache, &$this->metaCache, &$this->metaCacheTime, - &$this->jsonlFormatCache, &$this->fieldIndexCache, &$this->spatialIndexCache, - &$this->globalSpatialIndexCache, - ]; - foreach($caches as &$cache){ - foreach(array_keys($cache) as $key){ - if(strncmp((string)$key, $prefix, $len) === 0){ - unset($cache[$key]); + private static function scriptDir(): string { + $script = $_SERVER['SCRIPT_FILENAME'] ?? ''; + $cwd = getcwd(); + // A relative SCRIPT_FILENAME depends on the working directory: both are part of the memo key + $memoKey = $script . "\0" . $cwd; + if(isset(\noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey])){ + return \noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey]; + } + $dir = null; + if($script !== ''){ + $real = realpath($script); + if($real !== false){ + $dir = dirname($real); + } else { + $realDir = realpath(dirname($script)); + if($realDir !== false){ + $dir = $realDir; } } } - unset($cache); - foreach(array_keys(self::$staticFileExistsCache) as $key){ - if(strncmp((string)$key, $prefix, $len) === 0){ - unset(self::$staticFileExistsCache[$key]); - } + if($dir === null){ + $dir = $cwd !== false ? $cwd : __DIR__; } - foreach(array_keys(self::$bufferCounts) as $key){ - if(strncmp((string)$key, $prefix, $len) === 0){ - unset(self::$bufferCounts[$key]); - } + if(count(\noneDB\Internal\Cache\Runtime::get()->scriptDirs) > 16){ + \noneDB\Internal\Cache\Runtime::get()->scriptDirs = []; } + return \noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey] = $dir; } - // ========================================== - // ATOMIC FILE OPERATIONS - // ========================================== - /** - * Encode JSON, returning false instead of producing invalid output - * @return string|false + * Project root: the directory that contains vendor/ when installed with Composer, else noneDB.php's directory */ - private function encodeJson($data, $prettyPrint = false, $extraFlags = 0){ - $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | $extraFlags; - if($prettyPrint){ - $flags |= JSON_PRETTY_PRINT; - } - try { - return json_encode($data, $flags | JSON_THROW_ON_ERROR); - } catch(\JsonException $e){ - return false; + private static function projectRoot(): string { + $dir = str_replace('\\', '/', __DIR__); + $pos = strpos($dir, '/vendor/'); + if($pos !== false){ + return substr(__DIR__, 0, $pos); } + return __DIR__; } - /** - * Encode one JSONL record line (without the trailing newline) - * @throws \InvalidArgumentException when the record can't be represented as JSON (invalid UTF-8, NAN, INF, ...) - */ - private function encodeRecord(array $record){ - // Keep floats floats: 2.0 / -0.0 are stored as 2.0 / -0.0, not as the ints 2 / 0 - $json = $this->encodeJson($record, false, JSON_PRESERVE_ZERO_FRACTION); - if($json === false){ - throw new \InvalidArgumentException('noneDB: record cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value): ' . json_last_error_msg()); - } - return $json; + private static function isDevModeEnabled(): bool { + return getenv('NONEDB_DEV_MODE') === 'true' + || getenv('NONEDB_DEV_MODE') === '1' + || (defined('NONEDB_DEV_MODE') && NONEDB_DEV_MODE === true); + } + + private static function isAbsolutePath(string $path): bool { + return $path !== '' && ($path[0] === '/' || $path[0] === '\\' || preg_match('/^[A-Za-z]:[\\\\\/]/', $path) === 1); } /** - * Write a whole file atomically: temp file in the same directory + rename. - * Readers see either the old or the new content, never a partial file. - * @return bool + * Apply configuration values to instance properties + * @param array $config Configuration array + * @param string|null $baseDir Directory that relative dbDir values resolve against (null: the entry script's) */ - private function writeFileAtomic($path, $content){ - $this->noteWrite(); - $dir = dirname($path); - if(!is_dir($dir) && !@mkdir($dir, 0755, true) && !is_dir($dir)){ - return false; + private function applyConfig(array $config, ?string $baseDir){ + if(array_key_exists('secretKey', $config)){ + $this->setSecretKey($config['secretKey']); } - $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); - $written = @file_put_contents($tmp, $content); - if($written === false || $written !== strlen($content)){ - @unlink($tmp); - return false; + if(array_key_exists('dbDir', $config)){ + if(!is_string($config['dbDir']) || trim($config['dbDir']) === ''){ + throw new \InvalidArgumentException("noneDB: 'dbDir' must be a directory path (a non-empty string)"); + } + $this->dbDir = $this->resolveDbDir($config['dbDir'], $baseDir); } - if(!@rename($tmp, $path)){ - @unlink($tmp); - return false; + if(array_key_exists('autoCreateDB', $config)){ + $this->autoCreateDB = self::configBool('autoCreateDB', $config['autoCreateDB']); } - clearstatcache(true, $path); - return true; - } - /** - * Append bytes to a file. Returns the offset the bytes start at, or false. - * A partial write is rolled back so the file never ends in a torn line. - * @return int|false - */ - private function appendToFile($path, $bytes){ - $this->noteWrite(); - $fp = @fopen($path, 'c+b'); - if($fp === false){ - return false; + // Sharding settings + if(array_key_exists('shardingEnabled', $config)){ + $this->shardingEnabled = self::configBool('shardingEnabled', $config['shardingEnabled']); } - try { - if(fseek($fp, 0, SEEK_END) !== 0){ - return false; + if(array_key_exists('shardSize', $config)){ + $shardSize = self::configNumber($config['shardSize'], true) ?? 0; + if($shardSize < 1){ + throw new \InvalidArgumentException("noneDB: 'shardSize' must be a positive integer"); } - $offset = ftell($fp); - if($offset === false){ - return false; + $this->shardSize = $shardSize; + } + + // Compaction settings + if(array_key_exists('autoCompactThreshold', $config)){ + $threshold = self::configNumber($config['autoCompactThreshold'], false) ?? 0.0; + if($threshold <= 0){ + throw new \InvalidArgumentException("noneDB: 'autoCompactThreshold' must be greater than 0"); } - $length = strlen($bytes); - $done = 0; - while($done < $length){ - $n = fwrite($fp, $done === 0 ? $bytes : substr($bytes, $done)); - if($n === false || $n === 0){ - ftruncate($fp, $offset); - return false; - } - $done += $n; + $this->autoCompactThreshold = $threshold; + } + + // Lock settings + if(array_key_exists('lockTimeout', $config)){ + // Seconds, fractions allowed (0.5 used to become 0 = wait forever) + $timeout = $config['lockTimeout']; + if(!is_int($timeout) && !is_float($timeout) && !(is_string($timeout) && is_numeric($timeout))){ + $timeout = null; + } + $timeout = $timeout === null ? NAN : (float)$timeout; + if(!is_finite($timeout) || $timeout < 0){ + throw new \InvalidArgumentException("noneDB: 'lockTimeout' must be 0 (wait forever) or a positive number of seconds"); } - fflush($fp); - return $offset; - } finally { - fclose($fp); - clearstatcache(true, $path); + $this->lockTimeout = $timeout; + } + if(array_key_exists('lockRetryDelay', $config)){ + $delay = self::configNumber($config['lockRetryDelay'], true) ?? 0; + if($delay < 1){ + throw new \InvalidArgumentException("noneDB: 'lockRetryDelay' must be a positive number of microseconds"); + } + $this->lockRetryDelay = $delay; } - } - /** - * Read and decode a JSON file - * - * @param string $path File path - * @param mixed $default Value returned when the file doesn't exist or is empty - * @return mixed Decoded data, $default, or null when the file is corrupt - */ - private function atomicRead($path, $default = null){ - clearstatcache(true, $path); - if(!is_file($path)){ - return $default; + // Write buffer + if(array_key_exists('bufferEnabled', $config)){ + $this->bufferEnabled = self::configBool('bufferEnabled', $config['bufferEnabled']); } - $content = @file_get_contents($path); - if($content === false || $content === ''){ - return $default; + + // Use indexes for queries (indexes are always kept up to date) + if(array_key_exists('fieldIndexEnabled', $config)){ + $this->fieldIndexEnabled = self::configBool('fieldIndexEnabled', $config['fieldIndexEnabled']); } - $data = json_decode($content, true); - if($data === null && json_last_error() !== JSON_ERROR_NONE){ - return null; // corrupted + if(array_key_exists('spatialIndexEnabled', $config)){ + $this->spatialIndexEnabled = self::configBool('spatialIndexEnabled', $config['spatialIndexEnabled']); + } + if(array_key_exists('durability', $config)){ + $this->durable = self::durabilityMode($config['durability']); + } + if(array_key_exists('filePermissions', $config)){ + $this->filePermissions = self::permissions('filePermissions', $config['filePermissions'], 0600); + } + if(array_key_exists('directoryPermissions', $config)){ + $this->directoryPermissions = self::permissions('directoryPermissions', $config['directoryPermissions'], 0700); + } + if(array_key_exists('cacheMemoryLimit', $config)){ + $limit = $config['cacheMemoryLimit']; + if($limit === 'auto'){ + $this->cacheMemoryLimit = null; + } elseif(!(is_int($limit) && $limit >= 0) && !(is_string($limit) && ctype_digit($limit))){ + throw new \InvalidArgumentException("noneDB: 'cacheMemoryLimit' must be a non-negative integer (bytes) or 'auto'"); + } else { + $this->cacheMemoryLimit = (int)$limit; + } } - return $data !== null ? $data : $default; } /** - * Encode and write a JSON file atomically. Nothing is touched when encoding fails. - * - * @param string $path File path - * @param mixed $data Data to write (will be JSON encoded) - * @param bool $prettyPrint Use JSON_PRETTY_PRINT - * @return bool Success + * The 'durability' setting: 'performance' or 'durable' (true). Durable mode is never downgraded: where + * the platform can't provide it, the instance is not created (docs/DURABILITY.md §1). + * @throws \InvalidArgumentException a value other than the two modes + * @throws \RuntimeException UNSUPPORTED_DURABILITY */ - private function atomicWrite($path, $data, $prettyPrint = false){ - $json = $this->encodeJson($data, $prettyPrint); - if($json === false){ - return false; + public static function durabilityMode($value): bool { + self::durabilityValue($value); + if($value === 'durable' && !self::durableSupported()){ + throw new \RuntimeException("noneDB: durable mode is not supported on " . self::osFamily() . ": " + . self::platformNote(self::osFamily()) . " Use 'durability' => 'performance' or run on Linux.", \noneDB::UNSUPPORTED_DURABILITY); } - return $this->writeFileAtomic($path, $json); + return $value === 'durable'; } - // ========================================== - // STORAGE CORE (v3.2.0) - // ========================================== - // - // A database is one "segment" when it is not sharded ({base}.nonedb + {base}.nonedb.jidx) or one - // segment per shard ({base}_sN.nonedb + .jidx) listed in {base}.nonedb.meta. Segment ids are null - // (non-sharded) or the shard id. Every write runs under the database's exclusive lock (withLock), - // reads under a shared lock, so index read-modify-write cycles are never interleaved. - // - // Data files are append-only JSONL. Every record line carries its "key"; deletions append a - // tombstone line {"__nonedb_deleted":key}. The .jidx index maps key => [offset, length] and stores - // "s", the data size it covers. If the data file is shorter than "s" (e.g. a crash between the two - // renames of a compaction) or the index is missing/corrupt, the index is rebuilt from the data. - /** - * Get the file path for a specific shard - * @param string $dbname - * @param int $shardId - * @return string + * A durability value: 'performance' or 'durable', whatever the platform + * @throws \InvalidArgumentException */ - private function getShardPath($dbname, $shardId){ - return $this->dbBase($dbname) . "_s" . $shardId . ".nonedb"; + public static function durabilityValue($value): void { + if($value !== 'performance' && $value !== 'durable'){ + throw new \InvalidArgumentException("noneDB: 'durability' must be 'performance' or 'durable'"); + } } - /** - * Get the meta file path for a database - * @param string $dbname - * @return string - */ - private function getMetaPath($dbname){ - return $this->dbBase($dbname) . ".nonedb.meta"; + /** The platform durable mode is checked against */ + public static function osFamily(): string { + return PHP_OS_FAMILY; } - /** - * Data file of a segment (null = non-sharded database) - */ - private function segDataPath($dbname, $shardId){ - return $shardId === null ? $this->getDataPath($dbname) : $this->getShardPath($dbname, $shardId); + /** Durable mode is supported on Linux only (docs/DURABILITY.md §1) */ + public static function durableSupported(): bool { + return self::osFamily() === 'Linux'; + } + + /** Why durable mode is not available on a platform ('' where it is) */ + public static function platformNote(string $os): string { + switch($os){ + case 'Linux': + return ''; + case 'Darwin': + return 'fsync() does not flush the drive cache on macOS, and PHP cannot call F_FULLFSYNC.'; + case 'Windows': + return 'a directory cannot be synced on Windows, so a new file or a rename is not durable.'; + } + return 'only Linux is supported for durable mode.'; } /** - * Check if a database is sharded - * @param string $dbname - * @return bool + * A numeric setting: an int, a finite float or a numeric string (an integer when $integer); anything + * else (booleans, arrays, "10abc", NAN) is null, and the caller rejects it + * @return int|float|null */ - private function isSharded($dbname){ - $metaPath = $this->getMetaPath($dbname); - if(isset($this->shardedCache[$metaPath])){ - return $this->shardedCache[$metaPath]; + private static function configNumber($value, bool $integer){ + if(is_string($value) && is_numeric(trim($value))){ + $value = trim($value) + 0; } - - clearstatcache(true, $metaPath); - $result = file_exists($metaPath); - $this->shardedCache[$metaPath] = $result; - return $result; + if(is_int($value)){ + return $integer ? $value : (float)$value; + } + if(is_float($value) && is_finite($value) && (!$integer || floor($value) === $value)){ + return $integer ? (int)$value : $value; + } + return null; } /** - * True when the database has data (non-sharded file or sharded meta) + * Every key of $given must be one of $known (settings, or a method's options): an unknown one is an error that + * names it, suggests the known name that differs only in letter case or by up to two letters, and lists them + * @param string $noun 'setting' or 'option' + * @param string $where Where they come from: " in {file}", " of {method}()" + * @throws \InvalidArgumentException */ - private function dbExists($dbname){ - return $this->cachedFileExists($this->getDataPath($dbname)) || $this->isSharded($dbname); + public static function checkNames(array $given, array $known, string $noun, string $where = ''): void { + foreach(array_keys($given) as $name){ + if(in_array($name, $known, true)){ + continue; + } + $name = (string)$name; + $closest = null; + $distance = 3; + foreach($known as $candidate){ + $d = levenshtein(strtolower($name), strtolower($candidate)); + if($d < $distance){ + $closest = $candidate; + $distance = $d; + } + } + throw new \InvalidArgumentException("noneDB: unknown {$noun} '{$name}'{$where}" . ($closest === null ? '' : " (did you mean '{$closest}'?)") + . "; the {$noun}s are: " . implode(', ', $known)); + } } /** - * Read shard metadata - * @param string $dbname - * @return array|null + * An option that is true or false (an option is given in PHP: a boolean, nothing else) + * @throws \InvalidArgumentException */ - private function readMeta($dbname){ - $meta = $this->atomicRead($this->getMetaPath($dbname), null); - return is_array($meta) && isset($meta['shards']) && is_array($meta['shards']) ? $meta : null; + public static function optionBool(string $name, $value): bool { + if(!is_bool($value)){ + throw new \InvalidArgumentException("noneDB: '{$name}' must be true or false"); + } + return $value; } /** - * Get cached meta data. The cache is dropped whenever another process writes the - * database (generation check in withLock), so no TTL is needed. - * @param string $dbname - * @param bool $forceRefresh Force refresh from disk - * @return array|null + * A boolean setting: true/false, 1/0, "1"/"0", "true"/"false" (any case). Anything else is an error, + * so that a string like "false" or "no" never means true. + * @throws \InvalidArgumentException */ - private function getCachedMeta($dbname, $forceRefresh = false){ - $metaPath = $this->getMetaPath($dbname); - - if(!$forceRefresh && isset($this->metaCache[$metaPath])){ - return $this->metaCache[$metaPath]; + private static function configBool(string $name, $value): bool { + if(is_bool($value)){ + return $value; } - - $meta = $this->readMeta($dbname); - if($meta !== null){ - $this->metaCache[$metaPath] = $meta; - return $meta; + if($value === 1 || $value === 0){ + return $value === 1; } - unset($this->metaCache[$metaPath]); - return null; + if(is_string($value)){ + $lower = strtolower(trim($value)); + if($lower === '1' || $lower === 'true'){ + return true; + } + if($lower === '0' || $lower === 'false'){ + return false; + } + } + throw new \InvalidArgumentException("noneDB: '{$name}' must be true or false"); } /** - * Write shard metadata (caller holds the exclusive lock) - * @param string $dbname - * @param array $meta - * @return bool - * @throws \RuntimeException when the meta file can't be written + * Permission bits of a setting: an integer or an octal string ('0640'); the owner must have $owner (read and + * write, and search for directories), and the world no write access + * @throws \InvalidArgumentException */ - private function writeMeta($dbname, $meta){ - $path = $this->getMetaPath($dbname); - if(!$this->atomicWrite($path, $meta, true)){ - throw new \RuntimeException("noneDB: failed to write metadata of database '{$dbname}'"); + private static function permissions(string $name, $value, int $owner): int { + $mode = is_int($value) ? $value : (is_string($value) && preg_match('/^0?[0-7]{3}$/D', $value) === 1 ? (int)octdec($value) : -1); + if($mode < 0 || $mode > 0777 || ($mode & $owner) !== $owner || ($mode & 0002) !== 0){ + throw new \InvalidArgumentException("noneDB: '{$name}' must be permission bits like " . ($owner === 0600 ? '0640' : '0750') + . ' (an integer or an octal string): the owner needs ' . ($owner === 0600 ? 'read and write' : 'read, write and search') . ', the world may not write'); } - $this->metaCache[$path] = $meta; - $this->shardedCache[$path] = true; - $this->markFileExists($path); - return true; + return $mode; } /** - * Records per shard of a database (existing sharded databases keep the size stored in their meta) + * Resolve a configured dbDir to its canonical absolute path ending with a slash (STORAGE_FORMAT.md §3). + * Relative paths (with or without a leading ./) are relative to the config file's directory + * (file config) or to the entry script's directory (array config). */ - private function shardSizeOf($dbname){ - $meta = $this->isSharded($dbname) ? $this->getCachedMeta($dbname) : null; - if($meta !== null && isset($meta['shardSize']) && (int)$meta['shardSize'] > 0){ - return (int)$meta['shardSize']; + private function resolveDbDir(string $dbDir, ?string $baseDir): string { + if(!self::isAbsolutePath($dbDir)){ + $relative = preg_replace('#^(\./)+#', '', $dbDir); + $baseDir = $baseDir ?? self::scriptDir(); + $dbDir = rtrim($baseDir, '/\\') . '/' . $relative; } - return $this->shardSize; + return rtrim(self::canonicalPath($dbDir), '/\\') . '/'; } /** - * Segment ids of a database: [null] when not sharded, the shard ids otherwise, [] if it doesn't exist + * A path with its links and its "." and ".." resolved: the nearest existing part by realpath(), the parts + * after it (that don't exist yet) by their names */ - private function listSegments($dbname){ - if($this->isSharded($dbname)){ - $meta = $this->getCachedMeta($dbname); - if($meta === null){ - throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + public static function canonicalPath(string $path): string { + $missing = []; + for($at = rtrim($path, '/\\'); $at !== '' && !file_exists($at) && !is_link($at); $at = dirname($at)){ + $missing[] = basename($at); + if(dirname($at) === $at){ + break; + } + } + $resolved = realpath($at === '' ? DIRECTORY_SEPARATOR : $at); + $resolved = $resolved === false ? rtrim($at, '/\\') : rtrim($resolved, '/\\'); + foreach(array_reverse($missing) as $part){ + if($part === '..'){ + $resolved = dirname($resolved); + } elseif($part !== '.' && $part !== ''){ + $resolved .= DIRECTORY_SEPARATOR . $part; } - return array_map('intval', array_column($meta['shards'], 'id')); } - return $this->cachedFileExists($this->getDataPath($dbname)) ? [null] : []; + return $resolved === '' ? DIRECTORY_SEPARATOR : $resolved; } /** - * Segment that holds a key, or false if that shard doesn't exist - * @return int|null|false + * Reject missing, empty, non-string and placeholder secret keys + * @throws \InvalidArgumentException */ - private function segmentForKey($dbname, $key){ - if(!$this->isSharded($dbname)){ - return null; + private function validateSecretKey(){ + $key = $this->getSecretKey(); + if(!is_string($key) || trim($key) === ''){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' must be a non-empty string. Set it in your .nonedb file or constructor config " + . "(generate one with: php -r \"echo bin2hex(random_bytes(32));\")." + ); + } + if(in_array($key, $this->placeholderKeys, true)){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' is a publicly known key (an example placeholder or a published default key). " + . "Replace it with a random secret (php -r \"echo bin2hex(random_bytes(32));\"); databases written with it " + . "move to the new key with noneDB::migrateKey() (docs/CONFIGURATION.md)." + ); + } + if($this->dbDir === null){ + throw new \RuntimeException("noneDB: 'dbDir' is not configured"); } - $shardId = intdiv($key, $this->shardSizeOf($dbname)); - return in_array($shardId, $this->listSegments($dbname), true) ? $shardId : false; } - // ========================================== - // JSONL SEGMENT INDEX - // ========================================== - /** - * Get JSONL index path - * @param string $dbname - * @param int|null $shardId Null for non-sharded - * @return string + * Check if config file exists + * @return bool */ - private function getJsonlIndexPath($dbname, $shardId = null){ - return $this->segDataPath($dbname, $shardId) . ".jidx"; - } - - private function newJsonlIndex(){ - return [ - 'v' => 3, - 'format' => 'jsonl', - 'created' => time(), - 'n' => 0, - 'd' => 0, - 't' => 0, - 's' => 0, - 'o' => [] - ]; + public static function configExists(): bool { + return self::findConfigFile() !== null; } /** - * Read a segment's JSONL index (byte offset map), repairing it when it is missing, corrupt or - * doesn't match the data file. - * @param string $dbname - * @param int|null $shardId - * @return array|null Null when the segment doesn't exist + * Get the config file template path + * @return string|null */ - private function readJsonlIndex($dbname, $shardId = null){ - $dataPath = $this->segDataPath($dbname, $shardId); - if(!$this->cachedFileExists($dataPath)){ - return null; - } - $path = $dataPath . '.jidx'; - // A cached index was checked when it was loaded, and is dropped as soon as another process writes - if(isset($this->indexCache[$path])){ - return $this->indexCache[$path]; - } + public static function getConfigTemplate(): ?string { + $templatePath = __DIR__ . '/.nonedb.example'; + return file_exists($templatePath) ? $templatePath : null; + } - // Missing or unreadable (pre-3.0 files have no index either): repaired or converted - $index = $this->atomicRead($path, null); - if(!is_array($index) || !isset($index['o']) || !is_array($index['o'])){ - return $this->repairSegmentIndex($dbname, $shardId); - } + /** + * Clear config cache (useful for testing) + * @return void + */ + public static function clearConfigCache(): void { + \noneDB\Internal\Cache\Runtime::get()->configLoaded = false; + \noneDB\Internal\Cache\Runtime::get()->configData = null; + \noneDB\Internal\Cache\Runtime::get()->configFilePath = null; + \noneDB\Internal\Cache\Runtime::get()->configFileDir = null; + } - // The data file can only grow between index writes; shorter means the index belongs to other data - if(isset($index['s'])){ - clearstatcache(true, $dataPath); - $size = @filesize($dataPath); - if($size !== false && $size < $index['s']){ - return $this->repairSegmentIndex($dbname, $shardId); - } + /** + * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) + * @return bool + */ + public static function isDevMode(): bool { + return self::isDevModeEnabled(); + } + + /** + * Set development mode programmatically + * Useful for testing or when env vars are not available + * @param bool $enabled + * @return void + */ + public static function setDevMode(bool $enabled): void { + if($enabled && !defined('NONEDB_DEV_MODE')){ + define('NONEDB_DEV_MODE', true); } + } +} +} +namespace noneDB\Internal\Core { - $this->indexCache[$path] = $index; - return $index; +use function array_key_exists, array_keys, array_values, count, is_array, is_int; + +/** + * @internal + */ +final class Crud { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $bufferEnabled; + private $buffer; + private $findFilter; + private $io; + private $layout; + private $locks; + private $segments; + private $writer; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->findFilter, $this->io, $this->layout, $this->locks, $this->segments, $this->writer); } /** - * Write a segment's JSONL index (caller holds the exclusive lock) - * @param string $dbname - * @param array $index - * @param int|null $shardId - * @return bool - * @throws \RuntimeException when the index can't be written + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function writeJsonlIndex($dbname, $index, $shardId = null){ - $dataPath = $this->segDataPath($dbname, $shardId); - $path = $dataPath . '.jidx'; - clearstatcache(true, $dataPath); - $size = @filesize($dataPath); - $index['s'] = $size === false ? 0 : $size; - $index['updated'] = time(); - if(!$this->atomicWrite($path, $index)){ - unset($this->indexCache[$path]); - throw new \RuntimeException("noneDB: failed to write index of database '{$dbname}'"); - } - $this->indexCache[$path] = $index; - return true; + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferEnabled = &$this->k->bufferEnabled; } /** - * Rebuild a segment index from its data file (exclusive lock, upgrading a shared one). - * Re-checks first: another process may have repaired it while we waited for the lock. - * @param bool $force Rebuild even if the index on disk looks consistent - * @return array|null + * A handle to another service, bound on first use */ - private function repairSegmentIndex($dbname, $shardId, $force = false){ - return $this->withLock($dbname, true, function() use ($dbname, $shardId, $force){ - $dataPath = $this->segDataPath($dbname, $shardId); - $path = $dataPath . '.jidx'; - unset($this->indexCache[$path]); - clearstatcache(true, $dataPath); - if(!file_exists($dataPath)){ - return null; - } - // A pre-3.0 {"data": [...]} file (it has no index) is converted, never scanned as JSONL - $this->ensureJsonlFormat($dbname, $shardId); - if(!$force){ - $index = $this->atomicRead($path, null); - if(is_array($index) && isset($index['o']) && is_array($index['o'])){ - $size = @filesize($dataPath); - if(!isset($index['s']) || ($size !== false && $size >= $index['s'])){ - $this->indexCache[$path] = $index; - return $index; - } - } - } - $index = $this->rebuildSegmentIndexFromData($dbname, $shardId); - if($shardId !== null){ - $this->recountMeta($dbname); - } - return $index; - }); + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Scan a data file and rebuild its index: the last line of a key wins, tombstones delete, - * a torn last line (crash during append) is cut off. Caller holds the exclusive lock. - * @return array + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function rebuildSegmentIndexFromData($dbname, $shardId){ - $dataPath = $this->segDataPath($dbname, $shardId); - $old = $this->atomicRead($dataPath . '.jidx', null); + public function unbind(): void { + $this->buffer = null; + $this->findFilter = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->segments = null; + $this->writer = null; + $this->k = null; + $this->rt = null; + } - $offsets = []; - $lines = 0; - $tombstones = 0; - $maxKey = -1; - $offset = 0; - $fp = @fopen($dataPath, 'rb'); - if($fp !== false){ - while(($line = fgets($fp)) !== false){ - if(substr($line, -1) !== "\n"){ - break; // torn tail - } - $json = rtrim($line, "\r\n"); - $record = json_decode($json, true); - if(is_array($record)){ - if(array_key_exists('__nonedb_deleted', $record) && count($record) === 1){ - $k = (int)$record['__nonedb_deleted']; - unset($offsets[$k]); - $maxKey = max($maxKey, $k); - $tombstones++; - } elseif(isset($record['key']) && is_int($record['key'])){ - $offsets[$record['key']] = [$offset, strlen($json)]; - $maxKey = max($maxKey, $record['key']); - } - } - $offset += strlen($line); - $lines++; - } - fclose($fp); + /** + * The records matching the filters, in key order (a "key" filter: in its order) + * @param string $dbname + * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); + * a "key" entry selects by record key (int or list of ints) + * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + */ + public function find($dbname, $filters=0){ + $dbname = $this->layout->normalizeDbName($dbname); + if($filters === null || is_int($filters)){ + $filters = []; } - clearstatcache(true, $dataPath); - if(file_exists($dataPath) && filesize($dataPath) > $offset){ - $fp = fopen($dataPath, 'c+b'); - if($fp !== false){ - ftruncate($fp, $offset); - fclose($fp); - } + if(!is_array($filters)){ + return false; } + return $this->locks->withReadLock($dbname, function() use ($dbname, $filters){ + return $this->findUnlocked($dbname, $filters); + }); + } - ksort($offsets, SORT_NUMERIC); - $index = $this->newJsonlIndex(); - if(is_array($old) && isset($old['created'])){ - $index['created'] = $old['created']; - } - $index['o'] = $offsets; - $index['t'] = $tombstones; - $index['d'] = max(0, $lines - count($offsets) - $tombstones); - if($shardId === null){ - $index['n'] = max($maxKey + 1, is_array($old) && isset($old['n']) ? (int)$old['n'] : 0); - } else { - $base = $shardId * $this->shardSizeOf($dbname); - $index['n'] = $maxKey >= $base ? $maxKey - $base + 1 : 0; + public function findUnlocked($dbname, $filters=0){ + if($filters === null || is_int($filters)){ + $filters = []; + } + if(!$this->segments->dbExists($dbname)){ + // Reads never create databases + return $this->autoCreateDB ? [] : false; } - $this->writeJsonlIndex($dbname, $index, $shardId); - return $this->indexCache[$this->getJsonlIndexPath($dbname, $shardId)]; + return array_values($this->findFilter->selectRecords($dbname, $filters)); + } + + /** + * Check if 'key' exists at top level of data (not nested) + * @param array $data + * @return bool + */ + private function hasReservedKeyField($data){ + return is_array($data) && array_key_exists("key", $data); } /** - * Recompute per-shard and total counters in the meta file from the shard indexes + * Check if array is a list of records: sequential 0..n-1 keys, every element an array + * @param array $data + * @return bool */ - private function recountMeta($dbname){ - $meta = $this->getCachedMeta($dbname, true); - if($meta === null){ - return; + private function isRecordList($data){ + if(!is_array($data) || count($data) === 0){ + return false; } - $size = (int)$meta['shardSize']; - $total = 0; - $maxKey = -1; - foreach($meta['shards'] as $i => $shard){ - $index = $this->readJsonlIndex($dbname, (int)$shard['id']); - $count = $index === null ? 0 : count($index['o']); - $meta['shards'][$i]['count'] = $count; - $total += $count; - if($index !== null && !empty($index['o'])){ - $maxKey = max($maxKey, max(array_keys($index['o']))); + $i = 0; + foreach($data as $k => $v){ + if($k !== $i++ || !is_array($v)){ + return false; } } - $meta['totalRecords'] = $total; - $meta['nextKey'] = max((int)($meta['nextKey'] ?? 0), $maxKey + 1); - $this->writeMeta($dbname, $meta); + return true; } /** - * Detect if a data file is in JSONL format (vs the pre-3.0 single JSON document {"data": [...]}) - * @param string $path - * @return bool True if JSONL format + * Validate insert data + * @return array [items, error|null] */ - private function isJsonlFormat($path){ - if(isset($this->jsonlFormatCache[$path])){ - return $this->jsonlFormatCache[$path]; + private function validateInsertData($data){ + if(!is_array($data)){ + return [[], "insert data must be array"]; } - if(!$this->cachedFileExists($path)){ - return true; + if(empty($data)){ + return [[], "insert data must not be empty"]; } - - // A JSONL database always has its .jidx (or the index can be rebuilt from the lines). - // Only a file without index that decodes as exactly {"data": [...]} is the old v2 format; - // a JSONL record always has a "key" field, so a record whose first field is "data" is never mistaken. - $isJsonl = true; - if(!file_exists($path . '.jidx')){ - $content = @file_get_contents($path); - if($content !== false && $content !== ''){ - $decoded = json_decode($content, true); - if(is_array($decoded) && array_keys($decoded) === ['data'] && is_array($decoded['data'])){ - $isJsonl = false; + $keys = array_keys($data); + $looksLikeList = is_int($keys[0]) && is_array($data[$keys[0]]); + if($looksLikeList && !$this->isRecordList($data)){ + return [[], "insert data must be either one record or a list of records"]; + } + $items = $looksLikeList ? array_values($data) : [$data]; + return $this->io->withExactFloats(function() use ($items){ + foreach($items as $item){ + if($this->hasReservedKeyField($item)){ + return [[], "You cannot set key name to key"]; + } + if($this->io->encodeJson($item) === false){ + return [[], "insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"]; } } - } - - $this->jsonlFormatCache[$path] = $isJsonl; - return $isJsonl; + return [$items, null]; + }); } /** - * Ensure a segment is in JSONL format (auto-migrates the pre-3.0 format) + * Insert one record or a list of records: each gets the next key * @param string $dbname - * @param int|null $shardId - * @return bool + * @param array $data One record, or a list of records + * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] */ - private function ensureJsonlFormat($dbname, $shardId = null){ - $path = $this->segDataPath($dbname, $shardId); - if($this->isJsonlFormat($path)){ - return true; + public function insert($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + list($items, $error) = $this->validateInsertData($data); + if($error !== null){ + return array("n" => 0, "error" => $error); } - return $this->migrateToJsonl($path, $dbname, $shardId); + return $this->locks->withLock($dbname, true, function() use ($dbname, $items){ + if($this->bufferEnabled){ + return $this->buffer->insertBuffered($dbname, $items); + } + $this->buffer->flushPendingBuffers($dbname); + return $this->writer->insertRecords($dbname, $items); + }, 'insert'); } /** - * Migrate a pre-3.0 {"data": [...]} file to JSONL (exclusive lock) - * @param string $path Source file path - * @param string $dbname Database name - * @param int|null $shardId Shard ID or null for non-sharded - * @return bool Success + * Delete the records matching the filters + * @param string $dbname + * @param array $data Filters (same rules as find); [] deletes every record + * @return array ['n' => deleted count] */ - private function migrateToJsonl($path, $dbname, $shardId = null){ - return $this->withLock($dbname, true, function() use ($path, $dbname, $shardId){ - unset($this->jsonlFormatCache[$path]); - if($this->isJsonlFormat($path)){ - return true; // migrated by someone else meanwhile - } - - $data = json_decode((string)@file_get_contents($path), true); - if(!is_array($data) || !isset($data['data']) || !is_array($data['data'])){ - return false; - } - - $index = $this->newJsonlIndex(); - $base = ($shardId !== null) ? ($shardId * $this->shardSizeOf($dbname)) : 0; - $buffer = ''; - $offset = 0; - $slots = 0; - foreach($data['data'] as $localKey => $record){ - $slots = max($slots, (int)$localKey + 1); - if(!is_array($record)){ - continue; // deleted (null) or invalid slot: not stored, not garbage - } - $key = $base + (int)$localKey; - unset($record['key']); - $record['key'] = $key; - $json = $this->encodeJson($record); - if($json === false){ - continue; - } - $index['o'][$key] = [$offset, strlen($json)]; - $buffer .= $json . "\n"; - $offset += strlen($json) + 1; - } - $index['n'] = $slots; - - if(!$this->writeFileAtomic($path, $buffer)){ - return false; - } - $this->jsonlFormatCache[$path] = true; - $this->writeJsonlIndex($dbname, $index, $shardId); - return true; - }); + public function delete($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + if(!is_array($data)){ + return array("n" => 0, "error" => "Please check your delete parameters"); + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $data){ + return $this->deleteUnlocked($dbname, $data); + }, 'delete'); } - // ========================================== - // RECORD I/O - // ========================================== + private function deleteUnlocked($dbname, $data){ + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + return array("n" => 0); + } + $deleted = 0; + foreach($this->findFilter->segmentMatches($dbname, $data) as $segment => $records){ + $deleted += $this->writer->deleteSegmentRecords($dbname, $segment, $records); + } + return array("n" => $deleted); + } /** - * Batch read multiple JSONL records (one open, 64KB buffered sequential reads) - * @param string $path File path - * @param array $offsets [key => [offset, length], ...] - * @param bool $mismatch Set to true when a line doesn't hold the expected record - * @return array [key => record, ...] + * Set fields of the records matching the filters + * @param string $dbname + * @param array $data [filters, ['set' => [field => value, ...]]] + * @return array ['n' => updated count] */ - private function readJsonlRecordsBatch($path, $offsets, &$mismatch = false){ - $mismatch = false; - if(empty($offsets)){ - return []; + public function update($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + if(!is_array($data) || !array_key_exists(0, $data) || !is_array($data[0]) + || !isset($data[1]) || !is_array($data[1]) || !isset($data[1]['set']) || !is_array($data[1]['set'])){ + return array("n" => 0, "error" => "Please check your update parameters"); } - - $handle = @fopen($path, 'rb'); - if($handle === false){ - $mismatch = true; - return []; + $error = $this->updateSetError($data[1]['set']); + if($error !== null){ + return $error; } + return $this->locks->withLock($dbname, true, function() use ($dbname, $data){ + return $this->updateUnlocked($dbname, $data); + }, 'update'); + } - $records = []; - $bufferSize = 65536; - $buffer = ''; - $bufferStart = -1; - $bufferEnd = -1; + private function updateUnlocked($dbname, $data){ + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + return array("n" => 0); + } + $updated = 0; + foreach($this->findFilter->segmentMatches($dbname, $data[0]) as $segment => $records){ + $updated += $this->updateRecords($dbname, $segment, $records, $data[1]['set']); + } + return array("n" => $updated); + } - // Read in file order. Appends keep the index in that order; updated records moved to the end. - $positions = []; - $inOrder = true; - $previous = -1; - foreach($offsets as $key => $location){ - $positions[$key] = $location[0]; - if($location[0] < $previous){ - $inOrder = false; - } - $previous = $location[0]; + /** + * The result of update() for fields that can't be set ("key", values JSON can't represent); null if they can + * @return array|null + */ + public function updateSetError(array $set){ + if(array_key_exists("key", $set)){ + return array("n" => 0, "error" => "Please check your update parameters"); } - if(!$inOrder){ - asort($positions, SORT_NUMERIC); + if($this->io->encodeJson($set) === false){ + return array("n" => 0, "error" => "update data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"); } + return null; + } - foreach($positions as $key => $position){ - $offset = (int)$position; - $length = (int)$offsets[$key][1]; - if($length <= 0){ - $mismatch = true; - continue; - } - - if($offset >= $bufferStart && ($offset + $length) <= $bufferEnd){ - $json = substr($buffer, $offset - $bufferStart, $length); - } else { - fseek($handle, $offset, SEEK_SET); - $buffer = (string)fread($handle, max($length, $bufferSize)); - $bufferStart = $offset; - $bufferEnd = $offset + strlen($buffer); - $json = substr($buffer, 0, $length); - } - - $record = json_decode((string)$json, true); - if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ - $mismatch = true; - continue; + /** + * Set fields of records of one segment as read ([key => record]): update() and the query builder's update(). + * An empty set changes nothing and counts the records. Caller holds the exclusive lock. + * @return int + */ + public function updateRecords($dbname, $segment, array $records, array $set){ + if(empty($set)){ + return count($records); + } + foreach($records as $key => $record){ + foreach($set as $field => $value){ + $record[$field] = $value; } $records[$key] = $record; } - - fclose($handle); - return $records; + return $this->writer->updateSegmentRecords($dbname, $segment, $records); } +} +} +namespace noneDB\Internal\Core { - /** - * Read records of one segment, verified against their keys. A stale or corrupt index is - * rebuilt from the data file once. - * @param array|null $keys Keys to read (null = all) - * @return array [key => record] sorted by key - */ - private function readSegmentRecords($dbname, $shardId, $keys = null){ - for($attempt = 0; $attempt < 2; $attempt++){ - $index = $attempt === 0 - ? $this->readJsonlIndex($dbname, $shardId) - : $this->repairSegmentIndex($dbname, $shardId, true); - if($index === null){ - return []; - } - if($keys === null){ - $offsets = $index['o']; - } else { - $offsets = []; - foreach($keys as $key){ - if(isset($index['o'][$key])){ - $offsets[$key] = $index['o'][$key]; - } - } - } - $mismatch = false; - $records = $this->readJsonlRecordsBatch($this->segDataPath($dbname, $shardId), $offsets, $mismatch); - if(!$mismatch){ - ksort($records, SORT_NUMERIC); - return $records; - } - } - throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); +use function array_map, array_slice, bin2hex, count, file_get_contents, filesize, floatval, implode, is_array, + is_bool, is_int, is_link, is_string, max, microtime, min, pow, preg_match, random_bytes, round, scandir, sort, + str_replace, strval; +use const COUNT_RECURSIVE, SORT_STRING; + +/** + * @internal + */ +final class DatabaseOps { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $dbDir; + private $indexCache; + private $verifiedIndexes; + private $buffer; + private $cache; + private $checkpoints; + private $layout; + private $locks; + private $scope; + private $segments; + private $shardMeta; + private $store; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->layout, $this->locks, $this->scope, $this->segments, $this->shardMeta, $this->store); } /** - * Append records ([key => record]) to a segment and point the index at them. - * Everything is encoded before anything is written, so an unencodable record writes nothing. - * @throws \InvalidArgumentException|\RuntimeException + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function appendSegmentRecords($dbname, $shardId, array &$index, array $records){ - if(empty($records)){ - return; - } - $this->markIndexesPending($dbname, $shardId); - $lines = []; - foreach($records as $key => $record){ - unset($record['key']); - $record['key'] = (int)$key; - $lines[$key] = $this->encodeRecord($record); - } - $buffer = implode("\n", $lines) . "\n"; - $dataPath = $this->segDataPath($dbname, $shardId); - $offset = $this->appendToFile($dataPath, $buffer); - if($offset === false){ - throw new \RuntimeException("noneDB: failed to write data of database '{$dbname}'"); - } - $this->markFileExists($dataPath); - foreach($lines as $key => $json){ - $index['o'][$key] = [$offset, strlen($json)]; - $offset += strlen($json) + 1; - } + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->dbDir = &$this->k->dbDir; + $this->indexCache = &$this->k->indexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; } /** - * Append tombstones for deleted keys, so an index rebuilt from the data won't resurrect them + * A handle to another service, bound on first use */ - private function appendTombstones($dbname, $shardId, array $keys){ - if(empty($keys)){ - return; - } - $this->markIndexesPending($dbname, $shardId); - $buffer = ''; - foreach($keys as $key){ - $buffer .= '{"__nonedb_deleted":' . (int)$key . "}\n"; - } - if($this->appendToFile($this->segDataPath($dbname, $shardId), $buffer) === false){ - throw new \RuntimeException("noneDB: failed to write data of database '{$dbname}'"); - } + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Create an empty segment (data file + index) - * @param string $dbname - * @param int|null $shardId - * @return bool + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function createJsonlDatabase($dbname, $shardId = null){ - $path = $this->segDataPath($dbname, $shardId); - clearstatcache(true, $path); - if(!file_exists($path) && @touch($path) === false){ - throw new \RuntimeException("noneDB: cannot create data file of database '{$dbname}'"); - } - $this->markFileExists($path); - $this->jsonlFormatCache[$path] = true; - $index = $this->newJsonlIndex(); - if($shardId === null){ - $existing = $this->atomicRead($path . '.jidx', null); - if(is_array($existing) && isset($existing['o'])){ - return true; - } - } - $this->writeJsonlIndex($dbname, $index, $shardId); - return true; + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->store = null; + $this->k = null; + $this->rt = null; } // ========================================== - // WRITE PATHS (callers hold the exclusive lock) + // PUBLIC INDEX API // ========================================== /** - * Insert validated records. Assigns keys, writes data + indexes, updates field/spatial indexes - * and migrates to sharded format when the size threshold is reached. - * @param array $items Records without "key" - * @return array ['n' => int] or ['n' => 0, 'error' => string] + * Make the key indexes (.kix checkpoints) of a database again from its data files, then its field and + * spatial indexes where they changed. + * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. + * @param string $dbname + * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] */ - private function insertRecords($dbname, array $items, $startKey = null){ - if(empty($items)){ - return ["n" => 0]; + public function rebuildIndex($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->rebuildIndexUnlocked($dbname); + }, 'rebuildIndex'); + } + + private function rebuildIndexUnlocked($dbname){ + $start = microtime(true); + $this->buffer->flushPendingBuffers($dbname); + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return ['success' => false, 'error' => 'Database does not exist']; } - if(!$this->dbExists($dbname)){ - if(!$this->autoCreateDB){ - return ["n" => 0, "error" => "Database does not exist"]; - } - $this->createDBUnlocked($dbname); + $total = 0; + foreach($segments as $segment){ + $total += \noneDB\Internal\Storage\KeyIndex::count($this->checkpoints->rebuildSegmentIndexFromData($dbname, $segment)); + $this->cache->evictBetweenSegments(); } - - // With a start key the records get keys startKey, startKey+1, ... and keys that are already - // stored are skipped, so replaying the same insert after a crash never duplicates anything. - $keyed = null; - $endKey = null; - if($startKey !== null){ - $keyed = []; - foreach(array_values($items) as $i => $item){ - $keyed[(int)$startKey + $i] = $item; - } - $endKey = (int)$startKey + count($items); - foreach(array_keys($this->selectRecords($dbname, ['key' => array_keys($keyed)])) as $existing){ - unset($keyed[$existing]); - } + if($this->segments->isSharded($dbname)){ + $this->shardMeta->recountMeta($dbname); } + // After a manual restore the field and spatial indexes are as stale as the key index + $this->checkpoints->rebuildSecondaryIndexes($dbname, $segments); + return [ + 'success' => true, + 'totalRecords' => $total, + 'time' => round((microtime(true) - $start) * 1000, 2) . 'ms' + ]; + } - if($this->isSharded($dbname)){ - $this->insertShardedRecords($dbname, $items, $keyed, $endKey); - return ["n" => count($items)]; - } + /** + * Get index information for a database + * @param string $dbname + * @return array|null + */ + public function getIndexInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getIndexInfoUnlocked($dbname); + }); + } - $index = $this->readJsonlIndex($dbname, null); - if($index === null){ - $this->createJsonlDatabase($dbname, null); - $index = $this->readJsonlIndex($dbname, null); + private function getIndexInfoUnlocked($dbname){ + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return null; } - - if($keyed !== null){ - $records = $keyed; - $index['n'] = max((int)$index['n'], $endKey); - } else { - $records = []; - foreach($items as $item){ - $records[(int)$index['n']] = $item; - $index['n']++; + $total = 0; + $created = null; + $updated = null; + foreach($segments as $segment){ + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + if($index === null){ + continue; } + $total += \noneDB\Internal\Storage\KeyIndex::count($index); + $created = $created === null ? ($index['created'] ?? null) : min($created, $index['created'] ?? $created); + $updated = max((int)$updated, (int)($index['updated'] ?? 0)); + unset($index); + $this->cache->evictBetweenSegments(); } - $this->appendSegmentRecords($dbname, null, $index, $records); - $this->writeJsonlIndex($dbname, $index, null); - $this->afterRecordsChanged($dbname, null, [], $this->withKeys($records)); - - if($this->shardingEnabled && $this->autoMigrate && $index['n'] >= $this->shardSize){ - $this->migrateToSharded($dbname); - } - return ["n" => count($items)]; + $sharded = $this->segments->isSharded($dbname); + return [ + 'exists' => true, + 'version' => 1, // of the key-index checkpoint (docs/STORAGE_FORMAT.md §6.4) + 'created' => $created, + 'updated' => $updated, + 'totalRecords' => $total, + 'sharded' => $sharded, + 'path' => $sharded ? $this->shardMeta->getMetaPath($dbname) : $this->checkpoints->getJsonlIndexPath($dbname, null) + ]; } /** - * Insert into a sharded database. Keys come from meta.nextKey; key K lives in shard intdiv(K, shardSize). + * True when the database exists. With autoCreateDB a missing database is created; without it nothing is + * created and the answer is false. + * @param string $dbname + * @return bool + * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] */ - private function insertShardedRecords($dbname, array $items, $keyed = null, $endKey = null){ - $meta = $this->getCachedMeta($dbname, true); - if($meta === null){ - throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + public function checkDB($dbname=null){ + if($dbname === null || $dbname === '' || $dbname === false){ + return false; } - $size = (int)$meta['shardSize']; - $nextKey = (int)$meta['nextKey']; - - $byShard = []; - if($keyed !== null){ - foreach($keyed as $key => $item){ - $byShard[intdiv($key, $size)][$key] = $item; - } - $nextKey = max($nextKey, (int)$endKey); - $items = $keyed; - } else { - foreach($items as $item){ - $byShard[intdiv($nextKey, $size)][$nextKey] = $item; - $nextKey++; - } + $dbname = $this->layout->normalizeDbName($dbname); + if(!$this->autoCreateDB){ + // Only a check: a database that doesn't exist creates nothing (no directory, no lock file) + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->segments->dbExists($dbname); + }); } + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->checkDBUnlocked($dbname); + }, 'checkDB'); + } - // Reserve the keys first: a crash later leaves a gap, never a reused key - $shardPos = []; - foreach($meta['shards'] as $i => $shard){ - $shardPos[(int)$shard['id']] = $i; - } - foreach($byShard as $shardId => $records){ - if(!isset($shardPos[$shardId])){ - $meta['shards'][] = ["id" => $shardId, "file" => "_s" . $shardId, "count" => 0, "deleted" => 0]; - $shardPos[$shardId] = count($meta['shards']) - 1; - } - $meta['shards'][$shardPos[$shardId]]['count'] += count($records); - } - // Journal first: if we die after reserving, recovery recounts the meta from the shards - foreach(array_keys($byShard) as $shardId){ - $this->markIndexesPending($dbname, $shardId); + private function checkDBUnlocked($dbname){ + if($this->segments->dbExists($dbname)){ + return true; } - $previousMeta = $this->getCachedMeta($dbname); - $meta['nextKey'] = $nextKey; - $meta['totalRecords'] = (int)$meta['totalRecords'] + count($items); - $this->writeMeta($dbname, $meta); - - try { - foreach($byShard as $shardId => $records){ - $index = $this->readJsonlIndex($dbname, $shardId); - if($index === null){ - $this->createJsonlDatabase($dbname, $shardId); - $index = $this->readJsonlIndex($dbname, $shardId); - } - $this->appendSegmentRecords($dbname, $shardId, $index, $records); - $index['n'] = max((int)$index['n'], max(array_keys($records)) - $shardId * $size + 1); - $this->writeJsonlIndex($dbname, $index, $shardId); - $this->afterRecordsChanged($dbname, $shardId, [], $this->withKeys($records)); - } - } catch(\Throwable $e){ - // Nothing was appended for the failing shard; restore counters (keys stay reserved) - if($previousMeta !== null){ - $previousMeta['nextKey'] = $nextKey; - $this->writeMeta($dbname, $previousMeta); - $this->recountMeta($dbname); - } - throw $e; + if($this->autoCreateDB){ + return $this->createDBUnlocked($dbname); } + return false; } /** - * Records with their "key" field set + * Create a database (STORAGE_FORMAT.md §8.1) + * @param string $dbname + * @return bool True if created, false if it already existed */ - private function withKeys(array $records){ - foreach($records as $key => $record){ - unset($records[$key]['key']); - $records[$key]['key'] = (int)$key; - } - return $records; + public function createDB($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->createDBUnlocked($dbname); + }, 'createDB'); } /** - * Replace records ([key => full new record]) in one segment - * @return int Number of records written + * Create a database (STORAGE_FORMAT.md §8.1): generation 1 with an empty main segment and its first + * checkpoint, then CURRENT, then FORMAT. Caller holds the database lock exclusively (and STORE.LOCK + * shared: a database that doesn't exist is only locked through the missing-database path). */ - private function updateSegmentRecords($dbname, $shardId, array $oldRecords, array $newRecords){ - if(empty($newRecords)){ - return 0; - } - $index = $this->readJsonlIndex($dbname, $shardId); - foreach(array_keys($newRecords) as $key){ - if(!isset($index['o'][$key])){ - unset($newRecords[$key], $oldRecords[$key]); - } - } - if(empty($newRecords)){ - return 0; + public function createDBUnlocked($dbname){ + if($this->segments->dbExists($dbname)){ + return false; } - $this->appendSegmentRecords($dbname, $shardId, $index, $newRecords); - $index['d'] += count($newRecords); - $this->writeJsonlIndex($dbname, $index, $shardId); - $this->afterRecordsChanged($dbname, $shardId, $oldRecords, $this->withKeys($newRecords)); - $this->maybeCompact($dbname, $shardId); - return count($newRecords); + $this->layout->createDatabaseFiles($dbname); + $dataPath = $this->segments->getDataPath($dbname); + $this->cache->markFileExists($dataPath); + $index = $this->checkpoints->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] = $index; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] = true; + $this->checkpoints->writeCheckpoint($dbname, null, $index); + return true; } /** - * Delete records ([key => record]) from one segment - * @return int Number of records deleted - */ - private function deleteSegmentRecords($dbname, $shardId, array $oldRecords){ - if(empty($oldRecords)){ - return 0; + * Delete a database (STORAGE_FORMAT.md §8.2): every file but its lock file + * @param string $dbname + * @return array ['success' => bool, 'files' => int deleted file count] + */ + public function dropDB($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + // Reads of a missing database create nothing: only an existing one is locked exclusively + if(!$this->locks->withLock($dbname, false, function() use ($dbname){ return $this->segments->dbExists($dbname); })){ + return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; } - $index = $this->readJsonlIndex($dbname, $shardId); - foreach(array_keys($oldRecords) as $key){ - if(!isset($index['o'][$key])){ - unset($oldRecords[$key]); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + if(!$this->segments->dbExists($dbname)){ + return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; + } + $this->scope->refuseInUncommittedWrite($dbname, 'dropDB'); + $drop = $this->layout->dropDatabaseFiles($dbname, $this->locks->getLockPath($dbname), true); + if(!$drop['committed']){ + return ['success' => false, 'files' => 0, 'error' => "Database '{$dbname}' could not be dropped: its files can't be deleted (" + . implode(', ', $drop['left']) . ')']; + } + return ['success' => true, 'files' => $drop['removed']]; + }, 'dropDB'); + } + + /** + * Name, creation time and total size of a database (all shards included) + * @return array|false + */ + private function getDBInfo($dbname){ + return $this->locks->withLock($dbname, false, function() use ($dbname){ + $format = $this->layout->readFormatRecord($dbname); + if($format === null){ + return false; + } + $size = 0; + foreach($this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'segments/', '/\.jsonl$/') as $file){ + $size += (int)@filesize($file); + } + $info = array("name" => $dbname, "createdTime" => $format['createdAt'], "size" => $this->fileSizeConvert($size)); + // Shard files found without a shards.json entry are kept aside (see setOrphanShardsAside) + $orphans = array_map('basename', $this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'segments/', '/\.orphan\.\d+$/')); + if(!empty($orphans)){ + $info['orphanFiles'] = $orphans; + } + return $info; + }); + } + /** + * Convert bytes to human readable format + * @param float $bytes + * @return string + */ + private function fileSizeConvert($bytes){ + $bytes = floatval($bytes); + $arBytes = array( + 0 => array("UNIT" => "TB", "VALUE" => pow(1024, 4)), + 1 => array("UNIT" => "GB", "VALUE" => pow(1024, 3)), + 2 => array("UNIT" => "MB", "VALUE" => pow(1024, 2)), + 3 => array("UNIT" => "KB", "VALUE" => 1024), + 4 => array("UNIT" => "B", "VALUE" => 1), + ); + $result = "0 B"; + foreach($arBytes as $arItem){ + if($bytes >= $arItem["VALUE"]){ + $result = $bytes / $arItem["VALUE"]; + $result = str_replace(".", "," , strval(round($result, 2)))." ".$arItem["UNIT"]; + break; } } - if(empty($oldRecords)){ - return 0; + return $result; + } + + /** + * The databases of this key: names (false), names with information (true), or one database's information + * (its name; false when it doesn't exist) + */ + public function getDBs($info=false){ + // Handle three cases: false (names only), true (with metadata), string (specific db) + $withMetadata = false; + $specificDb = null; + + if(is_bool($info)){ + $withMetadata = $info; + }else{ + $specificDb = $this->layout->normalizeDbName($info); // reading information never creates the database } - $keys = array_keys($oldRecords); - $this->appendTombstones($dbname, $shardId, $keys); - foreach($keys as $key){ - unset($index['o'][$key]); + + // If specific database requested + if($specificDb !== null){ + return $this->getDBInfo($specificDb); } - $index['d'] += count($keys); // the old record lines - $index['t'] = (int)($index['t'] ?? 0) + count($keys); // tombstone lines - $this->writeJsonlIndex($dbname, $index, $shardId); - $this->afterRecordsChanged($dbname, $shardId, $oldRecords, []); - if($shardId !== null){ - $meta = $this->getCachedMeta($dbname, true); - foreach($meta['shards'] as $i => $shard){ - if((int)$shard['id'] === $shardId){ - $meta['shards'][$i]['count'] = max(0, $meta['shards'][$i]['count'] - count($keys)); - $meta['shards'][$i]['deleted'] += count($keys); + // List all databases of this key: {locator}.ndb4 directories whose FORMAT names a database that + // maps to that locator with this key + $dbs = []; + if(!$this->store->openStore(false)){ + return $dbs; + } + $names = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; // a link is no database (STORAGE_FORMAT.md §3.4) + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + $dbname = is_array($format) && is_string($format['name'] ?? null) ? $format['name'] : null; + if($dbname === null || preg_match("/^[A-Za-z0-9' -]+$/", $dbname) !== 1 || $this->store->locator($dbname) !== $m[1]){ + continue; + } + $names[] = $dbname; + } + sort($names, SORT_STRING); + foreach($names as $dbname){ + if($withMetadata){ + $info = $this->getDBInfo($dbname); + if($info !== false){ + $dbs[] = $info; } + }else{ + $dbs[] = $dbname; } - $meta['totalRecords'] = max(0, (int)$meta['totalRecords'] - count($keys)); - $meta['deletedCount'] = (int)($meta['deletedCount'] ?? 0) + count($keys); - $this->writeMeta($dbname, $meta); } - - $this->maybeCompact($dbname, $shardId); - return count($keys); + return $dbs; } /** - * Keep field and spatial indexes in sync after records of a segment changed - * @param array $removed [key => old record] - * @param array $added [key => new record] + * The first $limit records of a list of records (a result of find()) + * @param array $array + * @param int $limit At least 1 + * @return array|false False for invalid arguments */ - private function afterRecordsChanged($dbname, $shardId, array $removed, array $added){ - $this->applyFieldIndexChanges($dbname, $shardId, $removed, $added); - $this->applySpatialIndexChanges($dbname, $shardId, $removed, $added); + public function limit($array, $limit=0){ + if(!is_array($array) || !is_int($limit) || $limit <= 0){ + return false; + } + if(empty($array)){ + return []; + } + // Must be a list of records + if(count($array) === count($array, COUNT_RECURSIVE)) { + return false; + } + return array_slice($array, 0, $limit); } +} +} +namespace noneDB\Internal\Diagnostics { - /** - * Compact a segment once garbage exceeds the threshold (relative to live records) +use function array_chunk, array_column, array_diff_key, array_fill_keys, array_intersect_key, array_key_exists, array_keys, array_map, array_merge, array_slice, + array_values, clearstatcache, count, ctype_digit, fclose, fgets, file_exists, file_get_contents, filesize, + fopen, is_array, is_file, is_int, is_string, json_decode, ksort, max, scandir, sort, strlen, substr, time; +use const SORT_NUMERIC, SORT_STRING; + +/** + * @internal + */ +final class Integrity { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $integrityChecks; + private $lastIntegrity; + private $lockRegistry; + private $noWrites; + private $buffer; + private $cache; + private $checkpoints; + private $fieldIndex; + private $geoJson; + private $geometry; + private $io; + private $layout; + private $locks; + private $records; + private $recoveryLog; + private $rtree; + private $secondary; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->geoJson, $this->geometry, $this->io, $this->layout, $this->locks, $this->records, $this->recoveryLog, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->integrityChecks = &$this->rt->integrityChecks; + $this->lastIntegrity = &$this->rt->lastIntegrity; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recoveryLog = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Check a database against its own data, read-only: every line of every data file is + * decoded; the key index, field indexes, spatial indexes and shard meta are compared with what the + * data says. Nothing is written, repaired, flushed or recovered (rebuildIndex() repairs). + * @param string $dbname + * @return array ['ok' => bool, 'segments', 'lines', 'records', 'badLines' => [[segment, offset, length]], + * 'tornTail', 'indexMismatches', 'bufferedRecords', 'pendingRecovery', 'quarantineFiles', + * 'recoveryLog' => last entries] */ - private function maybeCompact($dbname, $shardId){ - $index = $this->readJsonlIndex($dbname, $shardId); - if($index === null){ - return; - } - $garbage = (int)$index['d']; - if($garbage > 0 && $garbage > (count($index['o']) + $garbage) * $this->jsonlGarbageThreshold){ - $this->compactJsonl($dbname, $shardId); + public function checkIntegrity($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + $report = $this->integrityReport($dbname); + // The result getMetrics() reports (docs/OPERATIONS.md §1), by database instance: a database dropped and + // created again is another one + $this->lastIntegrity = ['database' => $dbname, 'ok' => $report['ok'], 'time' => time(), 'records' => $report['records'], + 'issues' => count($report['badLines']) + count($report['tornTail']) + count($report['indexMismatches'])]; + $format = $this->layout->readFormatRecord($dbname); + if($format !== null){ + $this->integrityChecks[$format['databaseId']] = $this->lastIntegrity; } + return $report; } /** - * Compact a segment: rewrite live records (keys and order preserved), drop old versions and - * tombstones. Crash-safe: the data file is replaced first; if the index rename doesn't happen, - * the shorter data file is detected and the index rebuilt from it. - * @param string $dbname - * @param int|null $shardId - * @return array ['compacted' => int, 'freed' => int] + * The report of checkIntegrity(), not kept for getMetrics() (restore() verifies a generation with it) */ - private function compactJsonl($dbname, $shardId = null){ - return $this->withLock($dbname, true, function() use ($dbname, $shardId){ - $index = $this->readJsonlIndex($dbname, $shardId); - if($index === null){ - return ['compacted' => 0, 'freed' => 0]; + public function integrityReport($dbname){ + $check = function() use ($dbname){ + return $this->checkIntegrityLocked($dbname); + }; + $this->noWrites++; + try { + // A missing database is answered without a lock: nothing is created + return $this->locks->withLock($dbname, false, $check); + } finally { + $this->noWrites--; + } + } + + private function checkIntegrityLocked($dbname){ + $report = ['ok' => true, 'database' => $dbname, 'exists' => false, 'segments' => 0, 'lines' => 0, 'records' => 0, + 'badLines' => [], 'tornTail' => [], 'indexMismatches' => [], 'bufferedRecords' => 0, 'pendingRecovery' => false, + 'quarantineFiles' => [], 'recoveryLog' => []]; + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + if($lock !== null && (!empty($lock['flags']['j']) || !empty($lock['flags']['c']) || !empty($lock['flags']['d']) + || !empty($lock['flags']['g']) || $lock['epoch'] === null)){ + $report['pendingRecovery'] = true; // the next writer or reader recovers (journal, creation, drop, generation) + } + if(!$this->layout->dbPresent($dbname)){ + $report['ok'] = !$report['pendingRecovery']; + return $report; + } + // Generations other than CURRENT's: left by an interrupted switch, never removed without proof + $current = $this->layout->generationNumber($dbname); + foreach(@scandir($this->layout->dbHome($dbname) . 'generations') ?: [] as $generation){ + if(ctype_digit((string)$generation) && (int)$generation !== $current){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'generation', 'detail' => "generation $generation is not the current one ($current)"]; } - $path = $this->segDataPath($dbname, $shardId); - $records = $this->readSegmentRecords($dbname, $shardId); - $index = $this->readJsonlIndex($dbname, $shardId); + } + $segPrefix = $this->layout->genDir($dbname) . 'segments/s'; + $metaPath = $this->shardMeta->getMetaPath($dbname); + clearstatcache(true, $metaPath); + $sharded = file_exists($metaPath); + $meta = $sharded ? $this->shardMeta->readMeta($dbname) : null; + if($sharded && $meta === null){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'meta file unreadable']; + } + $segments = $sharded ? array_map(function($f) use ($segPrefix){ return (int)substr($f, strlen($segPrefix)); }, + $this->segments->listFilesWithPrefix($segPrefix, '/^\d+\.jsonl$/')) : (is_file($this->segments->getDataPath($dbname)) ? [null] : []); + sort($segments); + $report['exists'] = !empty($segments) || $sharded; + $size = $meta !== null ? (int)$meta['shardSize'] : $this->segments->shardSizeOf($dbname); + if($sharded && $meta === null){ + try { + $size = (int)$this->shardMeta->deriveShardMeta($dbname)['shardSize']; // as recovery would rebuild it + } catch(\RuntimeException $e){ + } + } + $manifest = $this->deriveOrReadManifest($dbname); + $liveTotal = 0; + $maxKey = -1; + $byShard = []; - $newIndex = $this->newJsonlIndex(); - $newIndex['created'] = $index['created'] ?? time(); - $newIndex['n'] = $index['n']; - $buffer = ''; + $done = false; // a segment was checked: the cache may drop what it loaded before the next one + foreach($segments as $shardId){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $report['segments']++; + $seg = $shardId === null ? 'main' : $shardId; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + // The live records of the data: key => offset, key => length (as a key index holds them) + $livePos = []; + $liveLen = []; $offset = 0; - foreach($records as $key => $record){ - $json = $this->encodeRecord($record); - $newIndex['o'][$key] = [$offset, strlen($json)]; - $buffer .= $json . "\n"; - $offset += strlen($json) + 1; + $fp = @fopen($dataPath, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $report['tornTail'][] = ['segment' => $seg, 'offset' => $offset, 'length' => strlen($line)]; + break; + } + $report['lines']++; + $entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $shardId === null ? 0 : $shardId * $size, + $shardId === null ? null : ($shardId + 1) * $size); + if($entry === null){ + $report['badLines'][] = [$seg, $offset, strlen($line)]; + } elseif($entry[0] === 't'){ + unset($livePos[$entry[1]], $liveLen[$entry[1]]); + $maxKey = max($maxKey, $entry[1]); + } else { + $livePos[$entry[1]] = $offset; + $liveLen[$entry[1]] = $entry[3]; + $maxKey = max($maxKey, $entry[1]); + } + $offset += strlen($line); + } + if($fp !== false){ + fclose($fp); + } + ksort($livePos, SORT_NUMERIC); + $liveTotal += count($livePos); + $byShard[$shardId === null ? 'main' : $shardId] = count($livePos); + + // Key index: its checkpoint fits the data file, and with the complete lines after it + // gives the same records at the same places. A missing one is rebuilt from the data on the next + // access (normal: nothing lost), a broken one too, but that is reported. + $kixPath = \noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath); + clearstatcache(true, $kixPath); + $raw = is_file($kixPath) && filesize($kixPath) > 0 ? @file_get_contents($kixPath) : null; // empty: missing + $file = $raw === null ? null : \noneDB\Internal\Storage\KeyIndex::decodeFile($raw); + clearstatcache(true, $dataPath); + $fileSize = (int)@filesize($dataPath); + $index = null; + if($raw !== null && $file === null){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'not a valid checkpoint (rebuilt on next access)']; + } elseif($file !== null){ + if(!$this->checkpoints->checkpointFits($dataPath, $file[0]['s'], $fileSize, $file[0]['h'] ?? null)){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'checkpoint does not fit the data file (rebuilt on next access)']; + } else { + $index = \noneDB\Internal\Storage\KeyIndex::fromFile($file[0], $file[1]); + } + } + if($index !== null){ + $this->checkpoints->replaySegment($dbname, $shardId, $index, $fileSize, null, false); + [$indexPos, $indexLen] = \noneDB\Internal\Storage\KeyIndex::locate($index, null); + if($indexPos != $livePos || $indexLen != $liveLen){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => count(array_diff_key($livePos, $indexPos)) . ' records not indexed, ' + . count(array_diff_key($indexPos, $livePos)) . ' indexed records not in the data or elsewhere']; + } + $indexPos = $indexLen = null; } - if(!$this->writeFileAtomic($path, $buffer)){ - throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data"); + // Field and spatial indexes against the records: what they must hold, from the records read a batch + // at a time (RecordIO::BATCH, in key order) + $expectedFields = array_fill_keys($manifest['fields'], []); + $expectedSpatial = array_fill_keys($manifest['spatial'], []); + foreach(empty($manifest['fields']) && empty($manifest['spatial']) ? [] : array_chunk($livePos, \noneDB\Internal\Storage\RecordIO::BATCH, true) as $chunk){ + $records = $this->records->readJsonlRecordsBatch($dataPath, $chunk, array_intersect_key($liveLen, $chunk)); + ksort($records, SORT_NUMERIC); + foreach($manifest['fields'] as $field){ + foreach($records as $key => $record){ + if(array_key_exists($field, $record) && $this->fieldIndex->isIndexableValue($record[$field])){ + $bucket = \noneDB\Internal\Query\Values::valueKey($record[$field]); + $held = $expectedFields[$field][$bucket] ?? null; + if($held === null){ + $expectedFields[$field][$bucket] = (int)$key; + } elseif(is_int($held)){ + $expectedFields[$field][$bucket] = [$held, (int)$key]; + } else { + $expectedFields[$field][$bucket][] = (int)$key; + } + } + } + } + foreach($manifest['spatial'] as $field){ + foreach($records as $key => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid'] && $this->geometry->calculateMBR($geometry) !== null){ + $expectedSpatial[$field][] = (int)$key; + } + } + } + $records = null; + } + // Field and spatial indexes: one that fits the segment (its epoch, the bytes it covers), with the + // lines after it replayed, must hold exactly the records; one that doesn't fit is rebuilt when + // used (not damage). + $keyEpoch = is_array($index) && isset($index['e']) && is_string($index['e']) ? $index['e'] : null; + foreach($manifest['fields'] as $field){ + $actual = $this->integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + continue; // built on first use + } + \noneDB\Internal\Index\FieldIndex::canonicalBuckets($expectedFields[$field]); + \noneDB\Internal\Index\FieldIndex::canonicalBuckets($actual); + if($actual !== $expectedFields[$field]){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'field ' . $field, 'detail' => 'buckets differ from the records']; + } + } + foreach($manifest['spatial'] as $field){ + $expected = $expectedSpatial[$field]; + $actual = $this->integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + continue; + } + sort($actual); + if($actual !== $expected){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'spatial ' . $field, 'detail' => 'entries differ from the records']; + } } - $this->writeJsonlIndex($dbname, $newIndex, $shardId); + } + $report['records'] = $liveTotal; - return ['compacted' => count($records), 'freed' => (int)$index['d']]; - }); + if($meta !== null){ + $listed = array_map('intval', array_column($meta['shards'], 'id')); + sort($listed); + if($listed !== $segments){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'shards listed in the meta differ from the shard files']; + } + if((int)($meta['totalRecords'] ?? -1) !== $liveTotal){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "totalRecords {$meta['totalRecords']}, data has $liveTotal"]; + } + if((int)$meta['nextKey'] <= $maxKey){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "nextKey {$meta['nextKey']} would reuse key $maxKey"]; + } + } + + foreach(array_merge([$this->buffer->getBufferPath($dbname)], $this->buffer->flushingFiles($dbname)) as $buffer){ + if(is_file($buffer)){ + $report['bufferedRecords'] += count($this->buffer->readBufferRecords($buffer)); + } + } + $report['quarantineFiles'] = array_map('basename', $this->segments->listFilesWithPrefix($this->layout->dbHome($dbname) . 'recovery/q.', '/\.bin$/')); + $report['recoveryLog'] = array_slice($this->recoveryLog->getRecoveryLogEntries($dbname), -20); + $report['ok'] = empty($report['badLines']) && empty($report['tornTail']) && empty($report['indexMismatches']) && !$report['pendingRecovery']; + return $report; } /** - * Convert a non-sharded database to sharded format (keys preserved, indexes rebuilt per shard) - * @param string $dbname - * @return bool + * The index list without writing anything (checkIntegrity) */ - private function migrateToSharded($dbname){ - return $this->withLock($dbname, true, function() use ($dbname){ - if($this->isSharded($dbname)){ - return true; - } - $legacyPath = $this->getDataPath($dbname); - if(!$this->cachedFileExists($legacyPath)){ - return false; - } - $this->flushPendingBuffers($dbname); + private function deriveOrReadManifest($dbname){ + $path = $this->fieldIndex->getIndexManifestPath($dbname); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? [] : json_decode($content, true); + if(!is_array($manifest) || ($content !== false && ($manifest['kind'] ?? null) !== 'indexes')){ + $manifest = $this->fieldIndex->deriveIndexManifest($dbname); + } + return ['fields' => array_values($manifest['fields'] ?? []), 'spatial' => array_values($manifest['spatial'] ?? [])]; + } - $index = $this->readJsonlIndex($dbname, null); - if($index === null){ - return false; - } - $records = $this->readSegmentRecords($dbname, null); - $size = $this->shardSize; - $maxKey = empty($records) ? -1 : max(array_keys($records)); - $nextKey = max((int)$index['n'], $maxKey + 1); - $numShards = max(1, intdiv(max($nextKey, 1) - 1, $size) + 1); + /** + * checkIntegrity(): buckets of a segment's field index that fits the key checkpoint, with the + * data lines up to $complete replayed; null when there is none that fits + */ + private function integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->secondary->readSecondaryFile($this->fieldIndex->getFieldIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'fx', $field, $keyEpoch, $complete)){ + return null; + } + $buckets = $this->fieldIndex->decodeFieldIndexBody($file[0], $file[1]); + if($buckets === null){ + return []; // pages that can't be read hold none of the records + } + $index = $this->fieldIndex->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $this->fieldIndex->replayFieldIndex($dbname, $shardId, $index, $complete); + return $index['b']; + } - $byShard = array_fill(0, $numShards, []); - foreach($records as $key => $record){ - $byShard[intdiv($key, $size)][$key] = $record; - } + /** + * checkIntegrity(): keys of a segment's R-tree that fits the key checkpoint, replayed up to + * $complete; null when there is none that fits + */ + private function integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->secondary->readSecondaryFile($this->rtree->getSpatialIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'rx', $field, $keyEpoch, $complete)){ + return null; + } + $tree = $this->rtree->treeOf($field, $file[0], $file[1]); + if($tree === null){ + return []; // a body that can't be read holds none of the records + } + $this->rtree->replaySpatialIndex($dbname, $shardId, $tree, $complete); + $this->rtree->rtreeMaps($tree); + return array_keys($tree['leafOf']); + } - $meta = [ - "version" => 1, - "shardSize" => $size, - "totalRecords" => count($records), - "deletedCount" => 0, - "nextKey" => $nextKey, - "shards" => [] - ]; + private function integrityHeaderFits($dbname, $shardId, array $header, $kind, $field, $keyEpoch, $complete){ + return ($header['kind'] ?? null) === $kind && ($header['field'] ?? null) === (string)$field && $keyEpoch !== null + && ($header['e'] ?? null) === $keyEpoch && is_int($header['s'] ?? null) && $header['s'] >= 0 && $header['s'] <= $complete + && $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), $header['s']) === ($header['h'] ?? null); + } +} +} +namespace noneDB\Internal\Diagnostics { - foreach($byShard as $shardId => $shardRecords){ - $slots = min($nextKey, ($shardId + 1) * $size) - $shardId * $size; - $shardIndex = $this->newJsonlIndex(); - $shardIndex['n'] = max(0, $slots); - $buffer = ''; - $offset = 0; - foreach($shardRecords as $key => $record){ - $json = $this->encodeRecord($record); - $shardIndex['o'][$key] = [$offset, strlen($json)]; - $buffer .= $json . "\n"; - $offset += strlen($json) + 1; - } - $shardPath = $this->getShardPath($dbname, $shardId); - if(!$this->writeFileAtomic($shardPath, $buffer)){ - throw new \RuntimeException("noneDB: migration of database '{$dbname}' failed while writing shard {$shardId}"); - } - $this->markFileExists($shardPath); - $this->jsonlFormatCache[$shardPath] = true; - $this->writeJsonlIndex($dbname, $shardIndex, $shardId); - - $meta['shards'][] = [ - "id" => $shardId, - "file" => "_s" . $shardId, - "count" => count($shardRecords), - "deleted" => 0 - ]; - } +use function array_sum, array_values, count, filesize, in_array, is_dir, is_file, is_link, memory_get_peak_usage, + memory_get_usage, preg_match, scandir, strlen, strncmp, substr; - // Flip to sharded only after every shard is safely on disk - $this->writeMeta($dbname, $meta); +/** + * @internal + */ +final class Metrics { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; - // Verify, then remove the non-sharded files - $verified = 0; - foreach(array_keys($byShard) as $shardId){ - $verified += count($this->readSegmentRecords($dbname, $shardId)); - } - if($verified !== count($records)){ - throw new \RuntimeException("noneDB: migration of database '{$dbname}' could not be verified"); - } - @unlink($legacyPath . '.jidx'); - @unlink($legacyPath); - @unlink($legacyPath . '.backup'); - $this->markFileNotExists($legacyPath); - unset($this->indexCache[$legacyPath . '.jidx'], $this->jsonlFormatCache[$legacyPath]); + private $integrityChecks; + private $lastIntegrity; + private $lockRegistry; + private $metrics; + private $noWrites; + private $buffer; + private $fieldIndex; + private $layout; + private $locks; + private $shardInfo; + private $sync; - // Indexes: rebuilt per shard from the manifest - $this->rebuildIndexesAfterMigration($dbname); - return true; - }); + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->fieldIndex, $this->layout, $this->locks, $this->shardInfo, $this->sync); } - // ========================================== - // WRITE BUFFER - // ========================================== - // Optional (off by default). Buffered records have no key yet; flushing renames the buffer - // first and then inserts its records through insertRecords(), so the normal indexes and - // sharding logic apply. Pending buffers (also those left by pre-3.2 versions) are flushed - // before every read and write, whether or not buffering is currently enabled. + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->integrityChecks = &$this->rt->integrityChecks; + $this->lastIntegrity = &$this->rt->lastIntegrity; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + } /** - * Get buffer file path of a database - * @param string $dbname - * @return string + * A handle to another service, bound on first use */ - private function getBufferPath($dbname){ - return $this->dbBase($dbname) . ".nonedb.buffer"; + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Check if buffer exists and has content - * @param string $bufferPath - * @return bool + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function hasBuffer($bufferPath){ - clearstatcache(true, $bufferPath); - return file_exists($bufferPath) && filesize($bufferPath) > 0; + public function unbind(): void { + $this->buffer = null; + $this->fieldIndex = null; + $this->layout = null; + $this->locks = null; + $this->shardInfo = null; + $this->sync = null; + $this->k = null; + $this->rt = null; } /** - * Number of records in a buffer file; counted once, then tracked by size + * ['process' => ...] and, for a database, ['database' => ...]: nothing is scanned or written */ - private function getBufferRecordCount($bufferPath){ - clearstatcache(true, $bufferPath); - if(!file_exists($bufferPath)){ - return 0; - } - $size = filesize($bufferPath); - if(isset(self::$bufferCounts[$bufferPath]) && self::$bufferCounts[$bufferPath][0] === $size){ - return self::$bufferCounts[$bufferPath][1]; + public function getMetrics($dbname){ + $metrics = ['process' => $this->processMetrics()]; + if($dbname !== null){ + $metrics['database'] = $this->databaseMetrics($this->layout->normalizeDbName($dbname)); } - $count = count($this->readBufferRecords($bufferPath)); - self::$bufferCounts[$bufferPath] = [$size, $count]; - return $count; + return $metrics; + } + + private function processMetrics(){ + $m = $this->metrics; + $durability = $this->sync->getDurability(); + return [ + 'indexCache' => ['hits' => $m['hits'], 'misses' => $m['misses'], 'evictions' => $m['evictions'], + 'limit' => \noneDB\Internal\Cache\CacheManager::cacheLimitBytes()], + 'indexReads' => ['pages' => $m['pages'], 'bytes' => $m['bytes']], + 'checkpoints' => ['written' => $m['checkpoints'], 'seconds' => $m['checkpointSeconds']], + 'compactions' => ['count' => $m['compactions'], 'seconds' => $m['compactionSeconds']], + 'locks' => ['acquired' => $m['locks'], 'waited' => $m['lockWaits'], 'waitSeconds' => $m['lockWaitSeconds'], 'timeouts' => $m['lockTimeouts']], + 'recovery' => ['events' => $m['recoveryEvents'], 'actions' => $m['recoveryActions']], + 'durability' => ['mode' => $durability['mode'], 'lastOperation' => $durability['lastOperation'], + 'lastSyncError' => $durability['lastSyncError'], 'syncs' => $durability['syncs']], + 'integrity' => $this->lastIntegrity, + 'memory' => ['usage' => memory_get_usage(), 'peak' => memory_get_peak_usage()], + ]; } /** - * Read all records from a buffer file (JSONL) - * @param string $bufferPath - * @return array + * A database from its metadata, under its shared lock, writing nothing (as checkIntegrity() reads) */ - private function readBufferRecords($bufferPath){ - $records = []; - $fp = @fopen($bufferPath, 'rb'); - if($fp === false){ - return $records; - } - while(($line = fgets($fp)) !== false){ - $line = trim($line); - if($line === ''){ - continue; - } - $record = json_decode($line, true); - if(is_array($record)){ - $records[] = $record; - } + private function databaseMetrics($dbname){ + $this->noWrites++; + try { + return $this->locks->withLock($dbname, false, function() use ($dbname){ + if(!$this->layout->dbPresent($dbname)){ + return ['name' => $dbname, 'exists' => false]; // a missing database is answered without a lock + } + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + $flags = $this->lockRegistry[$this->locks->getLockPath($dbname)]['flags']; + $buffered = $this->buffer->getBufferRecordCount($this->buffer->getBufferPath($dbname)); + foreach($this->buffer->flushingFiles($dbname) as $file){ + $buffered += count($this->buffer->readBufferRecords($file)); + } + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $databaseId = $this->layout->readFormatRecord($dbname)['databaseId']; + return [ + 'name' => $dbname, + 'exists' => true, + 'databaseId' => $databaseId, + 'generation' => $this->layout->generationNumber($dbname), + 'layout' => $info['sharded'] ? 'sharded' : 'single', + 'shardSize' => $info['shardSize'], + 'segments' => $info['sharded'] ? $info['shards'] : 1, + 'records' => $info['totalRecords'], + 'nextKey' => $info['nextKey'], + 'bufferedRecords' => $buffered, + 'fieldIndexes' => array_values($manifest['fields']), + 'spatialIndexes' => array_values($manifest['spatial']), + 'bytes' => $this->databaseBytes($dbname), + 'state' => ['buffered' => !empty($flags['b']), + 'recoveryPending' => !empty($flags['j']) || !empty($flags['c']) || !empty($flags['d']) || !empty($flags['g'])], + 'integrity' => $this->integrityChecks[$databaseId] ?? null, + ]; + }); + } finally { + $this->noWrites--; } - fclose($fp); - return $records; } /** - * True if the database has buffered records waiting (current or pre-3.2 layout) + * Sizes of the files of a database directory by their class (docs/OPERATIONS.md §1.2) */ - private function hasPendingBuffers($dbname){ - // Under the lock the state record says it (flag 'b', set before a buffer gets records and - // cleared after they are flushed), so normal operations don't touch the file system for this - if($this->isVerified($dbname)){ - return !empty(self::$lockRegistry[$this->getLockPath($dbname)]['flags']['b']); - } - return $this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)) || !empty($this->legacyShardBufferFiles($dbname)); + private function databaseBytes($dbname){ + $home = $this->layout->dbHome($dbname); + $active = 'generations/' . $this->layout->generationNumber($dbname) . '/'; + $bytes = ['data' => 0, 'indexes' => 0, 'buffer' => 0, 'metadata' => 0, 'recovery' => 0, 'other' => 0]; + $walk = function($dir, $prefix) use (&$walk, &$bytes, $active){ + foreach(@scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..' || is_link($dir . $entry)){ + continue; + } + if(is_dir($dir . $entry)){ + $walk($dir . $entry . '/', $prefix . $entry . '/'); + continue; + } + if(!is_file($dir . $entry)){ + continue; + } + $file = $prefix . $entry; + $inActive = strncmp($file, $active, strlen($active)) === 0; + $relative = $inActive ? substr($file, strlen($active)) : null; + if($inActive && preg_match('#^segments/[^/]+\.jsonl$#', $relative) === 1){ + $class = 'data'; + } elseif($inActive && preg_match('#^(?:segments/[^/]+\.kix|indexes/[^/]+\.(?:fx|gfx)|spatial/[^/]+\.(?:rx|grx))$#', $relative) === 1){ + $class = 'indexes'; + } elseif($inActive && strncmp($relative, 'buffer/', 7) === 0){ + $class = 'buffer'; + } elseif(in_array($inActive ? $relative : $file, $inActive ? ['metadata', 'shards.json', 'indexes.json'] : ['FORMAT', 'CURRENT', 'LOCK'], true)){ + $class = 'metadata'; + } elseif(strncmp($file, 'recovery/', 9) === 0){ + $class = 'recovery'; + } else { + $class = 'other'; + } + $bytes[$class] += (int)@filesize($dir . $entry); + } + }; + $walk($home, ''); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + $bytes['total'] = array_sum($bytes); + return $bytes; } +} +} +namespace noneDB\Internal\Durability { - /** - * Buffer files whose flush was interrupted: {buffer}.flushing.{firstKey} (and the pre-3.2 {buffer}.flushing) - */ - private function flushingFiles($dbname){ - $bufferPath = $this->getBufferPath($dbname); - $files = glob($bufferPath . '.flushing*') ?: []; - return array_values(array_filter($files, function($f){ - return preg_match('/\.nonedb\.buffer\.flushing(\.\d+)?$/', $f) === 1; - })); +use function array_reverse, basename, bin2hex, chmod, clearstatcache, dirname, fclose, fflush, file_exists, file_get_contents, + file_put_contents, fopen, fseek, ftell, ftruncate, fwrite, ini_get, ini_set, is_dir, is_file, is_link, is_string, + json_decode, json_encode, json_last_error, json_last_error_msg, mkdir, random_bytes, rmdir, rtrim, strlen, + microtime, substr, touch, unlink; +use const JSON_ERROR_NONE, JSON_PRESERVE_ZERO_FRACTION, JSON_PRETTY_PRINT, JSON_THROW_ON_ERROR, + JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE, SEEK_END; + +/** + * @internal + */ +final class FileIO { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $appendMismatch; + private $directoryPermissions; + private $durable; + private $exactFloats; + private $filePermissions; + private $metrics; + private $checkpoints; + private $locks; + private $recovery; + private $scope; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->locks, $this->recovery, $this->scope, $this->sync); } /** - * Pre-3.2 per-shard buffer files (and interrupted flushes) of a database + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function legacyShardBufferFiles($dbname){ - if(!$this->isSharded($dbname)){ - return []; - } - $files = glob($this->dbBase($dbname) . '_s*.nonedb.buffer*') ?: []; - return array_values(array_filter($files, function($f){ - return preg_match('/_s\d+\.nonedb\.buffer(\.flushing(\.\d+)?)?$/', $f) === 1; - })); + public function bind(): void { + $this->appendMismatch = &$this->k->appendMismatch; + $this->directoryPermissions = &$this->k->directoryPermissions; + $this->durable = &$this->k->durable; + $this->filePermissions = &$this->k->filePermissions; + $this->exactFloats = &$this->rt->exactFloats; + $this->metrics = &$this->rt->metrics; } /** - * Flush buffered records into the database (exclusive lock) - * @return int Number of records flushed + * A handle to another service, bound on first use */ - private function flushPendingBuffers($dbname){ - $base = $this->dbBase($dbname); - // Flushing inserts records, which can migrate to shards, which flushes again: don't re-enter - if(isset(self::$flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ - return 0; - } - return $this->withLock($dbname, true, function() use ($dbname, $base){ - self::$flushingNow[$base] = true; - try { - return $this->flushPendingBuffersLocked($dbname); - } finally { - unset(self::$flushingNow[$base]); - } - }); - } - - private function flushPendingBuffersLocked($dbname){ - { - $flushed = 0; - foreach($this->legacyShardBufferFiles($dbname) as $file){ - $flushed += $this->flushLegacyShardBuffer($dbname, $file); - } - - $bufferPath = $this->getBufferPath($dbname); - // Interrupted flushes first. A flush renames the buffer to {buffer}.flushing.{firstKey}: the - // rename records atomically which keys its records get, so replaying it is idempotent. - foreach($this->flushingFiles($dbname) as $file){ - $flushed += $this->flushBufferFile($dbname, $file, preg_match('/\.flushing\.(\d+)$/', $file, $m) ? (int)$m[1] : null); - } - if($this->hasBuffer($bufferPath)){ - if(!$this->dbExists($dbname)){ - $this->createDBUnlocked($dbname); - } - $startKey = $this->nextKeyOf($dbname); - $flushingPath = $bufferPath . '.flushing.' . $startKey; - if(!@rename($bufferPath, $flushingPath)){ - throw new \RuntimeException("noneDB: could not flush the write buffer of database '{$dbname}'"); - } - $flushed += $this->flushBufferFile($dbname, $flushingPath, $startKey); - } - unset(self::$bufferCounts[$bufferPath]); - $this->setBufferFlag($dbname, false); - $this->bufferLastFlush[$dbname] = time(); - return $flushed; - } + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Insert the records of one buffer file, starting at $startKey (null: pre-3.2 file, next free keys) - * @return int + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function flushBufferFile($dbname, $file, $startKey){ - $records = $this->readBufferRecords($file); - if(!empty($records)){ - $result = $this->insertRecords($dbname, $records, $startKey); - if(isset($result['error'])){ - throw new \RuntimeException("noneDB: flushing the write buffer of '{$dbname}' failed: " . $result['error']); - } - } - @unlink($file); - return count($records); + public function unbind(): void { + $this->checkpoints = null; + $this->locks = null; + $this->recovery = null; + $this->scope = null; + $this->sync = null; + $this->k = null; + $this->rt = null; } + // ========================================== + // ATOMIC FILE OPERATIONS + // ========================================== + /** - * The key the next inserted record gets + * A derived file's header line, its crc last + * @return string|false */ - private function nextKeyOf($dbname){ - if($this->isSharded($dbname)){ - $meta = $this->getCachedMeta($dbname, true); - return $meta === null ? 0 : (int)$meta['nextKey']; - } - $index = $this->readJsonlIndex($dbname, null); - return $index === null ? 0 : (int)$index['n']; + public function encodeHeader(array $header){ + $json = $this->encodeJson($header); + return $json === false ? false : \noneDB\Internal\Storage\KeyIndex::sealHeader($json); } + /** - * Flush a pre-3.2 shard buffer. Its records were already counted in the meta (count, totalRecords - * and nextKey) when they were buffered; they take the next free slots of their shard. - * @return int + * Encode JSON, returning false instead of producing invalid output + * @return string|false */ - private function flushLegacyShardBuffer($dbname, $file){ - if(!preg_match('/_s(\d+)\.nonedb\.buffer(\.flushing(\.\d+)?)?$/', $file, $m)){ - return 0; - } - $shardId = (int)$m[1]; - $size = $this->shardSizeOf($dbname); - $index = $this->readJsonlIndex($dbname, $shardId); - if($index === null){ - $this->createJsonlDatabase($dbname, $shardId); - $index = $this->readJsonlIndex($dbname, $shardId); + public function encodeJson($data, $prettyPrint = false, $extraFlags = 0){ + $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | $extraFlags; + if($prettyPrint){ + $flags |= JSON_PRETTY_PRINT; } - // Same idempotent scheme as the current buffer: the first key goes into the file name - if(preg_match('/\.flushing\.(\d+)$/', $file, $km)){ - $startKey = (int)$km[1]; - } else { - $startKey = $shardId * $size + (int)$index['n']; - $renamed = preg_replace('/(\.flushing)?$/', '', $file, 1) . '.flushing.' . $startKey; - if(!@rename($file, $renamed)){ - throw new \RuntimeException("noneDB: could not flush a write buffer of database '{$dbname}'"); + // json_encode() writes floats with serialize_precision digits: anything but -1 (the default, + // shortest exact form) would store rounded values (0.30000000000000004 as 0.3 at 10). + // Loops encoding many records run inside withExactFloats(): no ini_get()/finally per record. + if($this->exactFloats || ini_get('serialize_precision') === '-1'){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; } - $file = $renamed; } - $records = array_values(array_filter($this->readBufferRecords($file), function($r){ - return is_array($r) && !array_key_exists('key', $r); - })); - if(!empty($records)){ - $keyed = []; - foreach($records as $i => $record){ - $keyed[$startKey + $i] = $record; - } - foreach(array_keys($keyed) as $key){ - if(isset($index['o'][$key])){ - unset($keyed[$key]); // stored before an interruption - } + return $this->withExactFloats(function() use ($data, $flags){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; } - $this->appendSegmentRecords($dbname, $shardId, $index, $keyed); - $index['n'] = max((int)$index['n'], $startKey + count($records) - $shardId * $size); - $this->writeJsonlIndex($dbname, $index, $shardId); - $this->afterRecordsChanged($dbname, $shardId, [], $this->withKeys($keyed)); - $this->recountMeta($dbname); - } - @unlink($file); - return count($records); + }); } /** - * Append validated records to the database's write buffer (exclusive lock) - * @return array + * Run $fn with serialize_precision set to -1 (checked once instead of once per encoded value) */ - private function insertBuffered($dbname, array $validItems){ - if(!$this->dbExists($dbname)){ - if(!$this->autoCreateDB){ - return ["n" => 0, "error" => "Database does not exist"]; - } - $this->createDBUnlocked($dbname); - } - $this->registerShutdownHandler(); - - $lines = ''; - foreach($validItems as $item){ - $lines .= $this->encodeRecord($item) . "\n"; - } - - $bufferPath = $this->getBufferPath($dbname); - if($this->shouldFlushBuffer($bufferPath, $dbname)){ - $this->flushPendingBuffers($dbname); + public function withExactFloats(callable $fn){ + if($this->exactFloats){ + return $fn(); } - $count = $this->getBufferRecordCount($bufferPath); - $this->setBufferFlag($dbname, true); - if($this->appendToFile($bufferPath, $lines) === false){ - return ["n" => 0, "error" => "Failed to write buffer"]; + $precision = ini_get('serialize_precision'); + if($precision !== '-1'){ + ini_set('serialize_precision', '-1'); } - clearstatcache(true, $bufferPath); - self::$bufferCounts[$bufferPath] = [filesize($bufferPath), $count + count($validItems)]; - if(!isset($this->bufferLastFlush[$dbname])){ - $this->bufferLastFlush[$dbname] = time(); + $this->exactFloats = true; + try { + return $fn(); + } finally { + $this->exactFloats = false; + if($precision !== '-1'){ + ini_set('serialize_precision', (string)$precision); + } } + } - if($this->shouldFlushBuffer($bufferPath, $dbname)){ - $this->flushPendingBuffers($dbname); + /** + * Encode one JSONL record line (without the trailing newline) + * @throws \InvalidArgumentException when the record can't be represented as JSON (invalid UTF-8, NAN, INF, ...) + */ + public function encodeRecord(array $record){ + // Keep floats floats: 2.0 / -0.0 are stored as 2.0 / -0.0, not as the ints 2 / 0 + $json = $this->encodeJson($record, false, JSON_PRESERVE_ZERO_FRACTION); + if($json === false){ + throw new \InvalidArgumentException('noneDB: record cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value): ' . json_last_error_msg()); } - return ["n" => count($validItems)]; + return $json; } /** - * Check if buffer needs flushing (size, count, or time based) - * @param string $bufferPath - * @param string $flushKey Key for tracking last flush time - * @return bool - */ - private function shouldFlushBuffer($bufferPath, $flushKey){ - if(!$this->hasBuffer($bufferPath)){ + * Write a whole file atomically: temp file in the same directory + rename. + * Readers see either the old or the new content, never a partial file. + * @param string|iterable $content The bytes, or a sequence of chunks (a large derived file written as it is + * made); a chunk that can't be made throws, and the temp file is removed + * @param string|bool $mode What the file is (docs/DURABILITY.md §3): + * 'derived' (false): checkpoints and indexes, rebuilt from the data if lost: never synced + * 'durable': records of the database (FORMAT, CURRENT, metadata, shards.json, indexes.json): synced in + * durable mode: temp file, rename (the commit point), directory + * 'replace' (true): a data file replaced (compaction, the switch to shards): synced in both modes; a failed + * directory sync after the rename only counts in durable mode + * @return bool False when the file could not be written (nothing changed) + * @throws \RuntimeException DURABILITY_FAILED (a sync before the rename failed, nothing changed), + * OUTCOME_UNKNOWN (the directory sync after the rename failed) + */ + public function writeFileAtomic($path, $content, $mode = 'derived', $note = true){ + $started = microtime(true); + $mode = $mode === true ? 'replace' : ($mode === false ? 'derived' : $mode); + if($mode === 'replace'){ + $this->scope->refuseUncommittedFile($path, 'replacement'); + } + if($note){ + $this->locks->noteWrite($path); + } + $required = $this->durable && $mode !== 'derived'; + $sync = $required || $mode === 'replace'; + $dir = dirname($path); + if(!is_dir($dir) && !$this->makeDirectory($dir)){ return false; } - if(filesize($bufferPath) >= $this->bufferSizeLimit){ - return true; + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + if(!$sync && is_string($content)){ + $written = @file_put_contents($tmp, $content); + if($written === false || $written !== strlen($content)){ + @unlink($tmp); + return false; + } + $this->setPermissions($tmp); + } else { + $fp = @fopen($tmp, 'xb'); + if($fp === false){ + return false; + } + try { + $ok = true; + foreach(is_string($content) ? [$content] : $content as $chunk){ + if(!$this->writeAll($fp, $chunk, $path)){ + $ok = false; + break; + } + } + } catch(\Throwable $e){ + fclose($fp); + @unlink($tmp); + throw $e; + } + $ok = $ok && fflush($fp); + $synced = $ok && $sync ? $this->sync->syncHandle($fp, $path) : null; + fclose($fp); + if(!$ok){ + @unlink($tmp); + return false; + } + if($required){ + $this->sync->noteRequired('file', $synced === true); + } + if($sync && $synced !== true){ + @unlink($tmp); + throw $this->sync->failure('file', $path, false); + } + $this->setPermissions($tmp); } - if($this->getBufferRecordCount($bufferPath) >= $this->bufferCountLimit){ - return true; + if(!$this->sync->renameWithRetry($tmp, $path)){ + @unlink($tmp); + return false; } - if($this->bufferFlushInterval > 0){ - $lastFlush = $this->bufferLastFlush[$flushKey] ?? time(); - if((time() - $lastFlush) >= $this->bufferFlushInterval){ - return true; + clearstatcache(true, $path); + if($sync){ + $dirSynced = $this->sync->syncDirectory($dir) === true; + if($required){ + $this->sync->noteRequired('dir', $dirSynced); + if(!$dirSynced){ + throw $this->sync->failure('dir', $dir, true); + } } } - return false; + if($mode === 'derived'){ + // A checkpoint or an index file (getMetrics()) + $this->metrics['checkpoints']++; + $this->metrics['checkpointSeconds'] += microtime(true) - $started; + } + return true; } /** - * Register shutdown handler for flushing all buffers + * Write a file derived from the data (a checkpoint) atomically without moving the generation: its + * content is the same whoever writes it, so a reader may write it under the shared lock */ - private function registerShutdownHandler(){ - if($this->shutdownHandlerRegistered){ - return; - } - if($this->bufferAutoFlushOnShutdown){ - // Weak reference: the handler must not keep the instance alive until shutdown - $ref = \WeakReference::create($this); - register_shutdown_function(static function() use ($ref){ - $db = $ref->get(); - if($db !== null){ - $db->flushAllBuffers(); - } - }); - $this->shutdownHandlerRegistered = true; - } + public function writeDerivedFile($path, $content){ + return $this->writeFileAtomic($path, $content, 'derived', false); } - // ========================================== - // FILTER MATCHING (find / update / delete) - // ========================================== - /** - * Normalize a "key" filter value to a list of int keys (invalid values match nothing) - * @return array + * fwrite() every byte (a short write is continued; no progress is a failure) */ - private function normalizeKeyFilter($value){ - $values = is_array($value) ? $value : [$value]; - $keys = []; - foreach($values as $v){ - if(is_int($v)){ - $k = $v; - } elseif(is_string($v) && preg_match('/^(0|[1-9][0-9]*)$/', $v)){ - $k = (int)$v; - } else { - continue; - } - if($k >= 0 && !isset($keys[$k])){ - $keys[$k] = $k; + private function writeAll($fp, $bytes, $path){ + for($done = 0, $length = strlen($bytes); $done < $length; $done += $n){ + $chunk = $done === 0 ? $bytes : substr($bytes, $done); + $n = fwrite($fp, $chunk); + if($n === false || $n <= 0){ + return false; } } - return array_values($keys); + return true; } /** - * Strict equality filter used by find(), update() and delete(): - * every field must exist in the record and be identical (===) to the filter value. - * "key" filters compare against the record key. + * Create a directory and its missing parents. In durable mode each new directory's parent is synced, + * so the new entries survive a power loss with what is written into them. + * @throws \RuntimeException DURABILITY_FAILED */ - private function matchesFilters(array $record, array $filters){ - foreach($filters as $field => $value){ - if($field === 'key'){ - if(!isset($record['key']) || !in_array($record['key'], $this->normalizeKeyFilter($value), true)){ - return false; - } - continue; - } - if(!array_key_exists($field, $record) || $record[$field] !== $value){ + public function makeDirectory($dir){ + $dir = rtrim($dir, '/'); + $missing = []; + for($at = $dir; $at !== '' && $at !== '.' && $at !== '/' && !is_dir($at); $at = dirname($at)){ + $missing[] = $at; + } + foreach(array_reverse($missing) as $new){ + if(!@mkdir($new, $this->directoryPermissions) && !is_dir($new)){ return false; } + $this->setPermissions($new, true); + $this->sync->requireSync(null, dirname($new), 'dir'); } return true; } /** - * Records matching $filters, in key order (or in the order of a "key" filter) - * @return array [key => record] + * A file or directory noneDB created in dbDir gets the configured permissions, whatever the umask + * (STORAGE_FORMAT.md §3) */ - private function selectRecords($dbname, array $filters){ - $segments = $this->listSegments($dbname); - if(empty($segments)){ - return []; - } + public function setPermissions($path, $directory = false){ + @chmod($path, $directory ? $this->directoryPermissions : $this->filePermissions); + } - $keyFilter = null; - if(array_key_exists('key', $filters)){ - $keyFilter = $this->normalizeKeyFilter($filters['key']); - unset($filters['key']); - if(empty($keyFilter)){ - return []; - } + // File system writes: every write to a database file goes through writeFileAtomic(), + // appendToFile() or the helpers below. Each calls noteWrite() first: the generation moves before the + // file changes, so other processes drop their caches even if this one dies half-way, and a write + // without the database's exclusive lock is a LogicException. SourceWriteGuardTest fails on a raw + // write call anywhere else. + + public function fsUnlink($path){ + clearstatcache(true, $path); + if(!file_exists($path) && !is_link($path)){ + return false; // nothing to remove: no write } + $this->scope->refuseUncommittedFile($path, 'removal'); + $this->locks->noteWrite($path); + $ok = @unlink($path); + clearstatcache(true, $path); + return $ok; + } - $result = []; - if($keyFilter !== null){ - $bySegment = []; - foreach($keyFilter as $key){ - $segment = $this->segmentForKey($dbname, $key); - if($segment !== false){ - $bySegment[$segment === null ? '' : $segment][] = $key; - } - } - $found = []; - foreach($bySegment as $segment => $keys){ - $found += $this->readSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $keys); - } - foreach($keyFilter as $key){ - if(isset($found[$key]) && $this->matchesFilters($found[$key], $filters)){ - $result[$key] = $found[$key]; - } - } - return $result; - } - - $targets = $this->targetShardsFromGlobalIndex($dbname, $filters); - foreach($segments as $segment){ - if($targets !== null && !in_array($segment, $targets, true)){ - continue; - } - $candidates = $this->fieldIndexCandidates($dbname, $segment, $filters); - if($candidates !== null && empty($candidates)){ - continue; - } - foreach($this->readSegmentRecords($dbname, $segment, $candidates) as $key => $record){ - if(empty($filters) || $this->matchesFilters($record, $filters)){ - $result[$key] = $record; - } - } - } - return $result; + public function fsRename($from, $to){ + $this->scope->refuseUncommittedFile($from, 'rename'); + $this->scope->refuseUncommittedFile($to, 'replacement'); + $this->locks->noteWrite($from); + $this->locks->noteWrite($to); + $ok = $this->sync->renameWithRetry($from, $to); + clearstatcache(true, $from); + clearstatcache(true, $to); + return $ok; } - /** - * Group [key => record] by the segment that stores each key - * @return array [segmentId|'' => [key => record]] - */ - private function groupBySegment($dbname, array $records){ - $groups = []; - $sharded = $this->isSharded($dbname); - $size = $this->shardSizeOf($dbname); - foreach($records as $key => $record){ - $groups[$sharded ? intdiv($key, $size) : ''][$key] = $record; + public function fsTouch($path){ + $this->locks->noteWrite($path); + $created = !file_exists($path); + $ok = @touch($path); + if($ok && $created){ + $this->setPermissions($path); } - return $groups; + clearstatcache(true, $path); + return $ok; } - // ========================================== - // FIELD INDEXES - // ========================================== - // Which fields are indexed is recorded in {base}.nonedb.indexes ("manifest"), so new shards and - // field names with any characters are handled. Each segment has one file per field: - // {segment}.nonedb.fidx.{name} where name is the field itself when it matches [A-Za-z0-9_]+ and - // "~" + hex(field) otherwise. Index files of format v2 are complete and exact; older files are - // rebuilt on first use. - - private function getIndexManifestPath($dbname){ - return $this->dbBase($dbname) . '.nonedb.indexes'; + /** Create a directory of a database (and its parents; synced in durable mode) */ + public function fsMkdir($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(is_dir($path)){ + return true; + } + $this->locks->noteWrite($path . '/'); + return $this->makeDirectory($path); } - /** - * Indexed fields of a database: ['fields' => [...], 'spatial' => [...]] - * Databases created before v3.2 have no manifest; it is derived from their index files. - */ - private function getIndexManifest($dbname){ - $path = $this->getIndexManifestPath($dbname); - if(isset($this->fieldIndexCache[$path])){ - return $this->fieldIndexCache[$path]; - } - $manifest = $this->atomicRead($path, false); - if($manifest === false && $this->isVerified($dbname)){ - $manifest = []; // no index list: no indexes (pre-3.2 databases got theirs when verified) - } elseif(!is_array($manifest)){ - $manifest = $this->deriveIndexManifest($dbname); + /** Remove an empty directory of a database */ + public function fsRmdir($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(!is_dir($path)){ + return false; } - $manifest = [ - 'fields' => array_values(array_unique(array_map('strval', $manifest['fields'] ?? []))), - 'spatial' => array_values(array_unique(array_map('strval', $manifest['spatial'] ?? []))), - 'spatialOptions' => is_array($manifest['spatialOptions'] ?? null) ? $manifest['spatialOptions'] : [], - ]; - $this->fieldIndexCache[$path] = $manifest; - return $manifest; + $this->locks->noteWrite($path . '/'); + return @rmdir($path); } - private function saveIndexManifest($dbname, array $manifest){ - $manifest = [ - 'v' => 1, - 'fields' => array_values(array_unique($manifest['fields'] ?? [])), - 'spatial' => array_values(array_unique($manifest['spatial'] ?? [])), - 'spatialOptions' => array_intersect_key($manifest['spatialOptions'] ?? [], array_flip(array_values(array_unique($manifest['spatial'] ?? [])))), - ]; - $path = $this->getIndexManifestPath($dbname); - if(empty($manifest['fields']) && empty($manifest['spatial'])){ - @unlink($path); - } elseif(!$this->atomicWrite($path, $manifest)){ - throw new \RuntimeException("noneDB: failed to write index list of database '{$dbname}'"); + /** Cut a file to $size bytes */ + public function fsTruncate($path, $size){ + $this->locks->noteWrite($path); + $fp = @fopen($path, 'r+b'); + if($fp === false){ + return false; } - $this->fieldIndexCache[$path] = ['fields' => $manifest['fields'], 'spatial' => $manifest['spatial'], 'spatialOptions' => $manifest['spatialOptions']]; + $ok = ftruncate($fp, $size); + fclose($fp); + clearstatcache(true, $path); + return $ok; } /** - * Pre-3.2 databases: read field names from existing index files (their "field" property is exact, - * the file name may have been lossy) + * Append bytes to a file. Returns the offset the bytes start at, or false (nothing written). + * The append commits when its bytes are written and flushed and, in durable mode, the file is synced + * (and its directory when the append created it: an empty file before). A failure before that cuts the + * bytes back, so the file never ends in a torn line (docs/DURABILITY.md §3, §4). + * @param string|null $dbname Database whose rollback scope records the append (data and buffer files) + * @param int|null $expected Size the caller's index says the file has: anything else writes nothing + * and sets $this->appendMismatch + * @return int|false + * @throws \RuntimeException DURABILITY_FAILED (a required sync failed, the bytes were cut back), + * OUTCOME_UNKNOWN (the bytes could not be cut back) */ - private function deriveIndexManifest($dbname){ - $base = $this->dbBase($dbname); - $hashBase = $this->dbDir . $this->hashDBName($dbname); - $fields = []; - foreach(array_merge(glob($base . '.nonedb.fidx.*') ?: [], glob($base . '_s*.nonedb.fidx.*') ?: []) as $file){ - $index = $this->atomicRead($file, null); - if(is_array($index) && isset($index['field']) && is_string($index['field'])){ - $fields[] = $index['field']; - } elseif(preg_match('/\.fidx\.(.+)$/', $file, $m)){ - $fields[] = $this->decodeIndexFieldName($m[1]); + public function appendToFile($path, $bytes, $dbname = null, $expected = null, $segment = null){ + $this->locks->noteWrite($path); + $this->appendMismatch = false; + $fp = @fopen($path, 'c+b'); + if($fp === false){ + // The directory of a file's first append (buffer/) is created then: no check on every append + $dir = dirname($path); + if(is_dir($dir) || !$this->makeDirectory($dir) || ($fp = @fopen($path, 'c+b')) === false){ + return false; } } - $spatial = []; - foreach(array_merge(glob($hashBase . '.nonedb.sidx.*') ?: [], glob($hashBase . '_s*.nonedb.sidx.*') ?: []) as $file){ - $index = $this->atomicRead($file, null); - if(is_string($index)){ - $index = json_decode($index, true); // pre-3.2 files were JSON-encoded twice + try { + if(fseek($fp, 0, SEEK_END) !== 0){ + return false; + } + $offset = ftell($fp); + if($offset === false){ + return false; + } + if($offset === 0){ + $this->setPermissions($path); // created by this append (or empty) + } + if($expected !== null && $offset !== (int)$expected){ + // The file changed behind the cached index. An unterminated tail only (an append that + // never completed): quarantined and cut, then this append goes where the index expects it. + // Anything else: nothing is written, the caller drops its index (rebuilt on next access). + if($dbname === null || $offset < (int)$expected || $this->checkpoints->lastNewlineIn($path, (int)$expected, $offset) >= 0){ + $this->appendMismatch = true; + return false; + } + $this->recovery->quarantineAndCut($dbname, $segment, $path, (int)$expected, $offset); + fseek($fp, 0, SEEK_END); + $offset = ftell($fp); + if($offset !== (int)$expected){ + $this->appendMismatch = true; + return false; + } + } + if($dbname !== null){ + $this->scope->markScopeFile($dbname, $path, $offset, $fp); + } + // Written bytes the stream couldn't hand to the OS (disk full, I/O error): the append failed + if(!$this->writeAll($fp, $bytes, $path) || !fflush($fp)){ + $this->undoAppend($fp, $offset, $path, null); + return false; } - if(is_array($index) && isset($index['field']) && is_string($index['field'])){ - $spatial[] = $index['field']; - } elseif(preg_match('/\.sidx\.(.+)$/', $file, $m)){ - $spatial[] = $this->decodeIndexFieldName($m[1]); + if($dbname !== null){ + if(!$this->sync->trySync($fp, $path)){ + $this->undoAppend($fp, $offset, $path, $this->sync->failure('file', $path, false)); + } + if($offset === 0 && !$this->sync->trySync(null, dirname($path), 'dir')){ + $this->undoAppend($fp, $offset, $path, $this->sync->failure('dir', dirname($path), false)); + } } + return $offset; + } finally { + fclose($fp); + clearstatcache(true, $path); } - return ['fields' => array_values(array_unique($fields)), 'spatial' => array_values(array_unique($spatial))]; } /** - * File-name-safe, reversible encoding of a field name + * Cut a failed append back (synced in durable mode), then throw $cause if there is one. A cut that + * fails leaves the bytes possibly there: OUTCOME_UNKNOWN, so the write's rollback scope tries again + * and the next session recovers; never a plain failure that would leave them to be read as a record. + * @throws \RuntimeException */ - private function encodeIndexFieldName($field){ - $field = (string)$field; - return preg_match('/^[A-Za-z0-9_]+$/', $field) ? $field : '~' . bin2hex($field); - } - - private function decodeIndexFieldName($name){ - if(isset($name[0]) && $name[0] === '~' && ctype_xdigit(substr($name, 1)) && strlen($name) % 2 === 1){ - return hex2bin(substr($name, 1)); + private function undoAppend($fp, $offset, $path, ?\RuntimeException $cause){ + try { + $ok = ftruncate($fp, $offset) && $this->sync->syncUndo($fp, $path); + } catch(\Throwable $e){ + $ok = false; + } + if(!$ok){ + throw new \RuntimeException("noneDB: an append to " . basename($path) . " failed and could not be cut back: the outcome is unknown, read the state again", \noneDB::OUTCOME_UNKNOWN, $cause); + } + if($cause !== null){ + throw $cause; } - return $name; - } - - /** - * Get field index file path - * @param string $dbname Database name - * @param string $field Field name - * @param int|null $shardId Shard ID or null for non-sharded - * @return string Path to field index file - */ - private function getFieldIndexPath($dbname, $field, $shardId = null){ - return $this->segDataPath($dbname, $shardId) . ".fidx." . $this->encodeIndexFieldName($field); } /** - * Read a segment's field index. Returns null when the file is missing or predates v3.2 - * (such files may be incomplete and are rebuilt by ensureSegmentFieldIndex). - * @return array|null + * Read and decode a JSON file + * + * @param string $path File path + * @param mixed $default Value returned when the file doesn't exist or is empty + * @return mixed Decoded data, $default, or null when the file is corrupt */ - private function readFieldIndex($dbname, $field, $shardId = null){ - $path = $this->getFieldIndexPath($dbname, $field, $shardId); - if(array_key_exists($path, $this->fieldIndexCache)){ - return $this->fieldIndexCache[$path]; + public function atomicRead($path, $default = null){ + clearstatcache(true, $path); + if(!is_file($path)){ + return $default; } - $index = $this->atomicRead($path, null); - if(!is_array($index) || ($index['v'] ?? 1) < 2 || !isset($index['values']) || !is_array($index['values'])){ - $index = null; + $content = @file_get_contents($path); + if($content === false || $content === ''){ + return $default; } - $this->fieldIndexCache[$path] = $index; - return $index; + $data = json_decode($content, true); + if($data === null && json_last_error() !== JSON_ERROR_NONE){ + return null; // corrupted + } + return $data !== null ? $data : $default; } /** - * Write field index to file (caller holds the exclusive lock) + * Encode and write a JSON file atomically. Nothing is touched when encoding fails. + * + * @param string $path File path + * @param mixed $data Data to write (will be JSON encoded) + * @param bool $prettyPrint Use JSON_PRETTY_PRINT * @return bool Success */ - private function writeFieldIndex($dbname, $field, $index, $shardId = null){ - $path = $this->getFieldIndexPath($dbname, $field, $shardId); - $index['v'] = 2; - $index['field'] = (string)$field; - $index['updated'] = time(); - if(!$this->atomicWrite($path, $index)){ - unset($this->fieldIndexCache[$path]); - throw new \RuntimeException("noneDB: failed to write field index '{$field}' of database '{$dbname}'"); + public function atomicWrite($path, $data, $prettyPrint = false, $mode = 'derived'){ + $json = $this->encodeJson($data, $prettyPrint); + if($json === false){ + return false; } - $this->fieldIndexCache[$path] = $index; - $this->markFileExists($path); - return true; + return $this->writeFileAtomic($path, $json, $mode); } +} +} +namespace noneDB\Internal\Durability { - /** - * Delete field index file - * @return bool Success - */ - private function deleteFieldIndexFile($dbname, $field, $shardId = null){ - $path = $this->getFieldIndexPath($dbname, $field, $shardId); - unset($this->fieldIndexCache[$path]); - $this->markFileNotExists($path); - clearstatcache(true, $path); - return !file_exists($path) || @unlink($path); +use function basename, clearstatcache, fclose, + file_exists, fopen, fsync, is_array, microtime, rename, rtrim, stream_get_meta_data, strlen, strncmp, substr, + time, usleep; +use const LOCK_EX, PHP_OS_FAMILY; + +/** + * @internal + */ +final class Sync { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dbDir; + private $durable; + private $lockTimeout; + private $lastOperation; + private $lastSyncError; + private $lockRegistry; + private $syncStats; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; } /** - * Get list of indexed fields for a database - * @return array + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function getIndexedFields($dbname, $shardId = null){ - return $this->getIndexManifest($dbname)['fields']; + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->durable = &$this->k->durable; + $this->lockTimeout = &$this->k->lockTimeout; + $this->lastOperation = &$this->rt->lastOperation; + $this->lastSyncError = &$this->rt->lastSyncError; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->syncStats = &$this->rt->syncStats; } /** - * Check if a field has an index - * @return bool + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function isFieldIndexed($dbname, $field, $shardId = null){ - return in_array((string)$field, $this->getIndexedFields($dbname), true); + public function unbind(): void { + $this->k = null; + $this->rt = null; } + /** - * Invalidate field index cache for a database + * fsync() a file handle where the stream supports it (plain files). Never emits a warning: the result + * goes to the durability report. + * @return bool|null Null when it can't be done here */ - private function invalidateFieldIndexCache($dbname, $field = null, $shardId = null){ - if($field !== null){ - unset($this->fieldIndexCache[$this->getFieldIndexPath($dbname, $field, $shardId)]); - return; - } - $prefix = $this->dbDir . $this->hashDBName($dbname); - foreach(array_keys($this->fieldIndexCache) as $key){ - if(strpos($key, $prefix) === 0){ - unset($this->fieldIndexCache[$key]); - } + public function syncHandle($fp, $path = null){ + $meta = @stream_get_meta_data($fp); + if(!is_array($meta) || ($meta['wrapper_type'] ?? '') !== 'plainfile'){ + return null; } + $ok = @fsync($fp); + $this->countSync('file', $ok, $path); + return $ok; } /** - * Bucket key of a field value. Values of different types may share a bucket (1, "1", 1.0); results - * are always re-checked with strict comparison, so that only costs a few extra reads. - * @param mixed $value Field value - * @return string Index key + * fsync() a directory so a new, renamed or deleted entry in it survives a power loss (not possible on + * Windows) + * @return bool|null Null when it can't be done here */ - private function fieldIndexValueKey($value){ - if($value === null){ - return '__null__'; + public function syncDirectory($dir){ + if(PHP_OS_FAMILY === 'Windows'){ + return null; } - if(is_bool($value)){ - return $value ? '__true__' : '__false__'; + $dir = rtrim($dir, '/\\'); + $fp = @fopen($dir . '/', 'r'); + if($fp === false){ + $this->countSync('dir', false, $dir); + return false; + } + try { + $ok = @fsync($fp); + $this->countSync('dir', $ok, $dir); + return $ok; + } finally { + fclose($fp); } - if(is_float($value) && $value == 0.0){ - return '0'; // -0.0 === 0.0 + } + + private function countSync($kind, $ok, $path){ + $this->syncStats[$kind]++; + if(!$ok){ + $this->syncStats['failed']++; + $this->lastSyncError = ['path' => $path === null ? null : $this->relative((string)$path), 'op' => $kind, 'time' => time()]; } - return (string)$value; } /** - * True if a value can be stored in a field index + * A path relative to dbDir (the report and messages never show where dbDir is) */ - private function isIndexableValue($value){ - return is_scalar($value) || is_null($value); + private function relative($path){ + $dir = (string)$this->dbDir; + if($path === rtrim($dir, '/')){ + return '.'; + } + return strncmp($path, $dir, strlen($dir)) === 0 ? substr($path, strlen($dir)) : basename($path); } /** - * Build a segment's field index from its data - * @return array + * A sync durable mode requires (DURABILITY.md §3), done in durable mode only. A failure throws: + * DURABILITY_FAILED before the write's commit point, OUTCOME_UNKNOWN after it. + * @throws \RuntimeException */ - private function buildSegmentFieldIndex($dbname, $shardId, $field){ - $index = ['v' => 2, 'field' => (string)$field, 'created' => time(), 'values' => []]; - if($shardId !== null){ - $index['shardId'] = $shardId; + public function requireSync($fp, $path, $kind = 'file', $afterCommit = false){ + if($this->durable && !$this->trySync($fp, $path, $kind)){ + throw $this->failure($kind, $path, $afterCommit); } - foreach($this->readSegmentRecords($dbname, $shardId) as $key => $record){ - if(array_key_exists($field, $record) && $this->isIndexableValue($record[$field])){ - $index['values'][$this->fieldIndexValueKey($record[$field])][] = (int)$key; - } - } - return $index; } /** - * A segment's field index, (re)built when missing or from an older version (exclusive lock) - * @return array + * A sync durable mode requires, for a caller that undoes its change before it throws. Performance mode: + * true, nothing synced. */ - private function ensureSegmentFieldIndex($dbname, $shardId, $field){ - $index = $this->readFieldIndex($dbname, $field, $shardId); - if($index !== null){ - return $index; + public function trySync($fp, $path, $kind = 'file'): bool { + if(!$this->durable){ + return true; } - return $this->withLock($dbname, true, function() use ($dbname, $shardId, $field){ - unset($this->fieldIndexCache[$this->getFieldIndexPath($dbname, $field, $shardId)]); - $index = $this->readFieldIndex($dbname, $field, $shardId); - if($index === null){ - $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); - $this->writeFieldIndex($dbname, $field, $index, $shardId); - $index = $this->readFieldIndex($dbname, $field, $shardId); - if($shardId !== null){ - $this->rebuildGlobalFieldIndex($dbname, $field); - } - } - return $index; - }); + $ok = ($kind === 'dir' ? $this->syncDirectory($path) : $this->syncHandle($fp, $path)) === true; + $this->noteRequired($kind, $ok); + return $ok; } /** - * Candidate keys of a segment for equality filters, from field indexes. - * Null means "no usable index, scan the segment". - * @return array|null + * Record a sync durable mode required for the running write operation and whether it succeeded + * (getDurability()['lastOperation']) */ - private function fieldIndexCandidates($dbname, $shardId, array $filters){ - if(!$this->fieldIndexEnabled || empty($filters)){ - return null; - } - $indexed = $this->getIndexedFields($dbname); - if(empty($indexed)){ - return null; - } - $candidates = null; - foreach($filters as $field => $value){ - if($field === 'key' || !$this->isIndexableValue($value) || !in_array((string)$field, $indexed, true)){ - continue; - } - $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); - $keys = $index['values'][$this->fieldIndexValueKey($value)] ?? []; - $candidates = $candidates === null ? $keys : array_values(array_intersect($candidates, $keys)); - if(empty($candidates)){ - return []; + public function noteRequired($kind, bool $ok){ + foreach($this->lockRegistry as $lock => $entry){ + if($entry['mode'] === LOCK_EX && isset($entry['syncs'])){ + $this->lockRegistry[$lock]['syncs'][$kind] = ($entry['syncs'][$kind] ?? true) && $ok; } } - return $candidates; } /** - * Apply record changes of one segment to every field index of the database - * @param array $removed [key => old record] - * @param array $added [key => new record] + * Sync the undo of a change in durable mode (a cut back append): true when it is durable or nothing + * needs to be (performance mode) */ - private function applyFieldIndexChanges($dbname, $shardId, array $removed, array $added){ - // Maintained even when fieldIndexEnabled is off (that flag only stops reads from using indexes), - // so turning it back on never exposes a stale index - if(empty($removed) && empty($added)){ - return; + public function syncUndo($fp, $path): bool { + return !$this->durable || $this->syncHandle($fp, $path) === true; + } + + /** + * The exception of a required sync that failed + */ + public function failure($kind, $path, $afterCommit, ?\Throwable $previous = null): \RuntimeException { + $what = ($kind === 'dir' ? 'the directory ' : '') . $this->relative((string)$path); + if($afterCommit){ + return new \RuntimeException("noneDB: fsync of {$what} failed after the commit point: the outcome is unknown, read the state again (docs/DURABILITY.md §2)", \noneDB::OUTCOME_UNKNOWN, $previous); } - foreach($this->getIndexedFields($dbname) as $field){ - $index = $this->readFieldIndex($dbname, $field, $shardId); - if($index === null){ - // Missing (new shard) or pre-3.2 file: build it from the data, which already holds the change - $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); - $this->writeFieldIndex($dbname, $field, $index, $shardId); - if($shardId !== null){ - $this->rebuildGlobalFieldIndex($dbname, $field); - } - continue; - } + return new \RuntimeException("noneDB: fsync of {$what} failed; the write was rolled back (docs/DURABILITY.md §2)", \noneDB::DURABILITY_FAILED, $previous); + } - $touched = []; - $removals = []; - foreach($removed as $key => $record){ - if(array_key_exists($field, $record) && $this->isIndexableValue($record[$field])){ - $removals[$this->fieldIndexValueKey($record[$field])][(int)$key] = true; - } - } - foreach($removals as $valueKey => $keys){ - if(!isset($index['values'][$valueKey])){ - continue; - } - $kept = []; - foreach($index['values'][$valueKey] as $k){ - if(!isset($keys[$k])){ - $kept[] = $k; - } - } - if(empty($kept)){ - unset($index['values'][$valueKey]); - } else { - $index['values'][$valueKey] = $kept; + /** + * rename() that retries where it can fail for a moment: on Windows while another process has the target + * open (5, 10, 20, 40, 80, 160 ms, within lockTimeout). The target is never deleted first. + */ + public function renameWithRetry($from, $to){ + $retry = PHP_OS_FAMILY === 'Windows'; + $started = microtime(true); + foreach([0, 5, 10, 20, 40, 80, 160] as $i => $delay){ + if($delay > 0){ + if(!$retry || ($this->lockTimeout > 0 && microtime(true) - $started + $delay / 1000 > $this->lockTimeout)){ + return false; } - $touched[$valueKey] = true; - } - foreach($added as $key => $record){ - if(array_key_exists($field, $record) && $this->isIndexableValue($record[$field])){ - $valueKey = $this->fieldIndexValueKey($record[$field]); - $index['values'][$valueKey][] = (int)$key; - $touched[$valueKey] = true; + usleep($delay * 1000); + clearstatcache(true, $from); + if(!file_exists($from)){ + return false; } } - if(empty($touched)){ - continue; + if(@rename($from, $to)){ + return true; } - $this->writeFieldIndex($dbname, $field, $index, $shardId); + } + return false; + } - if($shardId !== null){ - $global = $this->readGlobalFieldIndex($dbname, $field); - if($global === null){ - $this->rebuildGlobalFieldIndex($dbname, $field); - continue; - } - $changed = false; - foreach(array_keys($touched) as $valueKey){ - $shards = $global['shardMap'][$valueKey] ?? []; - $has = in_array($shardId, $shards, true); - if(!empty($index['values'][$valueKey]) && !$has){ - $shards[] = $shardId; - $global['shardMap'][$valueKey] = $shards; - $changed = true; - } elseif(empty($index['values'][$valueKey]) && $has){ - $shards = array_values(array_diff($shards, [$shardId])); - if(empty($shards)){ - unset($global['shardMap'][$valueKey]); - } else { - $global['shardMap'][$valueKey] = $shards; - } - $changed = true; + /** + * Record how a write operation ended (getDurability()['lastOperation']) + * @param array $syncs kind => true (every required sync of that kind succeeded) | false + */ + public function finishOperation($op, $dbname, array $syncs, $outcome, $reason = null){ + $required = []; + $completed = []; + if($syncs){ // durable mode only: every write of performance mode ends here with none + foreach(['file', 'dir'] as $kind){ + if(isset($syncs[$kind])){ + $required[] = $kind; + if($syncs[$kind]){ + $completed[] = $kind; } } - if($changed){ - $this->writeGlobalFieldIndex($dbname, $field, $global); - } } } + $this->lastOperation = ['op' => $op, 'db' => (string)$dbname, 'outcome' => $outcome, + 'required' => $required, 'completed' => $completed, 'reason' => $reason]; } - // ==================== GLOBAL FIELD INDEX (shard skip) ==================== - /** - * Get path for global field index file + * Durability of this instance and of the last write operation of the process (docs/DURABILITY.md §6) + * @return array */ - private function getGlobalFieldIndexPath($dbname, $field){ - return $this->dbBase($dbname) . ".nonedb.gfidx." . $this->encodeIndexFieldName($field); + public function getDurability(): array { + $os = \noneDB\Internal\Config\Configuration::osFamily(); + $note = \noneDB\Internal\Config\Configuration::platformNote($os); + return [ + 'mode' => $this->durable ? 'durable' : 'performance', + 'platform' => ['os' => $os, 'durableSupported' => \noneDB\Internal\Config\Configuration::durableSupported(), + 'notes' => $note === '' ? [] : ['durable mode is not supported here: ' . $note]], + 'lastOperation' => $this->lastOperation, + 'lastSyncError' => $this->lastSyncError, + 'syncs' => $this->syncStats, + ]; } +} +} +namespace noneDB\Internal\Index { - /** - * Read global field index (null when missing or predating v3.2) - * @return array|null - */ - private function readGlobalFieldIndex($dbname, $field){ - $path = $this->getGlobalFieldIndexPath($dbname, $field); - if(array_key_exists($path, $this->fieldIndexCache)){ - return $this->fieldIndexCache[$path]; - } - $index = $this->atomicRead($path, null); - if(!is_array($index) || ($index['v'] ?? 1) < 2 || !isset($index['shardMap']) || !is_array($index['shardMap'])){ - $index = null; - } - $this->fieldIndexCache[$path] = $index; - return $index; +use noneDB\Internal\Storage\Segments; + +use function abs, array_fill_keys, array_flip, array_intersect, array_intersect_key, array_key_exists, array_keys, + array_map, array_merge, array_multisort, array_search, array_slice, array_sum, array_unique, array_values, + basename, bin2hex, chr, clearstatcache, count, end, fclose, fgets, file_exists, file_get_contents, filesize, fopen, + fread, fseek, fstat, hash, hex2bin, in_array, intdiv, is_array, is_file, is_float, is_int, is_nan, is_null, is_numeric, + is_scalar, is_string, json_decode, ksort, max, memory_get_usage, min, preg_match, random_bytes, sort, + str_repeat, strcmp, strlen, strncmp, strpos, substr, time, unpack; +use const PHP_SAPI, SORT_NUMERIC, SORT_STRING; + +/** + * @internal + */ +final class FieldIndex { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dataEpochs; + private $dataSizes; + private $fieldIndexCache; + private $fieldIndexEnabled; + private $globalSkip; + private $globalSpatialIndexCache; + private $indexCache; + private $segWrites; + private $verifiedIndexes; + private $cacheGrew; + private $cacheTick; + private $cacheTouch; + private $indexTokens; + private $lockRegistry; + private $metrics; + private $multiLock; + private $noWrites; + private $secondaryChecked; + private $sessionCounter; + private $cache; + private $checkpoints; + private $io; + private $layout; + private $locks; + private $records; + private $rtree; + private $secondary; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->records, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->segWrites = &$this->k->segWrites; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->indexTokens = &$this->rt->indexTokens; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->secondaryChecked = &$this->rt->secondaryChecked; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // FIELD INDEXES + // ========================================== + // Which fields are indexed is recorded in indexes.json of the active generation + // ({"kind":"indexes","v":1,"fields","spatial","spatialOptions"}). A segment's field index + // indexes/{seg}.{token}.fx (paged, docs/STORAGE_FORMAT.md §6.5.1) maps the exact identity of a value + // (valueKey(): "n", "t", "f", "i42", "d", "sAnkara") to the keys that hold it; in memory: + // ['b' => ['n' => [2, 4], 'f' => 10, 'i-3' => [1, 9], 'i42' => 7, 'sAnkara' => 3, 'sIstanbul' => [0, 5, 9]], 's', 'e', ...] + // Buckets are exact (1, 1.0, "1" and true are different values, as for ===) and don't depend on the + // precision, serialize_precision or locale settings. Values that are arrays are not indexed. The + // global field index indexes/{token}.gfx of a sharded database holds a filter of each shard's values as + // of its covered bytes (§6.5.2); a shard's newer lines are checked directly. + + public function getIndexManifestPath($dbname){ + return $this->layout->genPath($dbname, 'indexes.json'); } /** - * Write global field index (caller holds the exclusive lock) + * Indexed fields of a database: ['fields', 'spatial', 'spatialOptions']. No file: no indexes; a file + * that doesn't decode (zero-length after a power loss) is derived from the index files' headers. */ - private function writeGlobalFieldIndex($dbname, $field, $metadata){ - $path = $this->getGlobalFieldIndexPath($dbname, $field); - $metadata['v'] = 2; - $metadata['field'] = (string)$field; - $metadata['updated'] = time(); - if(!$this->atomicWrite($path, $metadata)){ - unset($this->fieldIndexCache[$path]); - throw new \RuntimeException("noneDB: failed to write global field index '{$field}' of database '{$dbname}'"); + public function getIndexManifest($dbname){ + $path = $this->getIndexManifestPath($dbname); + if(isset($this->fieldIndexCache[$path])){ + $this->cacheTouch[$path] = ++$this->cacheTick; + return $this->fieldIndexCache[$path]; } - $this->fieldIndexCache[$path] = $metadata; - return true; + clearstatcache(true, $path); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? [] : json_decode($content, true); + if(!is_array($manifest) || ($content !== false && ($manifest['kind'] ?? null) !== 'indexes')){ + $manifest = $this->deriveIndexManifest($dbname); + } + $manifest = [ + 'fields' => array_values(array_unique(array_map('strval', $manifest['fields'] ?? []))), + 'spatial' => array_values(array_unique(array_map('strval', $manifest['spatial'] ?? []))), + 'spatialOptions' => is_array($manifest['spatialOptions'] ?? null) ? $manifest['spatialOptions'] : [], + ]; + $this->fieldIndexCache[$path] = $manifest; + $this->cacheGrew = true; + return $manifest; } - /** - * Rebuild the shard map of a field from the shard field indexes (exclusive lock) - */ - private function rebuildGlobalFieldIndex($dbname, $field){ - $global = ['v' => 2, 'field' => (string)$field, 'shardMap' => []]; - foreach($this->listSegments($dbname) as $shardId){ - $index = $this->readFieldIndex($dbname, $field, $shardId); - if($index === null){ - $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); - $this->writeFieldIndex($dbname, $field, $index, $shardId); - } - foreach($index['values'] as $valueKey => $keys){ - if(!empty($keys)){ - $global['shardMap'][(string)$valueKey][] = $shardId; - } - } + public function saveIndexManifest($dbname, array $manifest){ + $file = [ + 'kind' => 'indexes', + 'v' => 1, + 'fields' => array_values(array_unique($manifest['fields'] ?? [])), + 'spatial' => array_values(array_unique($manifest['spatial'] ?? [])), + ]; + $file['spatialOptions'] = array_intersect_key($manifest['spatialOptions'] ?? [], array_flip($file['spatial'])); + $path = $this->getIndexManifestPath($dbname); + if(empty($file['fields']) && empty($file['spatial'])){ + $this->io->fsUnlink($path); + } elseif(!$this->io->atomicWrite($path, $file, false, 'durable')){ + throw new \RuntimeException("noneDB: failed to write index list of database '{$dbname}'"); } - $this->writeGlobalFieldIndex($dbname, $field, $global); - return $global; + $this->fieldIndexCache[$path] = ['fields' => $file['fields'], 'spatial' => $file['spatial'], + 'spatialOptions' => $file['spatialOptions']]; } /** - * Shards that can contain matches for the filters (null = all shards / not sharded) - * @return array|null + * An index list that doesn't decode: the fields named in the headers of the generation's .fx/.rx files */ - private function targetShardsFromGlobalIndex($dbname, array $filters){ - if(!$this->fieldIndexEnabled || !$this->isSharded($dbname) || empty($filters)){ - return null; - } - $indexed = $this->getIndexedFields($dbname); - $targets = null; - foreach($filters as $field => $value){ - if($field === 'key' || !$this->isIndexableValue($value) || !in_array((string)$field, $indexed, true)){ + public function deriveIndexManifest($dbname){ + $fields = []; + $spatial = []; + $options = []; + $gen = $this->layout->genDir($dbname); + foreach(array_merge($this->segments->listFilesWithPrefix($gen . 'indexes/', '/\.fx$/'), $this->segments->listFilesWithPrefix($gen . 'spatial/', '/\.rx$/')) as $file){ + $header = $this->secondary->readSecondaryHeader($file); + if(!is_array($header) || !isset($header['field']) || !is_string($header['field'])){ continue; } - $global = $this->readGlobalFieldIndex($dbname, $field); - if($global === null){ - $global = $this->withLock($dbname, true, function() use ($dbname, $field){ - return $this->rebuildGlobalFieldIndex($dbname, $field); - }); + if(substr($file, -3) === '.fx'){ + $fields[] = $header['field']; + } else { + $spatial[] = $header['field']; + if(isset($header['ns'])){ + $options[$header['field']] = ['nodeSize' => (int)$header['ns']]; + } } - $shards = array_map('intval', $global['shardMap'][$this->fieldIndexValueKey($value)] ?? []); - $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); } - return $targets; + return ['fields' => array_values(array_unique($fields)), 'spatial' => array_values(array_unique($spatial)), 'spatialOptions' => $options]; } /** - * Delete global field index file + * Path of a segment's field index */ - private function deleteGlobalFieldIndex($dbname, $field){ - $path = $this->getGlobalFieldIndexPath($dbname, $field); - clearstatcache(true, $path); - if(file_exists($path)){ - @unlink($path); - } - unset($this->fieldIndexCache[$path]); + public function getFieldIndexPath($dbname, $field, $shardId = null){ + return $this->secondary->secondaryPath($dbname, 'fx', $field, $shardId); } /** - * After a migration to sharded format: rebuild every field and spatial index per shard and - * remove the non-sharded index files + * True if a value can be stored in a field index */ - private function rebuildIndexesAfterMigration($dbname){ - $manifest = $this->getIndexManifest($dbname); - foreach($manifest['fields'] as $field){ - $this->deleteFieldIndexFile($dbname, $field, null); - foreach($this->listSegments($dbname) as $shardId){ - $this->writeFieldIndex($dbname, $field, $this->buildSegmentFieldIndex($dbname, $shardId, $field), $shardId); - } - $this->rebuildGlobalFieldIndex($dbname, $field); - } - foreach($manifest['spatial'] as $field){ - $this->rebuildSpatialIndexForDb($dbname, $field); - } - $this->saveIndexManifest($dbname, $manifest); + public function isIndexableValue($value){ + return is_scalar($value) || is_null($value); } - // ========================================== - // PUBLIC INDEX API - // ========================================== - /** - * Enable or disable indexing (kept for backward compatibility; key lookups always use the .jidx index) - * @param bool $enable + * An empty field index. Besides its buckets 'b' (covering the data bytes [0, s)): 'file', the write token of + * the .fx file whose body is exactly these buckets (null: none); 'idsAt', the data bytes since which the set + * of bucket ids is this one (null: unknown) */ - public function enableIndexing($enable = true){ - $this->indexEnabled = (bool)$enable; + public function newFieldIndex($field){ + return ['kind' => 'fx', 'field' => (string)$field, 's' => 0, 'ck' => -1, 'e' => null, 'b' => [], 'mk' => -1, + 'rev' => null, 'views' => null, 'bytes' => 0, 'file' => null, 'idsAt' => null]; } /** - * Check if indexing is enabled - * @return bool + * Build a segment's field index from its data (covers the key index's bytes) + * @return array */ - public function isIndexingEnabled(){ - return $this->indexEnabled; + public function buildSegmentFieldIndex($dbname, $shardId, $field){ + $key = $this->secondary->keyIndexWithEpoch($dbname, $shardId); + $index = $this->newFieldIndex($field); + // fieldIndexApply() inlined: every key occurs once, so a value is only ever added. A value of one key costs + // no array: when every value differs, those would be most of the memory + $field = (string)$field; + $buckets = []; + $lists = []; + foreach($this->records->eachSegmentBatch($dbname, $shardId) as $batch){ + foreach($batch as $k => $record){ + $k = (int)$k; + if($k > $index['mk']){ + $index['mk'] = $k; + } + if(!isset($record[$field]) && !array_key_exists($field, $record)){ + continue; + } + $value = $record[$field]; + if(is_string($value)){ + $bucket = 's' . $value; + } elseif(is_scalar($value) || $value === null){ + $bucket = \noneDB\Internal\Query\Values::valueKey($value); + } else { + continue; + } + if(isset($lists[$bucket])){ + $lists[$bucket][] = $k; + } elseif(isset($buckets[$bucket])){ + $lists[$bucket] = [$buckets[$bucket], $k]; + } else { + $buckets[$bucket] = $k; + } + } + } + foreach($lists as $bucket => $keys){ + $buckets[$bucket] = $keys; + } + $index['b'] = $buckets; + $index['mk'] = $key === null ? -1 : max($index['mk'], \noneDB\Internal\Storage\KeyIndex::maxKey($key)); + $index['s'] = $key === null ? 0 : (int)$key['s']; + $index['e'] = $key === null ? null : $key['e']; + $index['rev'] = null; + return $index; } /** - * Rebuild the key indexes (.jidx) of a database from its data files. - * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. - * @param string $dbname - * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] + * Apply one data line to a field index: the key now holds $record (null: deleted) */ - public function rebuildIndex($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->rebuildIndexUnlocked($dbname); - }); - } - - private function rebuildIndexUnlocked($dbname){ - $start = microtime(true); - $this->flushPendingBuffers($dbname); - $segments = $this->listSegments($dbname); - if(empty($segments)){ - return ['success' => false, 'error' => 'Database does not exist']; + private function fieldIndexApply(array &$index, $key, $record){ + $field = $index['field']; + $new = $record !== null && array_key_exists($field, $record) && $this->isIndexableValue($record[$field]) + ? \noneDB\Internal\Query\Values::valueKey($record[$field]) : null; + if($key > $index['mk']){ + // A key never seen: nothing to remove + $index['mk'] = $key; + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + if($index['rev'] !== null){ + $index['rev'][$key] = $new; + } + } + return; } - $total = 0; - foreach($segments as $segment){ - $this->ensureJsonlFormat($dbname, $segment); - $total += count($this->rebuildSegmentIndexFromData($dbname, $segment)['o']); + if($index['rev'] === null){ + $rev = []; + foreach($index['b'] as $bucket => $keys){ + foreach((array)$keys as $k){ + $rev[$k] = (string)$bucket; + } + } + $index['rev'] = $rev; } - if($this->isSharded($dbname)){ - $this->recountMeta($dbname); + $old = $index['rev'][$key] ?? null; + if($old === $new){ + return; // the value didn't change: nothing to do + } + if($old !== null){ + $index['file'] = null; + $keys = $index['b'][$old]; + if(is_int($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + $index['idsAt'] = null; + } else { + $at = array_search($key, $keys, true); + if($at !== false){ + unset($keys[$at]); + } + $keys = array_values($keys); + if(count($keys) === 1){ + $index['b'][$old] = $keys[0]; + } elseif(empty($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + $index['idsAt'] = null; + } else { + $index['b'][$old] = $keys; + } + } + unset($index['rev'][$key]); + } + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + $index['rev'][$key] = $new; + } + } + + private function fieldIndexAdd(array &$index, $bucket, $key){ + $index['file'] = null; + if(!isset($index['b'][$bucket])){ + $index['b'][$bucket] = $key; + $index['views'] = null; + $index['idsAt'] = null; + } elseif(is_int($index['b'][$bucket])){ + $index['b'][$bucket] = [$index['b'][$bucket], $key]; + } else { + $index['b'][$bucket][] = $key; } - return [ - 'success' => true, - 'totalRecords' => $total, - 'time' => round((microtime(true) - $start) * 1000, 2) . 'ms' - ]; } /** - * Get index information for a database - * @param string $dbname + * A segment's field index, current (the data lines after the bytes it covers are replayed). Null when + * there is none that fits the segment (missing, of another epoch, damaged): see ensureSegmentFieldIndex. * @return array|null */ - public function getIndexInfo($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname){ - return $this->getIndexInfoUnlocked($dbname); - }); - } - - private function getIndexInfoUnlocked($dbname){ - $segments = $this->listSegments($dbname); - if(empty($segments)){ - return null; + private function readFieldIndex($dbname, $field, $shardId = null, $path = null, $key = null){ + // The segment's key index (currentKeyIndex() inlined: this is the hot path of every lookup) + if($key === null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $key = isset($this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]) ? ($this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] ?? null) : null; + if($key !== null){ + $this->metrics['hits']++; + } elseif(($key = $this->checkpoints->readJsonlIndex($dbname, $shardId)) === null){ + return null; + } } - $total = 0; - $created = null; - $updated = null; - foreach($segments as $segment){ - $index = $this->readJsonlIndex($dbname, $segment); - if($index === null){ - continue; + $path = $path ?? $this->getFieldIndexPath($dbname, $field, $shardId); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->fieldIndexCache[$path])){ + $index = $this->fieldIndexCache[$path]; + if($index['s'] === (int)$key['s'] && $index['e'] === $key['e'] && $index['e'] !== null){ + $this->metrics['hits']++; + if(isset($index['due']) && $this->noWrites === 0 && $this->multiLock === 0){ + unset($index['due'], $this->fieldIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + return $this->fieldIndexCache[$path]; + } + return $index; // the usual case: nothing written since + } + if($index['e'] !== null && $index['e'] === $key['e'] && $index['s'] <= (int)$key['s']){ + $this->metrics['hits']++; + if($index['s'] < (int)$key['s']){ + unset($this->fieldIndexCache[$path], $index['due']); // sole owner while it changes + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + } + return $this->fieldIndexCache[$path]; } - $total += count($index['o']); - $created = $created === null ? ($index['created'] ?? null) : min($created, $index['created'] ?? $created); - $updated = max((int)$updated, (int)($index['updated'] ?? 0)); + unset($this->fieldIndexCache[$path]); } - $sharded = $this->isSharded($dbname); - return [ - 'exists' => true, - 'version' => 3, - 'created' => $created, - 'updated' => $updated, - 'totalRecords' => $total, - 'sharded' => $sharded, - 'path' => $sharded ? $this->getMetaPath($dbname) : $this->getJsonlIndexPath($dbname, null) - ]; - } - - /** - * check db - * if auto create db is true will be create db - * if auto create db is false and is not in db dir return false - * @param string $dbname - * @return bool - * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] - */ - public function checkDB($dbname=null){ - if($dbname === null || $dbname === '' || $dbname === false){ - return false; + $this->metrics['misses']++; + $file = $this->secondary->readSecondaryFile($path); + if($file === null || ($file[0]['v'] ?? null) !== 1 || !$this->secondary->secondaryFits($dbname, $shardId, $file[0], 'fx', $field)){ + return null; } - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->checkDBUnlocked($dbname); - }); - } - - private function checkDBUnlocked($dbname){ - $this->ensureDbDir(); - - // Existing database (non-sharded data file or sharded meta) - $fullDBPath = $this->getDataPath($dbname); - if($this->cachedFileExists($fullDBPath) || $this->cachedFileExists($this->getMetaPath($dbname))){ - return true; + $buckets = $this->decodeFieldIndexBody($file[0], $file[1]); + if($buckets === null){ + return null; + } + unset($this->fieldIndexCache[$path . '#pg']); // the whole index replaces its pages + $index = $this->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['ck'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $index['bytes'] = strlen($file[1]); + $index['file'] = $file[0]['w']; + $index['idsAt'] = $file[0]['s']; + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + $this->cacheGrew = true; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; } + return $this->fieldIndexCache[$path]; + } - if($this->autoCreateDB){ - return $this->createDBUnlocked($dbname); + public function replayFieldIndex($dbname, $shardId, array &$index, $to){ + foreach($this->secondary->segmentTail($dbname, $shardId, $index['s'], $to) as [$k, $record]){ + $this->fieldIndexApply($index, $k, $record); } - return false; + $index['s'] = $to; } /** - * create db function - * @param string $dbname - * @return bool True if created, false if it already existed + * Write a field index checkpoint (exclusive lock or a reader: the content is the data's) */ - public function createDB($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->createDBUnlocked($dbname); - }); - } - - private function createDBUnlocked($dbname){ - $this->ensureDbDir(); - $fullDBPath = $this->getDataPath($dbname); - if($this->cachedFileExists($fullDBPath) || $this->cachedFileExists($this->getMetaPath($dbname))){ + public function writeFieldIndex($dbname, $field, array &$index, $shardId = null){ + if($index['e'] === null){ return false; } - - // Create info file - @file_put_contents($fullDBPath . "info", (string)time()); - - // v3.0: Create empty JSONL format database - if(@touch($fullDBPath) === false){ - throw new \RuntimeException("noneDB: cannot create database file for '{$dbname}'"); + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$index['s']); + if($crc === null){ + return false; } - - // Create empty JSONL index - $index = [ - 'v' => 3, - 'format' => 'jsonl', - 'created' => time(), - 'n' => 0, - 'd' => 0, - 'o' => [] - ]; - $this->writeJsonlIndex($dbname, $index); - - // Mark file as existing in cache - $this->markFileExists($fullDBPath); - $this->jsonlFormatCache[$fullDBPath] = true; - + // Buckets that are still those of the file: its body (canonical, checked when it was read) is theirs + $old = $index['file'] === null ? null : $this->secondary->readSecondaryFile($path); + if($old !== null && ($old[0]['w'] ?? null) === $index['file']){ + $body = $old[1]; + [$pg, $c, $n] = [$old[0]['pg'], $old[0]['c'], $old[0]['n']]; + } else { + self::canonicalBuckets($index['b']); + $c = count($index['b']); + $n = 0; + foreach($index['b'] as $keys){ + $n += is_int($keys) ? 1 : count($keys); + } + [$body, $pg] = PagedFile::build($index['b']); + } + $header = ['kind' => 'fx', 'v' => 1, 'field' => (string)$field, 'w' => bin2hex(random_bytes(8)), 's' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc, + 'mk' => (int)$index['mk'], 'c' => $c, 'n' => $n, 'created' => time(), 'pg' => $pg]; + $this->secondary->writeSecondaryFile($dbname, $path, $header, $body); + unset($this->fieldIndexCache[$path . '#pg']); + $index['file'] = $header['w']; + $index['idsAt'] = $index['idsAt'] ?? (int)$index['s']; + $index['ck'] = (int)$index['s']; + $index['bytes'] = strlen($body); + $index['views'] = null; + $this->fieldIndexCache[$path] = $index; + $this->cacheGrew = true; return true; } /** - * Delete a database and every file that belongs to it (data, shards, indexes, buffers, metadata) - * @param string $dbname - * @return array ['success' => bool, 'files' => int deleted file count] + * Buckets in the fixed order of an index file, so the file is the same whoever writes it: bucket ids + * in byte order (ksort, in C), keys of a bucket ascending. Range queries sort their own views + * (fieldIndexViews), so no order by value is needed here. */ - public function dropDB($dbname){ - $dbname = $this->normalizeDbName($dbname); - if(!$this->dbFilesExist($dbname)){ - return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; - } - return $this->withLock($dbname, true, function() use ($dbname){ - $base = $this->dbBase($dbname); - $hashBase = $this->dbDir . $this->hashDBName($dbname); - $lockPath = $this->getLockPath($dbname); - $files = array_merge( - glob($base . '.nonedb*') ?: [], - glob($base . '_s*.nonedb*') ?: [], - glob($hashBase . '.nonedb.*') ?: [], - glob($hashBase . '_s*.nonedb.*') ?: [] - ); - $deleted = 0; - foreach(array_unique($files) as $file){ - // The lock file stays: other processes may be waiting on it, and it carries the generation counter - if($file === $lockPath || !is_file($file)){ - continue; - } - if(@unlink($file)){ - $deleted++; - } + public static function canonicalBuckets(array &$buckets){ + ksort($buckets, SORT_STRING); + $lists = []; + foreach($buckets as $bucket => $keys){ + if(is_array($keys)){ + $lists[] = $bucket; } - $this->invalidateDbCaches($dbname); - // Nothing is left: no buffer, no journal, no pre-3.2 files - self::$lockRegistry[$lockPath]['flags'] = ['v' => true]; - $this->saveLockState($lockPath); - return ['success' => true, 'files' => $deleted]; - }); + } + foreach($lists as $bucket){ + sort($buckets[$bucket]); + } } /** - * Name, creation time and total size of a database (all shards included) - * @return array|false + * The number of an "i…" or "d…" bucket */ - private function getDBInfo($dbname){ - $base = $this->dbBase($dbname); - $infoPath = $base . '.nonedbinfo'; - clearstatcache(); - if(!file_exists($infoPath)){ - return false; + private static function bucketNumber($bucket){ + if($bucket[0] === 'i'){ + return (int)substr($bucket, 1); } - $created = (int)@file_get_contents($infoPath); - $size = 0; - if(file_exists($base . '.nonedb')){ - $size += (int)filesize($base . '.nonedb'); - } - foreach(glob($base . '_s*.nonedb') ?: [] as $shardFile){ - if(preg_match('/_s\d+\.nonedb$/', $shardFile)){ - $size += (int)filesize($shardFile); + $unpacked = unpack('E', (string)hex2bin(substr($bucket, 1))); + return $unpacked === false ? 0.0 : $unpacked[1]; + } + + /** + * Delete a field's index file of a segment + */ + public function deleteFieldIndexFile($dbname, $field, $shardId = null){ + $ok = true; + foreach([$this->getFieldIndexPath($dbname, $field, $shardId)] as $path){ + unset($this->fieldIndexCache[$path]); + $this->cache->markFileNotExists($path); + clearstatcache(true, $path); + if(file_exists($path) && !$this->io->fsUnlink($path)){ + $ok = false; } } - return array("name" => $dbname, "createdTime" => $created, "size" => $this->fileSizeConvert($size)); + return $ok; } + /** - * Convert bytes to human readable format - * @param float $bytes - * @return string + * Get list of indexed fields for a database + * @return array */ - private function fileSizeConvert($bytes){ - $bytes = floatval($bytes); - $arBytes = array( - 0 => array("UNIT" => "TB", "VALUE" => pow(1024, 4)), - 1 => array("UNIT" => "GB", "VALUE" => pow(1024, 3)), - 2 => array("UNIT" => "MB", "VALUE" => pow(1024, 2)), - 3 => array("UNIT" => "KB", "VALUE" => 1024), - 4 => array("UNIT" => "B", "VALUE" => 1), - ); - $result = "0 B"; - foreach($arBytes as $arItem){ - if($bytes >= $arItem["VALUE"]){ - $result = $bytes / $arItem["VALUE"]; - $result = str_replace(".", "," , strval(round($result, 2)))." ".$arItem["UNIT"]; - break; - } - } - return $result; - } - - public function getDBs($info=false){ - // Handle three cases: false (names only), true (with metadata), string (specific db) - $withMetadata = false; - $specificDb = null; - - if(is_bool($info)){ - $withMetadata = $info; - }else{ - $specificDb = $this->normalizeDbName($info); // reading information never creates the database - } - - // If specific database requested - if($specificDb !== null){ - return $this->getDBInfo($specificDb); - } - - // List all databases: one entry per {hash}-{name}.nonedb (non-sharded) or {hash}-{name}.nonedb.meta (sharded) - $dbs = []; - if(!is_dir($this->dbDir)){ - return $dbs; - } - $seen = []; - foreach(new DirectoryIterator($this->dbDir) as $item) { - if($item->isDot() || !$item->isFile()) { - continue; - } - if(!preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+)\\.nonedb(\\.meta)?$/", $item->getFilename(), $m)){ - continue; - } - $dbname = $m[2]; - // Only databases that belong to this secretKey - if($this->hashDBName($dbname) !== $m[1] || isset($seen[$dbname])){ - continue; - } - $seen[$dbname] = true; - - if($withMetadata){ - $info = $this->getDBInfo($dbname); - if($info !== false){ - $dbs[] = $info; - } - }else{ - $dbs[] = $dbname; - } - } - return $dbs; + public function getIndexedFields($dbname){ + return $this->getIndexManifest($dbname)['fields']; } - /** - * limit function - * @param array $array Extract a slice of the array - * @param integer $limit - * @return array|false False for invalid arguments + * Check if a field has an index + * @return bool */ - public function limit($array, $limit=0){ - if(!is_array($array) || !is_int($limit) || $limit <= 0){ - return false; - } - if(empty($array)){ - return []; - } - // Must be a list of records - if(count($array) === count($array, COUNT_RECURSIVE)) { - return false; - } - return array_slice($array, 0, $limit); + public function isFieldIndexed($dbname, $field){ + return in_array((string)$field, $this->getIndexedFields($dbname), true); } /** - * Run a read operation: shared lock while the database exists, no lock (and no files created) - * when it doesn't. Buffered records are flushed first. + * Invalidate field index cache for a database */ - private function withReadLock($dbname, callable $fn){ - if(!$this->hasLockFile($dbname) && !$this->dbFilesExist($dbname)){ - return $fn(); + public function invalidateFieldIndexCache($dbname){ + $prefix = $this->layout->dbHome($dbname); + foreach(array_keys($this->fieldIndexCache) as $key){ + if(strpos($key, $prefix) === 0){ + unset($this->fieldIndexCache[$key]); + } } - return $this->withLock($dbname, false, function() use ($dbname, $fn){ - $this->flushPendingBuffers($dbname); - return $fn(); - }); } /** - * Uncached existence check used before taking a lock + * A segment's field index: the current one, or built from the data (exclusive lock). Null when none + * can be used (the caller scans). + * @return array|null */ - private function dbFilesExist($dbname){ - $base = $this->dbBase($dbname); - foreach(['.nonedb', '.nonedb.meta', '.nonedb.buffer', '.nonedb.buffer.flushing'] as $suffix){ - clearstatcache(true, $base . $suffix); - if(file_exists($base . $suffix)){ - return true; - } + public function ensureSegmentFieldIndex($dbname, $shardId, $field, $path = null, $key = null){ + $index = $this->readFieldIndex($dbname, $field, $shardId, $path, $key); + if($index !== null){ + return $index; } - return false; + if($this->noWrites > 0 || $this->multiLock > 0){ + return null; // checkIntegrity(), explain(), a snapshot: nothing is built, the segment is scanned + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + return $index; + }, 'buildFieldIndex'); } /** - * True when the database's lock file exists (the database may have been dropped since). Taking - * that lock creates no file, and the operation itself finds out whether the database exists. + * Keys of a field index with exactly this value + * @return array */ - private function hasLockFile($dbname){ - $path = $this->getLockPath($dbname); - if(isset(self::$lockHandles[$path]) || isset(self::$lockRegistry[$path])){ - return true; - } - clearstatcache(true, $path); - return file_exists($path); + public function fieldIndexKeys(array $index, $value){ + $keys = $index['b'][is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value)] ?? []; + return is_int($keys) ? [$keys] : $keys; } /** - * read db all data - * @param string $dbname - * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); - * a "key" entry selects by record key (int or list of ints) - * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + * Keys of a segment with this value through a point lookup in its paged file (§6.5.1): the pages on the + * path to the value and the data lines after the bytes the file covers, without loading the index. + * Null when the index is loaded, its tail is long (loading it writes a new checkpoint), or there is no + * file that fits. + * @return array|null */ - public function find($dbname, $filters=0){ - $dbname = $this->normalizeDbName($dbname); - if($filters === null || is_int($filters)){ - $filters = []; + private function pointKeys($dbname, $shardId, $field, $value, $path, $key = null){ + $key = $key ?? $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($key === null){ + return null; } - if(!is_array($filters)){ - return false; + $pk = $path . '#pg'; + $paged = $this->fieldIndexCache[$pk] ?? null; + if($paged !== null && $key['e'] !== null && ($paged['fit'] ?? null) === $key['e']){ + $this->cacheTouch[$pk] = ++$this->cacheTick; // kept, and checked against this epoch: openPagedIndex() in short + $this->metrics['hits']++; + } elseif(($paged = $this->openPagedIndex($dbname, $shardId, $path, $field, $key)) === null){ + return null; } - return $this->withReadLock($dbname, function() use ($dbname, $filters){ - return $this->findUnlocked($dbname, $filters); - }); - } - - private function findUnlocked($dbname, $filters=0){ - if($filters === null || is_int($filters)){ - $filters = []; + // A long tail is replayed by loading the index, which writes a new checkpoint of it (§6.5): point + // lookups don't leave the same lines to every new process. An index in repeated use is loaded too + // (decided once per operation). + $end = (int)$key['s']; + if($end < $paged['h']['s'] + || ($end !== $paged['h']['s'] && $this->secondary->secondaryTailIsLong($paged['h']['s'], $end, $paged['size']))){ + return null; } - if(!$this->dbExists($dbname)){ - // Reads never create databases - return $this->autoCreateDB ? [] : false; + if($paged['ses'] !== $this->sessionCounter){ + // One more operation on the pages, counted in place (the entry isn't copied); loadWholeNow() only + // once the operations may have paid for the load + $entries = (int)($paged['h']['c'] ?? intdiv($paged['size'], 16)); + $paged = null; + $this->fieldIndexCache[$pk]['ses'] = $this->sessionCounter; + $uses = ++$this->fieldIndexCache[$pk]['uses']; + if($uses >= 2 && $uses * self::ENTRIES_PER_LOOKUP >= $entries && $this->loadWholeNow($pk)){ + unset($this->fieldIndexCache[$pk]); + return null; + } + } + $bucket = is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value); + $found = $this->pagedLookup($path, $bucket); + if($found === false){ + // Another reader replaced the file since its header was read here: once more from the new one + $paged = $this->openPagedIndex($dbname, $shardId, $path, $field, $key); + $found = $paged === null ? null : $this->pagedLookup($path, $bucket); + } + if(!is_array($found)){ + unset($this->fieldIndexCache[$pk]); // a damaged page: the whole file is read (and rebuilt) + return null; + } + $covered = $this->fieldIndexCache[$pk]['h']['s']; // the file the pages came from (a new one after a retry) + if($end < $covered){ + return null; + } + if($end === $covered){ + return $found; + } + // The lines after the file, worked out once per end of the data: the newest bucket of each key in + // them (null: tombstone or no indexable value). A key in them leaves the file's answer; the keys + // whose newest line holds this value join it. + $tail = $this->fieldIndexCache[$pk]['tail'] ?? null; + if($tail === null || $tail['to'] !== $end){ + $last = []; + foreach($this->secondary->segmentTail($dbname, $shardId, $covered, $end) as [$k, $record]){ + $last[$k] = $record !== null && array_key_exists($field, $record) && $this->isIndexableValue($record[$field]) + ? \noneDB\Internal\Query\Values::valueKey($record[$field]) : null; + } + $by = []; + foreach($last as $k => $b){ + if($b !== null){ + $by[$b][] = $k; + } + } + $this->fieldIndexCache[$pk]['bytes'] += 96 * (count($last) - ($tail === null ? 0 : count($tail['last']))); + $tail = ['to' => $end, 'last' => $last, 'by' => $by]; + $this->fieldIndexCache[$pk]['tail'] = $tail; + $this->cacheGrew = true; + } + $keys = []; + foreach($found as $k){ + if(!array_key_exists($k, $tail['last'])){ + $keys[] = $k; + } } - return array_values($this->selectRecords($dbname, $filters)); + foreach($tail['by'][$bucket] ?? [] as $k){ + $keys[] = $k; + } + sort($keys); + return $keys; } + /** Entries of an .fx whose whole load costs about one paged lookup more than an array lookup (§6.5.1) */ + private const ENTRIES_PER_LOOKUP = 20; + + /** The body of an .fx in use (from its second operation) of at most this many bytes is read whole on a page miss (§6.5.1) */ + private const RAW_BODY = 262144; + + /** An .fx file of at most this many bytes is loaded whole at its first use (§6.5.1) */ + public const WHOLE_AT_FIRST_USE = 65536; + + /** The largest database (records) an exact map is made for: making it reads every shard's `.fx` whole (§6.5.2) */ + private const EXACT_MAP_RECORDS = 1000000; + /** - * Check if 'key' exists at top level of data (not nested) - * @param array $data - * @return bool + * True when an .fx file not loaded whole is small enough to be loaded at this use, and memory allows (its + * decoded form estimated at 16 bytes per byte of the file). Not in the process's first operation (lock + * session): a single cold operation reads the pages it needs, decoding pays off only when more follow. + * The size: the pages kept of it (header and body), else the file's. */ - private function hasReservedKeyField($data){ - return is_array($data) && array_key_exists("key", $data); + private function wholeAtFirstUse($path){ + if($this->sessionCounter < 2){ + return false; + } + $kept = $this->fieldIndexCache[$path . '#pg'] ?? null; + $size = $kept === null ? @filesize($path) : $kept['at'] + $kept['size']; + return $size !== false && $size <= self::WHOLE_AT_FIRST_USE && \noneDB\Internal\Cache\CacheManager::decodeFits(16 * $size); } /** - * Check if array is a list of records: sequential 0..n-1 keys, every element an array - * @param array $data - * @return bool + * True when an .fx in repeated use should be loaded whole now, while memory allows (plan D6: hot segments + * decoded within the budget): once its operations have paid for the load (at least two, one per + * ENTRIES_PER_LOOKUP entries). A query that runs once, an index used now and then and a process near its + * cache limit keep the pages. */ - private function isRecordList($data){ - if(!is_array($data) || count($data) === 0){ + private function loadWholeNow($key){ + $entry = &$this->fieldIndexCache[$key]; + if($entry['ses'] !== $this->sessionCounter){ + $entry['ses'] = $this->sessionCounter; + $entry['uses']++; + } + $uses = $entry['uses']; + $size = $entry['size']; + $header = $entry['h']; + unset($entry); + $entries = (int)($header['c'] ?? intdiv($size, 16)); + if($uses < 2 || $uses * self::ENTRIES_PER_LOOKUP < $entries){ return false; } - $i = 0; - foreach($data as $k => $v){ - if($k !== $i++ || !is_array($v)){ - return false; + $held = (int)($header['n'] ?? 4 * $size); + return \noneDB\Internal\Cache\CacheManager::decodeFits(128 + 48 * $entries + 24 * $held); + } + + /** + * The header of an .fx file that fits its segment, kept with its pages in fieldIndexCache[path#pg]; null + * when there is none + */ + private function openPagedIndex($dbname, $shardId, $path, $field, $key = null){ + $entry = $this->fieldIndexCache[$path . '#pg'] ?? null; + if($entry !== null){ + $this->metrics['hits']++; + } else { + $this->metrics['misses']++; + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + if(!is_array($header) || ($header['kind'] ?? null) !== 'fx' || ($header['v'] ?? null) !== 1 || ($header['field'] ?? null) !== (string)$field + || !PagedFile::validTree($header['pg'] ?? null) + || !is_string($header['w'] ?? null) || ($w = strpos($line, '"w":"' . $header['w'] . '"')) === false){ + fclose($fp); + return null; + } + // 'id': the header up to the end of its write token, read again before each page (§6.5.1) + // 'size': the body's bytes (the root is its last page) + $entry = ['h' => $header, 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line), 'pages' => [], 'dirs' => [], + 'size' => (int)$header['pg']['root'][0] + (int)$header['pg']['root'][1], 'bytes' => 2 * strlen($line), 'uses' => 0, 'ses' => -1]; + if((fstat($fp)['size'] ?? null) !== $entry['at'] + $entry['size']){ + fclose($fp); + return null; // the root is the file's last page: a body the file doesn't hold is never asked for + } + // Every lookup starts at the root: read with the header, from the same file + [$offset, $length] = $header['pg']['root']; + fseek($fp, $entry['at'] + (int)$offset); + $root = PagedFile::open((string)fread($fp, (int)$length)); + $this->cache->noteIndexRead((int)$length, 1); + $children = $root === null || $header['pg']['levels'] === 0 ? null : PagedFile::directory($root); + if($root !== null && ($header['pg']['levels'] === 0 || $children !== null)){ + if($children === null){ + $entry['pages'][(int)$offset] = $root; + } else { + $entry['dirs'][(int)$offset] = $children; + } + $entry['bytes'] += 2 * (int)$length; + } + fclose($fp); + $this->fieldIndexCache[$path . '#pg'] = $entry; + $this->cacheGrew = true; + } + $this->cacheTouch[$path . '#pg'] = ++$this->cacheTick; + // A segment index fits while its key index keeps the epoch it was checked against (the bytes it + // covers can't change within an epoch): checked once, not at every lookup + if($key === null || $key['e'] === null || ($entry['fit'] ?? null) !== $key['e']){ + if(!$this->secondary->secondaryFits($dbname, $shardId, $entry['h'], 'fx', $field)){ + unset($this->fieldIndexCache[$path . '#pg']); + return null; + } + if($key !== null && $key['e'] !== null){ + $this->fieldIndexCache[$path . '#pg']['fit'] = $entry['fit'] = $key['e']; } } - return true; + return $entry; } /** - * Validate insert data - * @return array [items, error|null] + * The keys of a bucket in an opened .fx ([] when the value isn't in it); null when a page on the path is + * damaged; false when another process replaced the file since its header was read (a reader's checkpoint): + * its pages are dropped, they don't fit the new file. Pages are kept as they are; a data page is searched + * where it is (§6.5.1 lookup step 3). */ - private function validateInsertData($data){ - if(!is_array($data)){ - return [[], "insert data must be array"]; + private function pagedLookup($path, $bucket){ + // Every page on the path kept: answered from them, no handle and no header read + $kept = $this->fieldIndexCache[$path . '#pg']; + $pg = $kept['h']['pg']; + [$offset, $length] = $pg['root']; + for($level = $pg['levels']; $level > 0 && isset($kept['dirs'][$offset]); $level--){ + $child = PagedFile::child($kept['dirs'][$offset], $bucket); + if($child === []){ + return []; + } + [$offset, $length] = $child; } - if(empty($data)){ - return [[], "insert data must not be empty"]; + if($level === 0 && isset($kept['pages'][$offset])){ + return PagedFile::fxMember($kept['pages'][$offset], $bucket); } - $keys = array_keys($data); - $looksLikeList = is_int($keys[0]) && is_array($data[$keys[0]]); - if($looksLikeList && !$this->isRecordList($data)){ - return [[], "insert data must be either one record or a list of records"]; + if($level === 0 && isset($kept['ok'][$offset])){ + // The page inside the body read whole, its crc checked: searched where it is + return PagedFile::fxMember($kept['raw'], $bucket, $offset, $offset + $length - 4); } - $items = $looksLikeList ? array_values($data) : [$data]; - foreach($items as $item){ - if($this->hasReservedKeyField($item)){ - return [[], "You cannot set key name to key"]; + unset($kept); + $entry = &$this->fieldIndexCache[$path . '#pg']; + [$offset, $length] = $pg['root']; + $fp = null; // one handle for the pages of this lookup, opened when a page isn't kept + try { + for($level = (int)$pg['levels']; $level > 0; $level--){ + // Directory pages are kept decoded (few, reused by every lookup) + $children = $entry['dirs'][$offset] ?? null; + if($children === null){ + $page = $this->readPage($path, $entry, $fp, $offset, $length); + if(!is_string($page)){ + return $page; + } + $children = PagedFile::directory($page); + if($children === null){ + return null; + } + $entry['dirs'][$offset] = $children; + $entry['bytes'] += 3 * $length; + } + $child = PagedFile::child($children, $bucket); + if($child === []){ + return []; + } + [$offset, $length] = $child; + } + // A data page is kept as it is, and searched where it is + $page = $entry['pages'][$offset] ?? null; + if($page === null){ + $page = $this->readPage($path, $entry, $fp, $offset, $length); + if(!is_string($page)){ + return $page; + } + if(isset($entry['raw'])){ + $entry['ok'][$offset] = true; // checked once; searched inside the body from now on + return PagedFile::fxMember($entry['raw'], $bucket, $offset, $offset + $length - 4); + } + $entry['pages'][$offset] = $page; + $entry['bytes'] += $length; } - if($this->encodeJson($item) === false){ - return [[], "insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"]; + return PagedFile::fxMember($page, $bucket); + } finally { + unset($entry); + if($fp !== null && $fp !== false){ + fclose($fp); } } - return [$items, null]; } /** - * insert to db - * @param string $dbname - * @param array $data One record, or a list of records - * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] + * Read one page of an opened paged file through the lookup's handle (opened on first use, its header + * checked against the one the entry holds): the page without its crc; null when it is damaged or the + * file can't be opened; false when the file was replaced (the entry is dropped) */ - public function insert($dbname, $data){ - $dbname = $this->normalizeDbName($dbname); - list($items, $error) = $this->validateInsertData($data); - if($error !== null){ - return array("n" => 0, "error" => $error); + private function readPage($path, array &$entry, &$fp, $offset, $length){ + if($length < 1 || $offset + $length > $entry['size']){ + return null; // a child outside the body } - return $this->withLock($dbname, true, function() use ($dbname, $items){ - if($this->bufferEnabled){ - return $this->insertBuffered($dbname, $items); + if(isset($entry['raw'])){ + return PagedFile::open(substr($entry['raw'], $offset, $length)); + } + if($fp === null){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $this->cache->noteIndexRead(strlen($entry['id']), 0); + if(fread($fp, strlen($entry['id'])) !== $entry['id']){ + unset($this->fieldIndexCache[$path . '#pg']); + return false; } - $this->flushPendingBuffers($dbname); - return $this->insertRecords($dbname, $items); - }); - } - - /** - * delete function - * @param string $dbname - * @param array $data Filters (same rules as find); [] deletes every record - * @return array ['n' => deleted count] - */ - public function delete($dbname, $data){ - $dbname = $this->normalizeDbName($dbname); - if(!is_array($data)){ - return array("n" => 0, "error" => "Please check your delete parameters"); } - return $this->withLock($dbname, true, function() use ($dbname, $data){ - return $this->deleteUnlocked($dbname, $data); - }); - } - - private function deleteUnlocked($dbname, $data){ - $this->flushPendingBuffers($dbname); - if(!$this->dbExists($dbname)){ - return array("n" => 0); + if($entry['uses'] >= 2 && $entry['size'] <= self::RAW_BODY + && \noneDB\Internal\Cache\CacheManager::decodeFits(2 * $entry['size'])){ + // An .fx in use in a later operation: its body read whole from this handle (header checked), and + // every page of it answered from those bytes; the pages kept so far are in them + fseek($fp, $entry['at']); + $raw = (string)fread($fp, $entry['size']); + $this->cache->noteIndexRead(strlen($raw)); + if(strlen($raw) === $entry['size']){ + $entry['raw'] = $raw; + $entry['bytes'] += $entry['size'] - array_sum(array_map('strlen', $entry['pages'])); + $entry['pages'] = []; + $entry['ok'] = []; + $this->cacheGrew = true; + return PagedFile::open(substr($raw, $offset, $length)); + } } - $deleted = 0; - foreach($this->groupBySegment($dbname, $this->selectRecords($dbname, $data)) as $segment => $records){ - $deleted += $this->deleteSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $records); + fseek($fp, $entry['at'] + $offset); + $page = PagedFile::open((string)fread($fp, $length)); + $this->cache->noteIndexRead($length, 1); + if($page !== null){ + $this->cacheGrew = true; } - return array("n" => $deleted); + return $page; } /** - * update function - * @param string $dbname - * @param array $data [filters, ['set' => [field => value, ...]]] - * @return array ['n' => updated count] + * Every bucket of a paged .fx body: [bucket => key | [keys]]; null when the file is damaged */ - public function update($dbname, $data){ - $dbname = $this->normalizeDbName($dbname); - if(!is_array($data) || !array_key_exists(0, $data) || !is_array($data[0]) - || !isset($data[1]) || !is_array($data[1]) || !isset($data[1]['set']) || !is_array($data[1]['set']) - || array_key_exists("key", $data[1]['set'])){ - return array("n" => 0, "error" => "Please check your update parameters"); + public function decodeFieldIndexBody(array $header, $body){ + $pg = $header['pg'] ?? null; + if(!is_array($pg)){ + return null; } - if($this->encodeJson($data[1]['set']) === false){ - return array("n" => 0, "error" => "update data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"); + $pages = PagedFile::dataPages((string)$body, $pg); + if($pages === null){ + return null; } - return $this->withLock($dbname, true, function() use ($dbname, $data){ - return $this->updateUnlocked($dbname, $data); - }); + $buckets = []; + foreach($pages as [$offset, $length]){ + $data = PagedFile::open(substr((string)$body, $offset, $length)); + if($data === null || !PagedFile::fxMembersInto($data, $buckets)){ + return null; + } + } + return $buckets; } - private function updateUnlocked($dbname, $data){ - $this->flushPendingBuffers($dbname); - if(!$this->dbExists($dbname)){ - return array("n" => 0); - } - $set = $data[1]['set']; - if(empty($set)){ - return array("n" => count($this->selectRecords($dbname, $data[0]))); + /** + * The equality filters field indexes can answer: [field => value], empty when there are none. A query works + * this out once, not per segment. + * @return array + */ + public function indexedFilters($dbname, array $filters){ + $usable = []; + if(!$this->fieldIndexEnabled || empty($filters)){ + return $usable; } - $updated = 0; - foreach($this->groupBySegment($dbname, $this->selectRecords($dbname, $data[0])) as $segment => $records){ - $new = []; - foreach($records as $key => $record){ - foreach($set as $field => $value){ - $record[$field] = $value; - } - $new[$key] = $record; + $indexed = $this->getIndexedFields($dbname); + foreach($filters as $field => $value){ + if($this->isIndexableValue($value) && in_array((string)$field, $indexed, true)){ + $usable[$field] = $value; } - $updated += $this->updateSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $records, $new); } - return array("n" => $updated); + return $usable; } /** - * Replace whole records by key (used by noneDBQuery::removeFields); caller holds the exclusive lock - * @param array $records [key => new record without "key"] - * @return int + * Candidate keys of a segment for equality filters, from field indexes. + * Null means "no usable index, scan the segment". + * @return array|null */ - private function replaceRecordsByKey($dbname, array $records){ - if(empty($records) || !$this->dbExists($dbname)){ - return 0; - } - $old = $this->selectRecords($dbname, ['key' => array_keys($records)]); - $written = 0; - foreach($this->groupBySegment($dbname, $old) as $segment => $oldRecords){ - $new = []; - foreach(array_keys($oldRecords) as $key){ - $new[$key] = $records[$key]; + public function fieldIndexCandidates($dbname, $shardId, array $filters){ + $usable = $this->indexedFilters($dbname, $filters); + return empty($usable) ? null : $this->segmentCandidates($dbname, $shardId, $usable); + } + + /** + * Candidate keys of a segment for filters of indexedFilters() + * @param array|null $key The segment's key index, when the caller just read it + * @return array + */ + public function segmentCandidates($dbname, $shardId, array $usable, $key = null){ + $candidates = null; + foreach($usable as $field => $value){ + // Loaded whole (or small enough to be at its first use): answered from memory, readFieldIndex() + // replays newer lines; else a point lookup + $path = $this->getFieldIndexPath($dbname, (string)$field, $shardId); + $keys = isset($this->fieldIndexCache[$path]) || $this->wholeAtFirstUse($path) + ? null : $this->pointKeys($dbname, $shardId, (string)$field, $value, $path, $key); + if($keys === null){ + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field, $path, $key); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; // never read without the lock + } + $index = null; // can't be built: scan + } + $keys = $index === null ? null : $this->fieldIndexKeys($index, $value); + } + if($keys === null){ + continue; + } + $candidates = $candidates === null ? $keys : array_values(array_intersect($candidates, $keys)); + if(empty($candidates)){ + return []; } - $written += $this->updateSegmentRecords($dbname, $segment === '' ? null : (int)$segment, $oldRecords, $new); } - return $written; + return $candidates; } - // ========================================== - // QUERY METHODS - // ========================================== + // ==================== RANGE VIEWS (I3) ==================== /** - * Get unique values for a field - * @param string $dbname - * @param string $field - * @return array|false + * Candidate keys of a segment whose field value may satisfy every range condition ($gt, $gte, $lt, + * $lte => operand): a superset, the caller checks every record. Null: no usable index (scan). + * @param array $conditions [operator => operand] + * @return array|null */ - public function distinct($dbname, $field){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field){ - return $this->distinctUnlocked($dbname, $field); - }); + public function fieldIndexRangeCandidates($dbname, $shardId, $field, array $conditions){ + if(!$this->fieldIndexEnabled || !$this->isFieldIndexed($dbname, $field)){ + return null; + } + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + return null; + } + if($index === null){ + return null; + } + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $views = $this->fieldIndexViews($index); + if(isset($this->fieldIndexCache[$path])){ + $this->fieldIndexCache[$path]['views'] = $views; + } + $buckets = null; // bucket => true + foreach($conditions as $op => $operand){ + if(!in_array($op, ['$gt', '$gte', '$lt', '$lte'], true)){ + return null; + } + $found = $this->rangeBuckets($views, $op, $operand); + if($found === null){ + return null; // an operand nothing compares with: no help from the index + } + $buckets = $buckets === null ? $found : array_intersect_key($buckets, $found); + if(empty($buckets)){ + return []; + } + } + if($buckets === null){ + return null; + } + $keys = []; + foreach(array_keys($buckets) as $bucket){ + $bucketKeys = $index['b'][$bucket] ?? []; + if(is_int($bucketKeys)){ + $keys[] = $bucketKeys; + } else { + foreach($bucketKeys as $k){ + $keys[] = $k; + } + } + } + return $keys; } - private function distinctUnlocked($dbname, $field){ - $all = $this->find($dbname, 0); - if($all === false) return false; - $values = []; - foreach($all as $record){ - if(isset($record[$field]) && !in_array($record[$field], $values, true)){ - $values[] = $record[$field]; + /** + * Sorted views of a field index's buckets: numbers (ints, floats and numeric strings, by value) and + * strings (byte order) + * @return array ['num' => [float values], 'numIds' => [their buckets], 'str' => [strings], 'strIds' => [their buckets]] + */ + private function fieldIndexViews(array $index){ + if($index['views'] !== null){ + return $index['views']; + } + // Sort keys first, then one array_multisort per view (a comparator per pair is far slower) + $numValues = []; + $numIds = []; + $strValues = []; + $strIds = []; + foreach(array_keys($index['b']) as $bucket){ + $bucket = (string)$bucket; + $type = $bucket[0]; + if($type === 'i' || $type === 'd'){ + $numValues[] = (float)self::bucketNumber($bucket); + $numIds[] = $bucket; + } elseif($type === 's'){ + $s = substr($bucket, 1); + $strValues[] = $s; + $strIds[] = $bucket; + if(is_numeric($s)){ + $numValues[] = (float)$s; + $numIds[] = $bucket; + } } } - return $values; + array_multisort($numValues, SORT_NUMERIC, $numIds, SORT_STRING); + array_multisort($strValues, SORT_STRING, $strIds, SORT_STRING); + return ['num' => $numValues, 'numIds' => $numIds, 'str' => $strValues, 'strIds' => $strIds]; } /** - * Sort array by field - * @param array $array Result from find() - * @param string $field Field to sort by - * @param string $order 'asc' or 'desc' - * @return array|false + * Buckets whose values may compare as $op with $operand (\noneDBQuery::compareValues semantics: numbers + * and numeric strings numerically, other strings by byte order; null, booleans and arrays compare + * with nothing). Ties of different types are included (rechecked by the caller). + * @return array|null bucket => true; null when the operand can't be compared */ - public function sort($array, $field, $order = 'asc'){ - if(!is_array($array)) return false; - if(count($array) === 0) return []; - $desc = strtolower((string)$order) === 'desc'; - $items = []; - foreach(array_values($array) as $i => $row){ - $items[] = [self::sortKey(is_array($row) && array_key_exists($field, $row) ? $row[$field] : null), $i, $row]; + private function rangeBuckets(array $views, $op, $operand){ + if(is_float($operand) && is_nan($operand)){ + return []; // NAN matches nothing } - usort($items, function($a, $b) use ($desc){ - $result = $a[0] <=> $b[0]; - if($desc){ - $result = -$result; + if(is_int($operand) || is_float($operand) || (is_string($operand) && is_numeric($operand))){ + $x = (float)$operand; + // A wider band at the boundary: values above 2^53 compare as floats here and as ints there + $slack = abs($x) >= 9007199254740992.0 ? max(1.0, abs($x) * 4.0e-16) : 0.0; + [$from, $length] = $this->rangeSlice($views['num'], $op, $x, $slack); + $found = array_fill_keys(array_slice($views['numIds'], $from, $length), true); + if(is_string($operand)){ + // A numeric string operand compares with non-numeric strings by byte order too + [$from, $length] = $this->rangeSliceStrings($views['str'], $op, $operand); + $found += array_fill_keys(array_slice($views['strIds'], $from, $length), true); } - return $result !== 0 ? $result : $a[1] <=> $b[1]; // stable - }); - return array_column($items, 2); + return $found; + } + if(is_string($operand)){ + [$from, $length] = $this->rangeSliceStrings($views['str'], $op, $operand); + return array_fill_keys(array_slice($views['strIds'], $from, $length), true); + } + return null; } /** - * Total order used by sort(): missing/null < booleans < numbers < strings < arrays. - * Values of the same kind compare naturally (numbers numerically, strings with <=>). - * @return int + * Where the entries of a sorted numeric view on the operator's side of $x are, [from, length] (binary search; $slack widens it). + * Without slack ($x below 2^53, where these floats are exact) a strict operator leaves out the values + * equal to $x: compareValues() finds them equal too, so they never match */ - private static function compareSortValues($a, $b){ - return self::sortKey($a) <=> self::sortKey($b); + private function rangeSlice(array $list, $op, $x, $slack){ + $n = count($list); + $strict = $slack == 0.0 && ($op === '$gt' || $op === '$lt'); + if($op === '$gt' || $op === '$gte'){ + $bound = $x - $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid] <= $bound : $list[$mid] < $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [$lo, $n - $lo]; + } + $bound = $x + $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid] < $bound : $list[$mid] <= $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [0, $lo]; } /** - * Sort key of a value: [kind, comparable value]. Two keys compare with <=> in the order above - * (kind first, then the value; arrays by their JSON), so sorts compute each key once instead of - * classifying both values in every comparison. - * @return array + * Where the entries of the sorted string view on the operator's side of $x are, [from, length] (the string + * $x itself only for $gte and $lte) */ - private static function sortKey($v){ - if($v === null) return [0, 0]; - if(is_bool($v)) return [1, $v]; - if(is_int($v) || is_float($v)) return [2, $v]; - if(is_string($v)) return [3, $v]; - return [4, (string)json_encode($v)]; + private function rangeSliceStrings(array $list, $op, $x){ + $n = count($list); + $lo = 0; + $hi = $n; + $strict = $op === '$gt' || $op === '$lt'; + if($op === '$gt' || $op === '$gte'){ + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid], $x); + if($strict ? $cmp <= 0 : $cmp < 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [$lo, $n - $lo]; + } + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid], $x); + if($strict ? $cmp < 0 : $cmp <= 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [0, $lo]; } - /** - * sortKey() for the query builder - * @return \Closure - */ - private function sortKeyFunction(){ - return \Closure::fromCallable([self::class, 'sortKey']); + // ==================== GLOBAL FIELD INDEX (shard skip) ==================== + // indexes/{token}.gfx (docs/STORAGE_FORMAT.md §6.5.2): per shard, a filter of the bucket ids of its field + // index at the bytes [0, s) its entry {"s","e","h"} names, bit-sliced: row r holds bit r of every shard's + // filter, one bit per shard, rows in pages with a crc. While the lock epoch is the one recorded (le), no data + // file was replaced, so every entry still describes its shard; otherwise each entry is checked against its + // shard (epoch and checksum). A shard is skipped only when it has an entry that fits, its filter rules the + // value out, and its lines after s don't have it either. No entry: never skipped. A shard's filter is + // replaced when its field index is checkpointed, never per write. In memory: the header ('le', 'sh', 'bits', + // 'width'; 'id' and 'at': the header bytes through w and where the pages start, of the file it was read + // from; 'uses', 'ses': its operations; 'body': the pages, once loaded whole) and 'cols', the filters this + // session changed and hasn't written yet ('grow': one of them needs larger sizes, the file is made again). + + public function getGlobalFieldIndexPath($dbname, $field){ + return $this->secondary->secondaryPath($dbname, 'gfx', $field); } /** - * Count records matching filter - * @param string $dbname - * @param mixed $filter - * @return int + * The global field index of a field: its header, checked (§6.5.2); null when there is no valid file + * @return array|null */ - public function count($dbname, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $filter){ - return $this->countUnlocked($dbname, $filter); - }); + private function readGlobalFieldIndex($dbname, $field, $path = null){ + $path = $path ?? $this->getGlobalFieldIndexPath($dbname, $field); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(array_key_exists($path, $this->fieldIndexCache)){ + $this->metrics['hits']++; + return $this->fieldIndexCache[$path]; // the header, or null (no valid file) + } + $this->metrics['misses']++; + clearstatcache(true, $path); + $fp = @fopen($path, 'rb'); + $line = $fp === false ? false : fgets($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + $size = $fp === false ? 0 : (int)(fstat($fp)['size'] ?? 0); + if($fp !== false){ + fclose($fp); + } + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + $global = null; + if(is_array($header) && ($header['kind'] ?? null) === 'gfx' && ($header['v'] ?? null) === 1 && ($header['field'] ?? null) === (string)$field + && is_string($header['w'] ?? null) && ($w = strpos($line, '"w":"' . $header['w'] . '"')) !== false + && self::isSize($header['bits'] ?? null, 64) && self::isSize($header['width'] ?? null, 8) && is_array($header['sh'] ?? null) + && $size === strlen($line) + self::globalBodyBytes($header['bits'], $header['width'])){ + $global = ['le' => $header['le'] ?? null, 'sh' => [], 'bits' => $header['bits'], 'width' => $header['width'], + 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line), 'cols' => [], 'uses' => 0, 'ses' => -1]; + foreach($header['sh'] as $shardId => $entry){ + if(SecondaryFiles::isEntry($entry) && (int)$shardId < $header['width']){ + $global['sh'][(int)$shardId] = $entry; + } + } + } + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$this->locks->getLockPath($dbname)][$path]); + $this->cacheGrew = true; + return $global; } - private function countUnlocked($dbname, $filter = 0){ - // Fast-path: no filter = use index/meta counters - if($filter === 0 || $filter === null || (is_array($filter) && empty($filter))){ - return $this->countFast($dbname); - } - $result = $this->findUnlocked($dbname, $filter); - return $result === false ? 0 : count($result); + /** A size of a global field index: a power of two of at least $min */ + private static function isSize($n, $min){ + return is_int($n) && $n >= $min && ($n & ($n - 1)) === 0; + } + + /** Bytes of the pages of a global field index of these sizes */ + private static function globalBodyBytes($bits, $width){ + $per = ShardFilters::pageRows($bits, $width); + return intdiv($bits, $per) * ($per * ($width >> 3) + 4); } /** - * Count without reading records: index entries (non-sharded) or meta totalRecords (sharded) - * @param string $dbname Already normalized - * @return int + * The shards whose filter may hold a bucket id (§6.5.2), with the filters this session changed and hasn't + * written yet: the AND of its six probe rows. The rows come from the body when it is loaded; else from the + * pages that hold them, read from the file after its header (read again from the same handle). The body is + * loaded whole at the file's second operation, when memory allows (every page's crc checked). Null when the + * file is damaged or gone; false when another process replaced it since its header was read. + * @return int[]|null|false ascending */ - private function countFast($dbname){ - if(!$this->dbExists($dbname)){ - return 0; + private function globalHolders($path, $bucket){ + $global = $this->fieldIndexCache[$path]; + $probes = ShardFilters::probes($bucket, $global['bits']); + $held = []; + if($global['id'] !== null){ + $rb = $global['width'] >> 3; + $per = ShardFilters::pageRows($global['bits'], $global['width']); + $stored = $per * $rb + 4; + $rows = []; + if(!isset($global['body'])){ + $pages = intdiv($global['bits'], $per); + $whole = $global['uses'] >= 2 && \noneDB\Internal\Cache\CacheManager::decodeFits($pages * $stored); + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + try { + $this->cache->noteIndexRead(strlen($global['id']), 0); + if(fread($fp, strlen($global['id'])) !== $global['id']){ + return false; + } + if($whole){ + fseek($fp, $global['at']); + $body = (string)fread($fp, $pages * $stored); + $this->cache->noteIndexRead(strlen($body), $pages); + for($p = 0; $p < $pages; $p++){ + if(ShardFilters::open(substr($body, $p * $stored, $stored), $stored - 4) === null){ + return null; + } + } + $global['body'] = $this->fieldIndexCache[$path]['body'] = $body; + $this->cacheGrew = true; + } else { + $read = []; + foreach($probes as $r){ + $p = intdiv($r, $per); + if(!isset($read[$p])){ + fseek($fp, $global['at'] + $p * $stored); + $read[$p] = ShardFilters::open((string)fread($fp, $stored), $stored - 4); + $this->cache->noteIndexRead($stored, 1); + if($read[$p] === null){ + return null; + } + } + $rows[] = substr($read[$p], ($r % $per) * $rb, $rb); + } + } + } finally { + fclose($fp); + } + } + $held = isset($global['body']) ? ShardFilters::holdersIn($global['body'], $global['bits'], $rb, $bucket) : ShardFilters::holders($rows); } - if($this->isSharded($dbname)){ - $meta = $this->getCachedMeta($dbname); - return $meta !== null ? (int)($meta['totalRecords'] ?? 0) : 0; + if(empty($global['cols'])){ + return $held; + } + $held = array_fill_keys($held, true); + foreach($global['cols'] as $shardId => $filter){ + if(ShardFilters::mayHold($filter, $probes)){ + $held[$shardId] = true; + } else { + unset($held[$shardId]); + } } - $index = $this->readJsonlIndex($dbname, null); - return $index !== null ? count($index['o']) : 0; + $held = array_keys($held); + sort($held); + return $held; } /** - * Pattern matching search (LIKE) - * @param string $dbname - * @param string $field - * @param string $pattern Use ^ for starts with, $ for ends with - * @return array|false + * Write the global field index (exclusive lock, or a reader: the content is a function of the shards' field + * indexes): its pages copied from the file it was read from, with the bits of the filters of 'cols' in place + * of the old ones. Made again from every shard's field index when the sizes must grow, there was no valid + * file, or the old one can't be copied (replaced or gone since, a damaged page). */ - public function like($dbname, $field, $pattern){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $pattern){ - return $this->likeUnlocked($dbname, $field, $pattern); - }); - } - - private function likeUnlocked($dbname, $field, $pattern){ - $all = $this->find($dbname, 0); - if($all === false) return false; - - $result = []; - $regex = $this->likePatternToRegex((string)$pattern); - - foreach($all as $record){ - if(!isset($record[$field])) continue; - $value = $record[$field]; - // Skip arrays and objects - can't do string matching on them - if(is_array($value) || is_object($value)) continue; - if(preg_match($regex, (string)$value) === 1){ - $result[] = $record; + public function writeGlobalFieldIndex($dbname, $field, array $global){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $source = empty($global['grow']) ? @fopen($path, 'rb') : false; + if($source !== false){ + $this->cache->noteIndexRead(strlen($global['id']), 0); + } + if($source === false || fread($source, strlen($global['id'])) !== $global['id']){ + if($source !== false){ + fclose($source); } + $this->rebuildGlobalFieldIndex($dbname, $field); + return; + } + fseek($source, $global['at']); + $pages = function() use ($source, $global){ + $rb = $global['width'] >> 3; + $per = ShardFilters::pageRows($global['bits'], $global['width']); + for($p = 0, $n = intdiv($global['bits'], $per); $p < $n; $p++){ + $rows = ShardFilters::open((string)fread($source, $per * $rb + 4), $per * $rb); + $this->cache->noteIndexRead($per * $rb + 4, 1); + if($rows === null){ + throw new \UnexpectedValueException('damaged page'); + } + foreach($global['cols'] as $shardId => $filter){ + $rows = ShardFilters::withColumn($rows, $p * $per, $rb, $shardId, $filter); + } + yield ShardFilters::seal($rows); + } + }; + try { + $this->writeGlobalFile($dbname, $field, $path, $global['sh'], $global['bits'], $global['width'], $pages()); + } catch(\UnexpectedValueException $e){ + $this->rebuildGlobalFieldIndex($dbname, $field); + } finally { + fclose($source); } - return $result; } /** - * Turn a like() pattern into a safe regex: "^" anchors the start, a trailing "$" the end, - * everything else is matched literally and case-insensitively (UTF-8 aware). - * @param string $pattern - * @return string + * Write a global field index file ($pages: its sealed pages, in order) and keep its header as the cached one */ - private function likePatternToRegex($pattern){ - $pattern = (string)$pattern; - $start = $pattern !== '' && $pattern[0] === '^'; - $end = strlen($pattern) > ($start ? 1 : 0) && substr($pattern, -1) === '$'; - $body = substr($pattern, $start ? 1 : 0, strlen($pattern) - ($start ? 1 : 0) - ($end ? 1 : 0)); - $flags = preg_match('//u', $body) === 1 ? 'iu' : 'i'; - return '~' . ($start ? '^' : '') . preg_quote($body, '~') . ($end ? '$' : '') . '~' . $flags; + private function writeGlobalFile($dbname, $field, $path, array $sh, $bits, $width, iterable $pages){ + $lockPath = $this->locks->getLockPath($dbname); + $le = isset($this->lockRegistry[$lockPath]) ? $this->locks->registryEpoch($lockPath) : null; + ksort($sh); + $header = ['kind' => 'gfx', 'v' => 1, 'field' => (string)$field, 'w' => bin2hex(random_bytes(8)), 'le' => $le, + 'bits' => $bits, 'width' => $width, 'sh' => (object)$sh]; + $line = $this->io->encodeHeader($header); + $content = (function() use ($line, $pages){ + yield $line . "\n"; + yield from $pages; + })(); + if($line === false || !$this->io->writeDerivedFile($path, $content)){ + throw new \RuntimeException("noneDB: failed to write index " . basename($path) . " of database '{$dbname}'"); + } + $this->cache->markFileExists($path); + $w = strpos($line, '"w":"' . $header['w'] . '"'); + $this->fieldIndexCache[$path] = ['le' => $le, 'sh' => $sh, 'bits' => $bits, 'width' => $width, + 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line) + 1, 'cols' => [], 'uses' => 0, 'ses' => -1]; + unset($this->globalSkip[$lockPath][$path], $this->lockRegistry[$lockPath]['globals'][$path]); } - // ========================================== - // AGGREGATION METHODS - // ========================================== - /** - * Sum numeric field values - * @param string $dbname - * @param string $field - * @param mixed $filter - * @return float|int + * A shard's field index checkpointed: its filter and entry replace the old ones in the global field index + * (written when the lock session ends) */ - public function sum($dbname, $field, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ - return $this->sumUnlocked($dbname, $field, $filter); - }); - } - - private function sumUnlocked($dbname, $field, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false) return 0; - $sum = 0; - foreach($result as $record){ - if(isset($record[$field]) && is_numeric($record[$field])){ - $sum += $record[$field]; + public function updateGlobalFieldShard($dbname, $field, $shardId, array $index){ + $entry = $this->globalEntry($dbname, $shardId, $index); + $hotKey = $this->getGlobalFieldIndexPath($dbname, $field) . '#hot'; + if(isset($this->fieldIndexCache[$hotKey])){ + // This process's exact map follows: the shard's entry replaced, its values added (a value it no longer + // holds stays listed: the shard is read for it) + $hot = &$this->fieldIndexCache[$hotKey]; + if($entry === null){ + unset($hot['sh'][$shardId]); + } else { + if(!self::sameIdsSince($index, $hot['sh'][$shardId] ?? null)){ + foreach(array_keys($index['b']) as $bucket){ + if(!isset($hot['map'][$bucket])){ + $hot['map'][$bucket] = $shardId; + } elseif(is_int($hot['map'][$bucket])){ + if($hot['map'][$bucket] !== $shardId){ + $hot['map'][$bucket] = [min($hot['map'][$bucket], $shardId), max($hot['map'][$bucket], $shardId)]; + } + } elseif(!in_array($shardId, $hot['map'][$bucket], true)){ + $hot['map'][$bucket][] = $shardId; + sort($hot['map'][$bucket]); // holders in shard order: the order of the records read + } + } + } + $hot['sh'][$shardId] = $entry; } + unset($hot, $this->globalSkip[$this->locks->getLockPath($dbname)][$hotKey]); } - return $sum; + $global = $this->readGlobalFieldIndex($dbname, $field); + if($global === null){ + // No valid file: made from every shard when the session ends, as a lookup would + $this->deferGlobalWrite($dbname, 'gfx', $field, ['le' => null, 'sh' => [], 'bits' => 64, 'width' => 8, 'id' => null, 'cols' => [], 'grow' => true]); + return; + } + $global = $this->secondary->freshGlobalEntries($dbname, $global); + if($entry === null){ + unset($global['sh'][$shardId], $global['cols'][$shardId]); // never skipped + } elseif(self::sameIdsSince($index, $global['sh'][$shardId] ?? null)){ + $global['sh'][$shardId] = $entry; // the bucket ids its bits were made from: they stay + } elseif(32 * count($index['b']) > $global['bits'] || $shardId >= $global['width']){ + $global['grow'] = true; + $global['sh'][$shardId] = $entry; + } else { + $global['cols'][$shardId] = ShardFilters::filter(array_keys($index['b']), $global['bits']); + $global['sh'][$shardId] = $entry; + } + $this->deferGlobalWrite($dbname, 'gfx', $field, $global); } /** - * Average of numeric field values - * @param string $dbname - * @param string $field - * @param mixed $filter - * @return float|int + * True when a shard's entry in the global index (or in the exact map) is of a checkpoint of its field index + * since which the index's bucket ids haven't changed: the filter and the values listed with it are the index's. + * Of the same epoch: offsets of a replaced data file (this process's compaction keeps its map) say nothing. */ - public function avg($dbname, $field, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ - return $this->avgUnlocked($dbname, $field, $filter); - }); + private static function sameIdsSince(array $index, ?array $entry){ + return $entry !== null && $index['idsAt'] !== null && $entry['e'] === $index['e'] && $entry['s'] >= $index['idsAt']; } - private function avgUnlocked($dbname, $field, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false || count($result) === 0) return 0; - $sum = 0; - $count = 0; - foreach($result as $record){ - if(isset($record[$field]) && is_numeric($record[$field])){ - $sum += $record[$field]; - $count++; + /** + * Keep a changed global field or spatial index in the cache and write it when the lock session ends + * (flushPendingGlobals()); without a lock at once + */ + public function deferGlobalWrite($dbname, $kind, $field, array $global){ + $lock = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$lock])){ + if($kind === 'gfx'){ + $this->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->rtree->writeGlobalSpatialIndex($dbname, $field, $global); } + return; } - return $count > 0 ? $sum / $count : 0; + if($kind === 'gfx'){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$lock][$path]); + } else { + $path = $this->rtree->getGlobalSpatialIndexPath($dbname, $field); + $this->globalSpatialIndexCache[$path] = $global; + } + $this->lockRegistry[$lock]['globals'][$path] = [$kind, $field, $this->locks->registryEpoch($lock)]; } /** - * Get minimum value of a field - * @param string $dbname - * @param string $field - * @param mixed $filter - * @return mixed|null + * The global entry of a shard's field index (the bytes it covers, its epoch and their checksum); null when + * it can't have one (no epoch, data file gone) + * @return array|null */ - public function min($dbname, $field, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ - return $this->minUnlocked($dbname, $field, $filter); - }); + private function globalEntry($dbname, $shardId, array $index){ + if($index['e'] === null){ + return null; + } + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$index['s']); + return $crc === null ? null : ['s' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc]; } - private function minUnlocked($dbname, $field, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false || count($result) === 0) return null; - $min = null; - foreach($result as $record){ - if(isset($record[$field])){ - if($min === null || $record[$field] < $min){ - $min = $record[$field]; - } - } + /** A shard's field index for the global one: as it is, or built from the data (and written) when there is none */ + private function shardFieldIndex($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); } - return $min; + return $index; } /** - * Get maximum value of a field - * @param string $dbname - * @param string $field - * @param mixed $filter - * @return mixed|null + * Make the global field index again from every shard's field index (exclusive lock, or a reader): the sizes and + * the shards' entries from a first pass, then the rows in parts of whole pages, each part one more pass over the + * shards that sets the probe bits falling into it. A part holds at most a quarter of the cache limit (all rows + * at once without a limit); the cache evicts between the shards: the process holds a part and one shard's index. + * @return array|null The header, as readGlobalFieldIndex() returns it */ - public function max($dbname, $field, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $filter){ - return $this->maxUnlocked($dbname, $field, $filter); - }); - } - - private function maxUnlocked($dbname, $field, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false || count($result) === 0) return null; - $max = null; - foreach($result as $record){ - if(isset($record[$field])){ - if($max === null || $record[$field] > $max){ - $max = $record[$field]; + public function rebuildGlobalFieldIndex($dbname, $field){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $lockPath = $this->locks->getLockPath($dbname); + unset($this->fieldIndexCache[$path], $this->fieldIndexCache[$path . '#hot'], $this->globalSkip[$lockPath][$path], + $this->globalSkip[$lockPath][$path . '#hot'], $this->lockRegistry[$lockPath]['globals'][$path]); + $shards = $this->segments->listSegments($dbname); + sort($shards); + $largest = 0; + $sh = []; + foreach($shards as $shardId){ + // The sizes need each shard's count of values and its entry: a file that fits its data gives them by its + // header (its bucket ids are read once, below); a missing or stale one by the index built for it + $header = $this->secondary->readSecondaryHeader($this->getFieldIndexPath($dbname, $field, $shardId)); + if($header !== null && ($header['v'] ?? null) === 1 && $this->secondary->secondaryFits($dbname, $shardId, $header, 'fx', $field)){ + $count = (int)$header['c']; + $entry = $this->globalEntry($dbname, $shardId, ['e' => $header['e'], 's' => (int)$header['s']]); + } else { + $index = $this->shardFieldIndex($dbname, $shardId, $field); + $count = count($index['b']); + $entry = $this->globalEntry($dbname, $shardId, $index); + unset($index); + } + $largest = max($largest, $count); + if($entry !== null){ + $sh[$shardId] = $entry; // a shard without one is never skipped: no bits + } + $this->cache->evictBetweenSegments(); + } + $bits = ShardFilters::size(32 * $largest, 64); + $width = ShardFilters::size(empty($shards) ? 1 : end($shards) + 1, 8); + $rb = $width >> 3; + $per = ShardFilters::pageRows($bits, $width); + $pages = intdiv($bits, $per); + $limit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes(); + $part = $limit > 0 ? max(1, intdiv(intdiv($limit, 4), $per * $rb)) : $pages; // pages per part + // ShardFilters::probes() inline, per part the places of the first bucket ids kept (about 256 bytes each): at + // most 16 384, and within a quarter of the cache limit. A value that many shards hold is hashed once + $keep = $limit > 0 ? min(16384, intdiv($limit, 1024)) : 16384; + $this->writeGlobalFile($dbname, $field, $path, $sh, $bits, $width, (function() use ($dbname, $field, $sh, $bits, $rb, $per, $pages, $part, $keep){ + $mask = ($bits - 1) & 0xFFFFFFFF; + // A byte with one bit set, per bit: an array lookup where chr(ord() | bit) makes two calls per probe + $withBit = []; + for($bit = 0; $bit < 8; $bit++){ + for($byte = 0; $byte < 256; $byte++){ + $withBit[$bit][chr($byte)] = chr($byte | (1 << $bit)); } } - } - return $max; + for($first = 0; $first < $pages; $first += $part){ + $lo = $first * $per; + $hi = min($pages, $first + $part) * $per; + $rows = str_repeat("\0", ($hi - $lo) * $rb); // rows [lo, hi), one after the other + $seen = []; + $kept = 0; + foreach(array_keys($sh) as $shardId){ + // The shard's filter written straight into its column: bit r of the filter is its bit in row r + $index = $this->shardFieldIndex($dbname, $shardId, $field); + $byte = $shardId >> 3; + $set = $withBit[$shardId & 7]; + foreach(array_keys($index['b']) as $id){ + $id = (string)$id; + $places = $seen[$id] ?? null; + if($places === null){ + $d = unpack('N2', hash('xxh3', $id, true)); + $y = $d[2] | 1; + $places = []; + foreach([$d[1] & $mask, ($d[1] + $y) & $mask, ($d[1] + 2 * $y) & $mask, + ($d[1] + 3 * $y) & $mask, ($d[1] + 4 * $y) & $mask, ($d[1] + 5 * $y) & $mask] as $r){ + if($r >= $lo && $r < $hi){ + $places[] = ($r - $lo) * $rb; + } + } + if($kept < $keep){ + $seen[$id] = $places; + $kept++; + } + } + foreach($places as $at){ + $at += $byte; + $rows[$at] = $set[$rows[$at]]; + } + } + unset($index); + $this->cache->evictBetweenSegments(); + } + for($p = 0, $n = intdiv($hi - $lo, $per); $p < $n; $p++){ + yield ShardFilters::seal(substr($rows, $p * $per * $rb, $per * $rb)); + } + unset($rows); + } + })()); + return $this->fieldIndexCache[$path]; } - // ========================================== - // UTILITY METHODS - // ========================================== - /** - * Get first matching record - * @param string $dbname - * @param mixed $filter - * @return array|null + * True when a global entry still describes its shard's data bytes [0, s) */ - public function first($dbname, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $filter){ - return $this->firstUnlocked($dbname, $filter); - }); - } - - private function firstUnlocked($dbname, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false || count($result) === 0) return null; - return $result[0]; + public function globalEntryFits($dbname, $shardId, array $entry, $lockEpoch){ + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + if($lockEpoch !== null && $lock !== null && $this->locks->registryEpoch($this->locks->getLockPath($dbname)) === $lockEpoch){ + return true; // no data file was replaced since the entry was written + } + if(($this->dataEpochs[$dataPath] ?? null) === null){ + $this->dataEpochs[$dataPath] = $this->segmentEpochOnDisk($dbname, $shardId); + } + if($this->dataEpochs[$dataPath] !== $entry['e']){ + return false; + } + $checked = $entry['e'] . ':' . $entry['s'] . ':' . $entry['h']; // the checksum too: another file may cover the same bytes + if(($this->secondaryChecked[$dataPath] ?? null) === $checked){ + return true; + } + if($this->checkpoints->segmentCrc($dataPath, (int)$entry['s']) !== $entry['h']){ + return false; + } + $this->secondaryChecked[$dataPath] = $checked; + return true; } /** - * Get last matching record - * @param string $dbname - * @param mixed $filter - * @return array|null + * A segment's data epoch without loading its key index: the cached index, else the checkpoint header */ - public function last($dbname, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $filter){ - return $this->lastUnlocked($dbname, $filter); - }); - } - - private function lastUnlocked($dbname, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false || count($result) === 0) return null; - return $result[count($result) - 1]; + private function segmentEpochOnDisk($dbname, $shardId){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + if(isset($this->indexCache[$path])){ + return $this->indexCache[$path]['e']; + } + $fp = @fopen($path, 'rb'); + if($fp === false){ + return false; + } + $head = (string)fread($fp, 512); + fclose($fp); + $this->cache->noteIndexRead(strlen($head)); + return preg_match('/"e":"([0-9a-f]{8})"/', $head, $m) === 1 ? $m[1] : false; } /** - * Check if records exist matching filter - * @param string $dbname - * @param mixed $filter - * @return bool + * Current data size of a shard: its key index's when this process has it verified (its own writes + * keep it current), else the file's (cached until a write) */ - public function exists($dbname, $filter){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $filter){ - return $this->existsUnlocked($dbname, $filter); - }); - } - - private function existsUnlocked($dbname, $filter){ - $result = $this->find($dbname, $filter); - return $result !== false && count($result) > 0; + private function shardDataSize($dbname, $shardId, $dataPath = null){ + $dataPath = $dataPath ?? $this->segments->segDataPath($dbname, $shardId); + if(isset($this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)], $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)])){ + return (int)$this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]['s']; + } + if(!isset($this->dataSizes[$dataPath])){ + clearstatcache(true, $dataPath); + $this->dataSizes[$dataPath] = (int)@filesize($dataPath); + } + return $this->dataSizes[$dataPath]; + } + + /** + * The shards a lookup in a global field index looks at besides the value's holders: [shard => false] + * has no entry that fits its data (always read), [shard => true] has lines after its entry (read when + * they may hold the value). Worked out over all shards once per state of the data (another process's + * write or a new epoch starts over: invalidateDbCaches(), bumpEpoch()) and kept current with this + * process's own writes (noteWrite()), so a lookup costs O(holders + such shards), not O(shards). + * @return array shard id => bool + */ + private function globalCheckShards($dbname, array $global, $path, $lockPath){ + $seq = $this->segWrites[$lockPath]['seq'] ?? 0; + $state = $this->globalSkip[$lockPath][$path] ?? null; + if($state !== null && $state['seq'] === $seq){ + return $state['check']; + } + if($state === null){ + // While the lock epoch is the recorded one no data file was replaced: every entry fits + $fresh = $global['le'] !== null && isset($this->lockRegistry[$lockPath]) && $global['le'] === $this->locks->registryEpoch($lockPath); + $check = []; + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + foreach($this->segments->listSegments($dbname) as $shardId){ + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || (!$fresh && !$this->globalEntryFits($dbname, $shardId, $entry, null))){ + $check[$shardId] = false; + } elseif($this->shardDataSize($dbname, $shardId, $prefix . $shardId . '.jsonl') > (int)$entry['s']){ + $check[$shardId] = true; + } + } + $state = ['seq' => $seq, 'check' => $check]; + } else { + // This process appended to some shards since: their entries still fit (the bytes they cover + // didn't change), their newer lines are checked; a new shard has no entry + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + $len = strlen($prefix); + foreach($this->segWrites[$lockPath]['paths'] as $dataPath => $at){ + if($at > $state['seq'] && strncmp($dataPath, $prefix, $len) === 0){ + $shardId = (int)substr($dataPath, $len); + if(!isset($state['check'][$shardId])){ + $state['check'][$shardId] = isset($global['sh'][$shardId]); + } + } + } + $state['seq'] = $seq; + } + $this->globalSkip[$lockPath][$path] = $state; + return $state['check']; } /** - * Range query (min <= value <= max) - * @param string $dbname - * @param string $field - * @param mixed $min - * @param mixed $max - * @param mixed $filter Additional filter - * @return array|false + * True when a shard's data lines after a global entry (whose bytes are covered) may hold a value + * matching $match (callable(record) => bool) */ - public function between($dbname, $field, $min, $max, $filter = 0){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $min, $max, $filter){ - return $this->betweenUnlocked($dbname, $field, $min, $max, $filter); - }); + public function shardTailMayMatch($dbname, $shardId, array $entry, callable $match){ + if($this->shardDataSize($dbname, $shardId) <= (int)$entry['s']){ + return false; + } + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null){ + return true; + } + foreach($this->secondary->segmentTail($dbname, $shardId, (int)$entry['s'], (int)$index['s']) as [$k, $record]){ + if($record !== null && $match($record)){ + return true; + } + } + return false; } - private function betweenUnlocked($dbname, $field, $min, $max, $filter = 0){ - $result = $this->find($dbname, $filter); - if($result === false) return false; - // Same comparison as query()->between(): numbers (and numeric strings) numerically, strings - // lexically; null, booleans, arrays and number-vs-text are never in range - $filtered = []; - foreach($result as $record){ - if(!array_key_exists($field, $record)){ + /** + * Shards that can contain matches for the equality filters (null = all shards / not sharded) + * @return array|null + */ + public function targetShardsFromGlobalIndex($dbname, array $filters){ + if(!$this->fieldIndexEnabled || !$this->segments->isSharded($dbname) || empty($filters)){ + return null; + } + $manifest = $this->getIndexManifest($dbname); + $targets = null; + $lockPath = null; + foreach($filters as $field => $value){ + if($field === 'key' || !(is_scalar($value) || $value === null) || !in_array((string)$field, $manifest['fields'], true)){ continue; } - $lo = noneDBQuery::compareValues($record[$field], $min); - $hi = noneDBQuery::compareValues($record[$field], $max); - if($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0){ - $filtered[] = $record; + // Paths built here, not by the helpers: this runs for every lookup of a sharded database + $lockPath = $lockPath ?? $this->locks->getLockPath($dbname); + $path = $this->layout->genDir($dbname) . 'indexes/' . ($this->indexTokens[(string)$field] ?? $this->secondary->indexToken($field)) . '.gfx'; + $bucket = is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value); + // The value's holders: by the exact map when this process has one, else by the filters + $hotKey = $path . '#hot'; + $hot = $this->fieldIndexCache[$hotKey] ?? null; + $held = null; + if($hot === null){ + $global = $this->fieldIndexCache[$path] ?? null; + if(isset($global['body']) && empty($global['cols']) && array_key_exists($hotKey, $this->fieldIndexCache)){ + // No map will come and the body is held: this is the warm lookup of such a process + $this->cacheTouch[$path] = ++$this->cacheTick; + $this->metrics['hits']++; + $held = ShardFilters::holdersIn($global['body'], $global['bits'], $global['width'] >> 3, $bucket); + } else { + $global = $this->readGlobalFieldIndex($dbname, $field, $path); + if($global !== null && isset($global['ses']) && $global['ses'] !== $this->sessionCounter){ + $this->fieldIndexCache[$path]['ses'] = $this->sessionCounter; // an operation of the file + $global['uses'] = ++$this->fieldIndexCache[$path]['uses']; + } + $hot = $global === null ? null : $this->globalHotMap($dbname, $field, $hotKey, $global); + } + } else { + $this->cacheTouch[$hotKey] = ++$this->cacheTick; + $this->metrics['hits']++; + } + if($hot !== null){ + $held = $hot['map'][$bucket] ?? []; + $held = is_int($held) ? [$held] : $held; + $state = $this->globalSkip[$lockPath][$hotKey] ?? null; + $check = $state !== null && $state['seq'] === ($this->segWrites[$lockPath]['seq'] ?? 0) + ? $state['check'] : $this->globalCheckShards($dbname, $hot, $hotKey, $lockPath); + $shards = empty($check) ? $held : $this->withTailHolders($dbname, $field, $value, $hot, $held, $check); + $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); + continue; + } + $held = $held ?? ($global === null ? null : $this->globalHolders($path, $bucket)); + if($held === false){ + // Another process replaced the file since its header was read here: read again + unset($this->fieldIndexCache[$path]); + $global = $this->readGlobalFieldIndex($dbname, $field, $path); + $held = $global === null ? null : $this->globalHolders($path, $bucket); + } + if(!is_array($held)){ + // None, damaged (or replaced again): made again from the shards + if($this->noWrites > 0 || $this->multiLock > 0){ + continue; // checkIntegrity(), a snapshot: build nothing, skip nothing + } + $global = $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildGlobalFieldIndex($dbname, $field); + }, 'buildFieldIndex'); + $held = $this->globalHolders($path, $bucket); + if(!is_array($held)){ + continue; + } } + // The shards whose entry can't rule a value out (worked out once per state of the data) + $state = $this->globalSkip[$lockPath][$path] ?? null; + $check = $state !== null && $state['seq'] === ($this->segWrites[$lockPath]['seq'] ?? 0) + ? $state['check'] : $this->globalCheckShards($dbname, $global, $path, $lockPath); + $shards = empty($check) ? $held : $this->withTailHolders($dbname, $field, $value, $global, $held, $check); + $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); } - return $filtered; + return $targets; } - // ========================================== - // SHARDING PUBLIC METHODS - // ========================================== - /** - * Get sharding information for a database - * @param string $dbname - * @return array|false + * The shards a lookup reads: the holders by the global index, the shards its entries can't rule the value out + * for ($check: shard => false, no fitting entry: read; true: its newer lines may hold it) and of those with + * newer lines the ones whose lines hold it + * @return int[] ascending */ - public function getShardInfo($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname){ - return $this->getShardInfoUnlocked($dbname); - }); + private function withTailHolders($dbname, $field, $value, array $global, array $held, array $check){ + $shards = array_fill_keys($held, true); + $match = null; + foreach($check as $shardId => $tail){ + if(isset($shards[$shardId])){ + continue; + } + $match = $match ?? function($record) use ($field, $value){ + return array_key_exists($field, $record) && $record[$field] === $value; + }; + if(!$tail || $this->shardTailMayMatch($dbname, $shardId, $global['sh'][$shardId], $match)){ + $shards[$shardId] = true; + } + } + $shards = array_keys($shards); + sort($shards); + return $shards; } - private function getShardInfoUnlocked($dbname){ - if(!$this->dbExists($dbname)){ - return false; + /** + * The exact map of the values of a global field index to the shards that hold them, built from every shard's + * field index file at the global index's second operation (lock session) when memory allows: warm lookups + * without the filters (docs/STORAGE_FORMAT.md §6.5.2 "Lookup"). The files are read, not loaded into the cache; + * a shard whose file doesn't fit its data has no entry (it is read). The map describes each shard at the bytes + * its file covers, as the global file's entries do: newer lines are checked, a shard whose data was replaced is + * read. It stays until the cache drops it. ['le', 'sh' => [shard => entry], 'map' => [bucket => shard | + * [shards]]]; null when there is none: before the second operation, in checkIntegrity() or a snapshot, or + * when it doesn't fit (cached as null, the filters answer) + * @param string $key Its cache key (the global file's path and "#hot"); the caller found nothing under it + * @return array|null + */ + private function globalHotMap($dbname, $field, $key, array $global){ + if(array_key_exists($key, $this->fieldIndexCache)){ + return null; // didn't fit (an exact map would have been found by the caller) } - if(!$this->isSharded($dbname)){ - $index = $this->readJsonlIndex($dbname, null); - if($index === null){ - return false; + if(!isset($global['uses']) || $global['uses'] < 2 || $this->noWrites > 0 || $this->multiLock > 0){ + return null; + } + // Made where it can pay: a web server's request ends with its statics (§6.5.2), and over a million records the + // build (every shard's file read whole) costs seconds. Elsewhere the filters answer + $sapi = PHP_SAPI; + $records = self::EXACT_MAP_RECORDS; + if($sapi !== 'cli' || (int)($this->shardMeta->getCachedMeta($dbname)['totalRecords'] ?? 0) > $records){ + $this->fieldIndexCache[$key] = null; + return null; + } + $limit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes(); + // The file's sizes bound the map: at most bits / 32 values per shard entry (bits ≥ 32 · the largest c), about + // 32 bytes a pair. A map whose bound is over the room left under half of the limit is not built, and nothing + // is read for it (§6.5.2) + if($limit > 0 && memory_get_usage() + count($global['sh']) * $global['bits'] > intdiv($limit, 2)){ + $this->fieldIndexCache[$key] = null; + return null; + } + $lock = $this->locks->getLockPath($dbname); + $hot = ['le' => isset($this->lockRegistry[$lock]) ? $this->locks->registryEpoch($lock) : null, 'sh' => [], 'map' => []]; + foreach($this->segments->listSegments($dbname) as $shardId){ + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $header = $this->secondary->readSecondaryHeader($path); + $buckets = null; + if($header !== null && ($header['v'] ?? null) === 1 && $this->secondary->secondaryFits($dbname, $shardId, $header, 'fx', $field)){ + $held = $this->fieldIndexCache[$path] ?? null; + if($held !== null && $held['file'] === $header['w']){ + $buckets = $held['b']; // this process holds the file's buckets as they are: no second decode + } else { + $file = $this->secondary->readSecondaryFile($path); + // (the file read whole must be the one whose header was checked) + $buckets = $file === null || ($file[0]['w'] ?? null) !== $header['w'] ? null : $this->decodeFieldIndexBody($file[0], $file[1]); + } + } + if($buckets !== null){ + $map = &$hot['map']; + foreach(array_keys($buckets) as $bucket){ + if(!isset($map[$bucket])){ + $map[$bucket] = $shardId; + } elseif(is_int($map[$bucket])){ + $map[$bucket] = [$map[$bucket], $shardId]; + } else { + $map[$bucket][] = $shardId; + } + } + unset($map); + $hot['sh'][$shardId] = ['s' => (int)$header['s'], 'e' => $header['e'], 'h' => $header['h']]; + } + unset($file, $buckets, $held); + $this->cache->evictBetweenSegments(); + if($limit > 0 && memory_get_usage() > intdiv($limit, 2)){ + $this->fieldIndexCache[$key] = null; // the map would take more than the cache leaves it + return null; } - return array( - "sharded" => false, - "shards" => 0, - "totalRecords" => count($index['o']), - "shardSize" => $this->shardSize - ); } + $this->fieldIndexCache[$key] = $hot; + $this->cacheGrew = true; + return $hot; + } - $meta = $this->getCachedMeta($dbname); - if($meta === null){ - return false; + /** + * Delete the global field index file + */ + public function deleteGlobalFieldIndex($dbname, $field){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + clearstatcache(true, $path); + if(file_exists($path)){ + $this->io->fsUnlink($path); } - return array( - "sharded" => true, - "shards" => count($meta['shards']), - "totalRecords" => (int)$meta['totalRecords'], - "deletedCount" => (int)($meta['deletedCount'] ?? 0), - "shardSize" => (int)$meta['shardSize'], - "nextKey" => (int)$meta['nextKey'] - ); + $lock = $this->locks->getLockPath($dbname); + unset($this->fieldIndexCache[$path], $this->fieldIndexCache[$path . '#hot'], $this->globalSkip[$lock][$path], $this->globalSkip[$lock][$path . '#hot']); } - // ========================================== - // FIELD INDEX PUBLIC API (v3.0.0) - // ========================================== /** - * Create an index on a field for faster equality lookups (find / where with field => value) - * @param string $dbname Database name - * @param string $field Field name (any name except "key") - * @return array ['success' => bool, 'indexed' => int, 'values' => int, 'error' => ?string] + * After the switch to the sharded layout: the single layout's index files go; every shard's field and + * spatial indexes are built from the data */ - public function createFieldIndex($dbname, $field){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field){ - return $this->createFieldIndexUnlocked($dbname, $field); - }); + public function rebuildIndexesAfterSwitch($dbname){ + $manifest = $this->getIndexManifest($dbname); + foreach($manifest['fields'] as $field){ + $this->deleteFieldIndexFile($dbname, $field, null); + $this->deleteGlobalFieldIndex($dbname, $field); + foreach($this->segments->listSegments($dbname) as $shardId){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + unset($index); + $this->cache->evictBetweenSegments(); + } + $this->rebuildGlobalFieldIndex($dbname, $field); + } + foreach($manifest['spatial'] as $field){ + $this->rtree->rebuildSpatialIndexForDb($dbname, $field); + } } +} +} +namespace noneDB\Internal\Index { - private function createFieldIndexUnlocked($dbname, $field){ - if(!is_string($field) || $field === '' || $field === 'key'){ - return ['success' => false, 'indexed' => 0, 'values' => 0, 'error' => 'Invalid field name']; - } - $this->flushPendingBuffers($dbname); - if(!$this->dbExists($dbname)){ - if(!$this->autoCreateDB){ - return ['success' => false, 'indexed' => 0, 'values' => 0, 'error' => 'Database does not exist']; - } - $this->createDBUnlocked($dbname); +use function array_pop, array_values, count, crc32, intdiv, is_array, is_int, is_string, json_decode, + json_encode, ord, pack, strcmp, strcspn, strlen, strpos, strrpos, substr, unpack; + +use const JSON_UNESCAPED_LINE_TERMINATORS, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE; + +/** + * @internal + */ +final class PagedFile { + /** Target size of a page, crc included */ + public const PAGE = 4096; + + /** Flags of the canonical JSON of .fx data pages (§6.5.1): only `"`, `\\` and control characters escaped */ + public const JSON = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS; + + /** A directory page: its entries, their table (u32 start of each, then the u16 count), the crc of all that */ + private static function seal(array $entries): string { + $data = ''; + $starts = []; + foreach($entries as $entry){ + $starts[] = strlen($data); + $data .= $entry; } + $data .= (empty($starts) ? '' : pack('V*', ...$starts)) . pack('v', count($starts)); + return $data . pack('V', crc32($data)); + } - $indexed = 0; - $values = []; - $segments = $this->listSegments($dbname); - foreach($segments as $segment){ - $index = $this->buildSegmentFieldIndex($dbname, $segment, $field); - $this->writeFieldIndex($dbname, $field, $index, $segment); - foreach($index['values'] as $valueKey => $keys){ - $indexed += count($keys); - $values[(string)$valueKey] = true; + /** True when $entry still fits a directory page of $count entries and $bytes bytes (table and crc included) */ + private static function fits(int $bytes, int $count, string $entry): bool { + return $bytes + strlen($entry) + 4 * ($count + 1) + 2 + 4 <= self::PAGE; + } + + /** + * Where the members of a data page starting at $at end in the encoded members $json: after the last one that + * fits (braces and crc included), or after the first when it alone doesn't. A member ends before a `,"` that no + * `:` follows: inside an id a quote is escaped, so `,":` is the end of an id that ends in a comma, and every id + * starts with a type letter. + */ + private static function pageEnd(string $json, int $at): int { + $n = strlen($json); + if($n - $at <= self::PAGE - 6){ + return $n; + } + for($end = strrpos($json, ',"', $at + self::PAGE - 6 - $n); $end !== false && $end > $at; $end = strrpos($json, ',"', $end - 1 - $n)){ + if($json[$end + 2] !== ':'){ + return $end; } } - if($this->isSharded($dbname)){ - $this->rebuildGlobalFieldIndex($dbname, $field); + for($end = strpos($json, ',"', $at + 1); $end !== false; $end = strpos($json, ',"', $end + 1)){ + if($json[$end + 2] !== ':'){ + return $end; + } } + return $n; + } - $manifest = $this->getIndexManifest($dbname); - $manifest['fields'][] = $field; - $this->saveIndexManifest($dbname, $manifest); - - $result = ['success' => true, 'indexed' => $indexed, 'values' => count($values), 'error' => null]; - if($this->isSharded($dbname)){ - $result['shards'] = count($segments); + /** The id of the member starting at $at in the encoded members $json: up to the first quote no backslash escapes */ + private static function idAt(string $json, int $at): string { + $close = $at + 1; + while($json[$close += strcspn($json, '"\\', $close)] === '\\'){ + $close += 2; } - return $result; + return json_decode(substr($json, $at, $close + 1 - $at)); } /** - * Drop a field index - * @param string $dbname Database name - * @param string $field Field name - * @return array ['success' => bool, 'error' => string|null] + * Build the body from a segment's buckets in bucket id order (id => key | [keys], ascending), sealed as data + * pages, then the directory levels: [body, pg]. A member is `"id":key` for a bucket of one key, else `"id":[keys]`; + * all of them are encoded at once and cut into pages, members not split, each page as full as it gets. */ - public function dropFieldIndex($dbname, $field){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field){ - return $this->dropFieldIndexUnlocked($dbname, $field); - }); - } - - private function dropFieldIndexUnlocked($dbname, $field){ - $manifest = $this->getIndexManifest($dbname); - if(!in_array((string)$field, $manifest['fields'], true)){ - return ['success' => false, 'error' => 'Index does not exist']; + public static function build(array $buckets): array { + $json = substr((string)json_encode($buckets, self::JSON), 1, -1); + $body = ''; + $level = []; + for($at = 0, $n = strlen($json); $at < $n; $at = $end + 1){ + $end = self::pageEnd($json, $at); + $page = '{' . substr($json, $at, $end - $at) . '}'; + $page .= pack('V', crc32($page)); + $level[] = [self::idAt($json, $at), strlen($body), strlen($page)]; + $body .= $page; } - $segments = $this->listSegments($dbname); - foreach($segments as $segment){ - $this->deleteFieldIndexFile($dbname, $field, $segment); - // Also remove a lossy pre-3.2 file name for this field, if any - $legacy = $this->segDataPath($dbname, $segment) . '.fidx.' . preg_replace('/[^a-zA-Z0-9_]/', '_', (string)$field); - if($legacy !== $this->getFieldIndexPath($dbname, $field, $segment) && is_file($legacy)){ - $old = $this->atomicRead($legacy, null); - if(is_array($old) && ($old['field'] ?? null) === (string)$field){ - @unlink($legacy); + if(empty($level)){ + $body = '{}' . pack('V', crc32('{}')); + $level[] = ['', 0, strlen($body)]; + } + $data = strlen($body); + $pages = count($level); + $levels = 0; + while(count($level) > 1){ + $levels++; + $next = []; + $current = []; + $bytes = 0; + $first = ''; + foreach($level as [$id, $offset, $length]){ + $entry = pack('V', strlen($id)) . $id . pack('V2', $offset, $length); + // At least two entries per directory page (long ids): each level has fewer pages + if(count($current) >= 2 && !self::fits($bytes, count($current), $entry)){ + $page = self::seal($current); + $next[] = [$first, strlen($body), strlen($page)]; + $body .= $page; + $current = []; + $bytes = 0; + } + if(empty($current)){ + $first = $id; } + $current[] = $entry; + $bytes += strlen($entry); } + $page = self::seal($current); + $next[] = [$first, strlen($body), strlen($page)]; + $body .= $page; + $pages += count($next); + $level = $next; } - $this->deleteGlobalFieldIndex($dbname, $field); - $manifest['fields'] = array_values(array_diff($manifest['fields'], [(string)$field])); - $this->saveIndexManifest($dbname, $manifest); - $this->invalidateFieldIndexCache($dbname, $field); - return ['success' => true, 'error' => null]; + return [$body, ['root' => [$level[0][1], $level[0][2]], 'levels' => $levels, 'pages' => $pages, 'data' => $data]]; + } + + /** True when a header's `pg` can name a tree: a root of some bytes, no more levels than two entries a page allow */ + public static function validTree($pg): bool { + return is_array($pg) && is_int($pg['root'][0] ?? null) && is_int($pg['root'][1] ?? null) && $pg['root'][1] > 0 + && is_int($pg['levels'] ?? null) && $pg['levels'] >= 0 && $pg['levels'] < 64; + } + + /** A page without its crc (entries and table), or null when the crc doesn't match */ + public static function open($bytes): ?string { + if(!is_string($bytes) || strlen($bytes) < 6){ + return null; + } + return crc32($bytes) === \noneDB\Internal\Storage\KeyIndex::CRC_RESIDUE ? substr($bytes, 0, -4) : null; + } + + /** Where a page's table starts (= the bytes of its entries), or null when the count doesn't fit the page */ + private static function tableAt(string $page): ?int { + $len = strlen($page); + $at = $len - 2 - 4 * (ord($page[$len - 2]) | (ord($page[$len - 1]) << 8)); + return $at < 0 ? null : $at; + } + + /** The child of a decoded directory page that can hold $id (the last whose first id is <= $id): [offset, length]; [] when none */ + public static function child(array $children, string $id): array { + $lo = 0; + $hi = count($children) - 1; + $found = -1; + while($lo <= $hi){ + $mid = ($lo + $hi) >> 1; + if(strcmp($children[$mid][0], $id) <= 0){ + $found = $mid; + $lo = $mid + 1; + } else { + $hi = $mid - 1; + } + } + return $found < 0 ? [] : [$children[$found][1], $children[$found][2]]; + } + + /** True when a page's table names exactly the starts of the entries a walk found ($starts) */ + private static function checkedStarts(string $page, array $starts, int $table): bool { + if(count($starts) !== intdiv(strlen($page) - 2 - $table, 4)){ + return false; + } + return $starts === [] || array_values(unpack('V' . count($starts), $page, $table)) === $starts; + } + + /** A directory page: [[first id, offset, length], ...]; null when its table doesn't name its entries */ + public static function directory(string $page): ?array { + $table = self::tableAt($page); + if($table === null){ + return null; + } + $children = []; + $starts = []; + for($o = 0; $o < $table;){ + $starts[] = $o; + if($o + 12 > $table || $o + 12 + ($n = unpack('V', $page, $o)[1]) > $table){ + return null; // an entry that leaves the page: its crc fits, its content doesn't + } + $child = unpack('V2', $page, $o + 4 + $n); + $children[] = [substr($page, $o + 4, $n), $child[1], $child[2]]; + $o += 12 + $n; + } + return self::checkedStarts($page, $starts, $table) ? $children : null; + } + + /** + * The keys of the member named $id in an .fx data page (its JSON object, without the crc), ascending; + * [] when the page doesn't hold it; null when its value isn't a key or a list of keys. A byte search in + * canonical JSON (§6.5.1 lookup step 3), for the name without its opening quote: a type letter stops a + * byte search less often than a quote, which every name starts and ends with. A match preceded by `{"` or + * `,"` is a member's name (inside a name a quote is escaped, a value holds none), and it names exactly $id. + */ + public static function fxMember(string $page, string $id, int $start = 0, ?int $end = null): ?array { + // The name as the page writes it (the last one kept: a query looks one value up in segment after segment) + static $lastId = null, $lastTail = null; + if($id !== $lastId){ + $name = json_encode($id, self::JSON); + if($name === false){ + return []; // not a name any member can have + } + $lastId = $id; + $lastTail = substr($name, 1) . ':'; + } + $tail = $lastTail; + // $start / $end: the page's bytes inside a whole body, copied: a search never runs past the page + if($start > 0 || ($end !== null && $end < strlen($page))){ + $page = substr($page, $start, ($end ?? strlen($page)) - $start); + } + for($at = strpos($page, $tail, 2); $at !== false; $at = strpos($page, $tail, $at + 1)){ + if($page[$at - 1] === '"' && ($page[$at - 2] === ',' || $page[$at - 2] === '{')){ + return self::fxKeysAt($page, $at + strlen($tail)); + } + } + return []; + } + + /** The keys of the .fx member whose value starts at $at; null when it isn't a key or a list of keys */ + private static function fxKeysAt(string $page, int $at): ?array { + if(($page[$at] ?? '') === '['){ + $close = strpos($page, ']', $at); + $length = $close === false ? 0 : $close + 1 - $at; + } else { + $length = strcspn($page, ',}', $at); + } + $keys = $length > 0 ? json_decode(substr($page, $at, $length), true) : null; + if(is_int($keys)){ + return [$keys]; + } + return is_array($keys) ? $keys : null; + } + + /** + * Add the members of an .fx data page (its JSON object, without the crc) to $map in the shape of a + * loaded index: id => key (one key) or [keys] (ascending). False when the page isn't a JSON object in + * canonical form (encoding it again doesn't give its bytes): its byte searches couldn't be trusted. + */ + public static function fxMembersInto(string $page, array &$map): bool { + if($page === '{}'){ + return true; + } + $members = json_decode($page, true); + if(!is_array($members) || json_encode($members, self::JSON) !== $page){ + return false; + } + $map += $members; + return true; + } + + /** + * Every data page of a body, in order, from its directory: [[offset, length], ...]; null when a page's + * crc or table doesn't match or the pages don't cover the data bytes + */ + public static function dataPages(string $body, array $pg): ?array { + if(!self::validTree($pg)){ + return null; + } + // Depth first, children in order, with a stack (a recursive closure would hold the body in a cycle + // until the garbage collector runs) + $pages = []; + $stack = [[$pg['root'][0], $pg['root'][1], $pg['levels']]]; + while(!empty($stack)){ + [$offset, $length, $level] = array_pop($stack); + if($level === 0){ + $pages[] = [$offset, $length]; + continue; + } + $data = self::open(substr($body, $offset, $length)); + $children = $data === null ? null : self::directory($data); + if($children === null){ + return null; + } + for($i = count($children) - 1; $i >= 0; $i--){ + $stack[] = [$children[$i][1], $children[$i][2], $level - 1]; + } + } + $expected = 0; + foreach($pages as [$offset, $length]){ + if($offset !== $expected){ + return null; + } + $expected += $length; + } + return $expected === ($pg['data'] ?? -1) ? $pages : null; + } +} +} +namespace noneDB\Internal\Index { + +use function array_diff, array_key_exists, array_values, count, in_array, is_int, is_string, sort; + +/** + * @internal + */ +final class FieldIndexApi { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $buffer; + private $cache; + private $db; + private $fieldIndex; + private $layout; + private $locks; + private $records; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->db, $this->fieldIndex, $this->layout, $this->locks, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->db = null; + $this->fieldIndex = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // FIELD INDEX PUBLIC API + // ========================================== + + /** + * Create an index on a field for faster equality lookups (find / where with field => value) + * @param string $dbname Database name + * @param string $field Field name (any name except "key") + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] (+ 'shards' => int when sharded) + */ + public function createFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->createFieldIndexUnlocked($dbname, $field); + }, 'createFieldIndex'); + } + + private function createFieldIndexUnlocked($dbname, $field){ + if(!is_string($field) || $field === '' || $field === 'key'){ + return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + } + if($this->segments->dbExists($dbname) && $this->fieldIndex->isFieldIndexed($dbname, $field)){ + $this->dropFieldIndexUnlocked($dbname, $field); // creating it again rebuilds it + } + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + } + $this->db->createDBUnlocked($dbname); + } + + $indexed = 0; + $segments = $this->segments->listSegments($dbname); + foreach($segments as $segment){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $segment); // leftovers of an earlier index of this field + $index = $this->fieldIndex->buildSegmentFieldIndex($dbname, $segment, $field); + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $segment); + foreach($index['b'] as $keys){ + $indexed += is_int($keys) ? 1 : count($keys); + } + unset($index); + $this->cache->evictBetweenSegments(); + } + if($this->segments->isSharded($dbname)){ + $this->fieldIndex->deleteGlobalFieldIndex($dbname, $field); + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + } + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['fields'][] = $field; + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + + $result = ['success' => true, 'indexed' => $indexed, 'error' => null]; + if($this->segments->isSharded($dbname)){ + $result['shards'] = count($segments); + } + return $result; + } + + /** + * Drop a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => string|null] + */ + public function dropFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropFieldIndexUnlocked($dbname, $field); + }, 'dropFieldIndex'); + } + + private function dropFieldIndexUnlocked($dbname, $field){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(!in_array((string)$field, $manifest['fields'], true)){ + return ['success' => false, 'error' => 'Index does not exist']; + } + $segments = $this->segments->listSegments($dbname); + foreach($segments as $segment){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $segment); + } + $this->fieldIndex->deleteGlobalFieldIndex($dbname, $field); + $manifest['fields'] = array_values(array_diff($manifest['fields'], [(string)$field])); + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + $this->fieldIndex->invalidateFieldIndexCache($dbname); + return ['success' => true, 'error' => null]; } /** @@ -4534,9 +6892,9 @@ private function dropFieldIndexUnlocked($dbname, $field){ * @return bool */ public function hasFieldIndex($dbname, $field){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field){ - return $this->dbExists($dbname) && $this->isFieldIndexed($dbname, $field); + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->segments->dbExists($dbname) && $this->fieldIndex->isFieldIndexed($dbname, $field); }); } @@ -4546,17 +6904,17 @@ public function hasFieldIndex($dbname, $field){ * @return array ['fields' => array, 'sharded' => bool] */ public function getFieldIndexes($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ return $this->getFieldIndexesUnlocked($dbname); }); } private function getFieldIndexesUnlocked($dbname){ - if(!$this->dbExists($dbname)){ + if(!$this->segments->dbExists($dbname)){ return ['fields' => [], 'sharded' => false]; } - return ['fields' => $this->getIndexedFields($dbname), 'sharded' => $this->isSharded($dbname)]; + return ['fields' => $this->fieldIndex->getIndexedFields($dbname), 'sharded' => $this->segments->isSharded($dbname)]; } /** @@ -4566,14 +6924,14 @@ private function getFieldIndexesUnlocked($dbname){ * @return array Result from createFieldIndex */ public function rebuildFieldIndex($dbname, $field){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ return $this->rebuildFieldIndexUnlocked($dbname, $field); - }); + }, 'rebuildFieldIndex'); } private function rebuildFieldIndexUnlocked($dbname, $field){ - if($this->isFieldIndexed($dbname, $field)){ + if($this->fieldIndex->isFieldIndexed($dbname, $field)){ $this->dropFieldIndexUnlocked($dbname, $field); } return $this->createFieldIndexUnlocked($dbname, $field); @@ -4589,1889 +6947,12904 @@ private function rebuildFieldIndexUnlocked($dbname, $field){ * @return array|null Array of matching keys, or null if no index */ public function getKeysFromFieldIndex($dbname, $field, $value, $shardId = null){ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $value, $shardId){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $value, $shardId){ return $this->getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId); }); } private function getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId = null){ - if(!$this->dbExists($dbname) || !$this->isFieldIndexed($dbname, $field) || !$this->isIndexableValue($value)){ + if(!$this->segments->dbExists($dbname) || !$this->fieldIndex->isFieldIndexed($dbname, $field) || !$this->fieldIndex->isIndexableValue($value)){ return null; } - $segments = $shardId !== null ? [(int)$shardId] : $this->listSegments($dbname); + $segments = $shardId !== null ? [(int)$shardId] : $this->segments->listSegments($dbname); $keys = []; foreach($segments as $segment){ - if(!in_array($segment, $this->listSegments($dbname), true)){ + if(!in_array($segment, $this->segments->listSegments($dbname), true)){ + continue; + } + $index = $this->fieldIndex->ensureSegmentFieldIndex($dbname, $segment, $field); + $found = $index === null ? null : $this->fieldIndex->fieldIndexKeys($index, $value); + if($found === null){ + // No usable index: the records decide, exactly + foreach($this->records->eachSegmentBatch($dbname, $segment) as $batch){ + foreach($batch as $key => $record){ + if(array_key_exists($field, $record) && $record[$field] === $value){ + $keys[] = (int)$key; + } + } + } + $this->cache->evictBetweenSegments(); continue; } - $index = $this->ensureSegmentFieldIndex($dbname, $segment, $field); - foreach($index['values'][$this->fieldIndexValueKey($value)] ?? [] as $key){ + foreach($found as $key){ $keys[] = (int)$key; } + $this->cache->evictBetweenSegments(); } sort($keys); return $keys; } +} +} +namespace noneDB\Internal\Index { - // ========================================== - // SPATIAL INDEX - GEOJSON SUPPORT (v3.1.0) - // ========================================== +use function basename, clearstatcache, count, fclose, fgets, fopen, fread, fseek, hash, intdiv, is_array, + is_int, is_string, json_decode, max, min, preg_match, preg_replace, rtrim, stream_get_contents, strlen, + strpos, strtolower, substr; +use const LOCK_EX, LOCK_SH; + +/** + * @internal + */ +final class SecondaryFiles { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $currentGenerations; + private $fieldIndexCache; + private $genPaths; + private $indexCache; + private $lockPaths; + private $spatialIndexCache; + private $verifiedIndexes; + private $indexTokens; + private $lockRegistry; + private $metrics; + private $multiLock; + private $noWrites; + private $secondaryChecked; + private $tailCache; + private $cache; + private $checkpoints; + private $fieldIndex; + private $io; + private $layout; + private $locks; + private $rtree; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->genPaths = &$this->k->genPaths; + $this->indexCache = &$this->k->indexCache; + $this->lockPaths = &$this->k->lockPaths; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->indexTokens = &$this->rt->indexTokens; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * File name part of an index on $field (at most 40 bytes) + */ + public function indexToken($field){ + $field = (string)$field; + if(isset($this->indexTokens[$field])){ + return $this->indexTokens[$field]; + } + $token = preg_match('/^[a-z0-9_]{1,40}$/D', $field) === 1 ? $field + : substr(strtolower(preg_replace('/[^A-Za-z0-9_]/', '_', $field)), 0, 24) . '-' . substr(hash('sha256', $field), 0, 12); + if(count($this->indexTokens) > 4096){ + $this->indexTokens = []; + } + return $this->indexTokens[$field] = $token; + } /** - * Validate a GeoJSON geometry object - * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection - * - * @param array $geometry GeoJSON geometry object - * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] + * Path of a secondary index file: 'fx' / 'rx' per segment, 'gfx' / 'grx' per database */ - public function validateGeoJSON($geometry): array { - if (!is_array($geometry)) { - return ['valid' => false, 'error' => 'Geometry must be an array', 'type' => null]; + public function secondaryPath($dbname, $kind, $field, $shardId = null){ + $k = $shardId ?? -1; + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['x'][$kind][$field][$k]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['x'][$kind][$field][$k]; } - if (!isset($geometry['type'])) { - return ['valid' => false, 'error' => 'Missing type property', 'type' => null]; + $dir = $this->layout->genDir($dbname); + if(isset($this->genPaths[$dbname]['x'][$kind][$field][$k])){ + return $this->genPaths[$dbname]['x'][$kind][$field][$k]; } + $dir .= ($kind === 'fx' || $kind === 'gfx' ? 'indexes/' : 'spatial/'); + $path = $kind === 'gfx' || $kind === 'grx' ? $dir . $this->indexToken($field) . '.' . $kind + : $dir . \noneDB\Internal\Storage\Layout::segmentName($shardId) . '.' . $this->indexToken($field) . '.' . $kind; + return $this->genPaths[$dbname]['x'][$kind][$field][$k] = $path; + } - $type = $geometry['type']; - - // GeometryCollection has different structure - if ($type === 'GeometryCollection') { - return $this->validateGeometryCollection($geometry); + /** + * The key index of a segment for a secondary index check: the cached one when it is known to cover + * the data file (no function call chain on the hot path), else readJsonlIndex() + */ + public function currentKeyIndex($dbname, $shardId){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + if(isset($this->verifiedIndexes[$path], $this->indexCache[$path])){ + $this->metrics['hits']++; + return $this->indexCache[$path]; } + return $this->checkpoints->readJsonlIndex($dbname, $shardId); + } - if (!isset($geometry['coordinates'])) { - return ['valid' => false, 'error' => 'Missing coordinates property', 'type' => null]; + /** + * [header, body] of a secondary index file; null when it is missing or its header is unreadable + */ + public function readSecondaryFile($path){ + clearstatcache(true, $path); + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; } + $line = fgets($fp); + $body = $line === false ? '' : (string)stream_get_contents($fp); + fclose($fp); + $this->cache->noteIndexRead(($line === false ? 0 : strlen($line)) + strlen($body)); + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + return is_array($header) ? [$header, $body] : null; + } - $coords = $geometry['coordinates']; + /** + * Only the header line of a secondary index file (null: missing or unreadable) + */ + public function readSecondaryHeader($path){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + fclose($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + return $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + } - if (!is_array($coords)) { - return ['valid' => false, 'error' => 'Coordinates must be an array', 'type' => null]; + /** + * Write a secondary index file (header line + body). A checkpoint: no generation bump, its content is + * the same whoever writes it. + * @throws \RuntimeException + */ + public function writeSecondaryFile($dbname, $path, array $header, $body){ + $line = $this->io->encodeHeader($header); + if($line === false || !$this->io->writeDerivedFile($path, $line . "\n" . $body)){ + throw new \RuntimeException("noneDB: failed to write index " . basename($path) . " of database '{$dbname}'"); } + $this->cache->markFileExists($path); + } - switch ($type) { - case 'Point': - return $this->validateGeoJSONPoint($coords); - case 'LineString': - return $this->validateGeoJSONLineString($coords); - case 'Polygon': - return $this->validateGeoJSONPolygon($coords); - case 'MultiPoint': - return $this->validateGeoJSONMultiPoint($coords); - case 'MultiLineString': - return $this->validateGeoJSONMultiLineString($coords); - case 'MultiPolygon': - return $this->validateGeoJSONMultiPolygon($coords); - default: - return ['valid' => false, 'error' => "Unknown geometry type: $type", 'type' => null]; + /** + * The segment's key index, with an epoch: an index without one (rebuilt without a checkpoint) gets its + * checkpoint written first (exclusive lock), since secondary indexes name the epoch they belong to + * @return array|null + */ + public function keyIndexWithEpoch($dbname, $shardId){ + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] !== null){ + return $index; } + $lock = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_EX){ + return $index; + } + $this->checkpoints->writeCheckpoint($dbname, $shardId, $index); + return $this->checkpoints->readJsonlIndex($dbname, $shardId); } /** - * Validate Point coordinates [lon, lat] or [lon, lat, altitude] + * True when a secondary index header belongs to the segment as it is: same field, the epoch of its + * key index, and the bytes it covers unchanged (checked once per epoch and offset) */ - private function validateGeoJSONPoint(array $coords): array { - if (!is_array($coords) || count($coords) < 2) { - return ['valid' => false, 'error' => 'Point must have at least 2 coordinates', 'type' => null]; + public function secondaryFits($dbname, $shardId, array $header, $kind, $field){ + if(($header['kind'] ?? null) !== $kind || ($header['field'] ?? null) !== (string)$field + || !isset($header['s'], $header['e'], $header['h']) || !is_int($header['s']) || $header['s'] < 0 || !is_string($header['e']) || !is_string($header['h'])){ + return false; } - if (!isset($coords[0], $coords[1]) || !is_numeric($coords[0]) || !is_numeric($coords[1]) - || is_string($coords[0]) || is_string($coords[1]) || !is_finite((float)$coords[0]) || !is_finite((float)$coords[1])) { - return ['valid' => false, 'error' => 'Point coordinates must be finite numbers', 'type' => null]; + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] === null || $header['e'] !== $index['e'] || $header['s'] > (int)$index['s']){ + return false; } - $lon = (float)$coords[0]; - $lat = (float)$coords[1]; - if ($lon < -180 || $lon > 180) { - return ['valid' => false, 'error' => 'Longitude must be between -180 and 180', 'type' => null]; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $checked = $index['e'] . ':' . $header['s'] . ':' . $header['h']; // the checksum too: another file may cover the same bytes + if(($this->secondaryChecked[$dataPath] ?? null) === $checked){ + return true; } - if ($lat < -90 || $lat > 90) { - return ['valid' => false, 'error' => 'Latitude must be between -90 and 90', 'type' => null]; + // The bytes the key index checkpoint covers had their crc checked when it was loaded or written + $known = $header['s'] === (int)$index['ck'] ? ($index['ckh'] ?? null) : null; + if(($known ?? $this->checkpoints->segmentCrc($dataPath, $header['s'])) !== $header['h']){ + return false; } - return ['valid' => true, 'error' => null, 'type' => 'Point']; + $this->secondaryChecked[$dataPath] = $checked; + return true; } /** - * Validate LineString coordinates [[lon,lat], [lon,lat], ...] + * Records of the complete data lines [$from, $to) of a segment, in file order: [[key, record or null + * for a tombstone], ...]. Lines that are neither are skipped (salvaged ones count, as in the key index). */ - private function validateGeoJSONLineString(array $coords): array { - if (!is_array($coords) || count($coords) < 2) { - return ['valid' => false, 'error' => 'LineString must have at least 2 positions', 'type' => null]; + public function segmentTail($dbname, $shardId, $from, $to){ + if($to <= $from){ + return []; } - foreach ($coords as $i => $position) { - $result = $this->validateGeoJSONPoint($position); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid position at index $i: " . $result['error'], 'type' => null]; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $cached = $this->tailCache[$dataPath] ?? null; + if($cached !== null && $cached[0] === $from && $cached[1] === $to){ + return $cached[2]; + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return []; + } + fseek($fp, $from); + $bytes = ''; + while(strlen($bytes) < $to - $from){ + $chunk = fread($fp, min(1048576, $to - $from - strlen($bytes))); + if($chunk === false || $chunk === ''){ + break; } + $bytes .= $chunk; } - return ['valid' => true, 'error' => null, 'type' => 'LineString']; + fclose($fp); + $base = $shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->segments->shardSizeOf($dbname); + $rows = []; + $pos = 0; + $empty = []; + while(($newline = strpos($bytes, "\n", $pos)) !== false){ + $json = substr($bytes, $pos, $newline - $pos); + $pos = $newline + 1; + $record = json_decode($json, true); + if(is_array($record) && count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $rows[] = [$record['__nonedb_deleted'], null]; + continue; + } + if(is_array($record) && isset($record['key']) && is_int($record['key'])){ + $rows[] = [$record['key'], $record]; + continue; + } + $entry = $this->checkpoints->salvageLine($json, $from + $pos - strlen($json) - 1, $empty, $base, $limit); + if($entry !== null){ + $rows[] = $entry[0] === 't' ? [$entry[1], null] : [$entry[1], json_decode(substr($json, $entry[2], $entry[3]), true)]; + } + } + if(count($this->tailCache) > 64){ + $this->tailCache = []; + } + $this->tailCache[$dataPath] = [$from, $to, $rows]; + return $rows; } /** - * Validate Polygon coordinates [[[lon,lat], ...], ...] - * First ring is outer boundary, subsequent rings are holes + * A global index (gfx / grx) about to be rewritten, which stamps it with the current lock epoch: when + * it was written under another epoch (a data file may have been replaced since), only the entries that + * still fit their shard are kept, so the new stamp vouches for no stale entry */ - private function validateGeoJSONPolygon(array $coords): array { - if (!is_array($coords) || count($coords) < 1) { - return ['valid' => false, 'error' => 'Polygon must have at least one ring', 'type' => null]; + public function freshGlobalEntries($dbname, array $global){ + $lockPath = $this->locks->getLockPath($dbname); + if($global['le'] !== null && isset($this->lockRegistry[$lockPath]) && $global['le'] === $this->locks->registryEpoch($lockPath)){ + return $global; } - foreach ($coords as $ringIndex => $ring) { - if (!is_array($ring) || count($ring) < 4) { - return ['valid' => false, 'error' => "Ring $ringIndex must have at least 4 positions", 'type' => null]; - } - // Validate each position - foreach ($ring as $i => $position) { - $result = $this->validateGeoJSONPoint($position); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid position at ring $ringIndex, index $i: " . $result['error'], 'type' => null]; - } - } - // Check if ring is closed (first == last, compared numerically: 29 and 29.0 are the same) - $first = $ring[0]; - $last = $ring[count($ring) - 1]; - if ((float)$first[0] != (float)$last[0] || (float)$first[1] != (float)$last[1]) { - return ['valid' => false, 'error' => "Ring $ringIndex must be closed (first and last positions must be identical)", 'type' => null]; - } - if ($this->ringSelfIntersects($ring)) { - return ['valid' => false, 'error' => "Ring $ringIndex must not intersect itself", 'type' => null]; - } - if ($ringIndex > 0) { - foreach ($ring as $position) { - if (!$this->pointInRing((float)$position[0], (float)$position[1], $coords[0]) && !$this->pointOnRing((float)$position[0], (float)$position[1], $coords[0])) { - return ['valid' => false, 'error' => "Hole $ringIndex must lie inside the outer ring", 'type' => null]; - } - } + foreach($global['sh'] as $shardId => $entry){ + if($this->fieldIndex->globalEntryFits($dbname, $shardId, $entry, null)){ + continue; } + unset($global['sh'][$shardId], $global['cols'][$shardId]); } - return ['valid' => true, 'error' => null, 'type' => 'Polygon']; + return $global; } /** - * True when two non-adjacent edges of a closed ring cross + * True when the tail of a secondary index (data after the bytes it covers) is long enough to write a + * new checkpoint of it */ - private function ringSelfIntersects(array $ring): bool { - $n = count($ring) - 1; // edges - for ($i = 0; $i < $n; $i++) { - for ($j = $i + 2; $j < $n; $j++) { - if ($i === 0 && $j === $n - 1) { - continue; // first and last edge share the closing vertex - } - if ($this->lineSegmentsIntersect($ring[$i], $ring[$i + 1], $ring[$j], $ring[$j + 1])) { - return true; - } - } - } - return false; + public function secondaryTailIsLong($covered, $end, $fileBytes){ + return $end - $covered >= max(self::SECONDARY_TAIL_BYTES, intdiv((int)$fileBytes, 4)); } /** - * True when a point lies on a ring's boundary + * True for an entry of a global index: it names the bytes of a shard's segment index that it describes (§6.5) */ - private function pointOnRing(float $lon, float $lat, array $ring): bool { - $n = count($ring); - for ($i = 0; $i < $n - 1; $i++) { - if ($this->crossProductDirection($ring[$i], $ring[$i + 1], [$lon, $lat]) == 0 - && $this->onSegment($ring[$i], $ring[$i + 1], [$lon, $lat])) { - return true; - } - } - return false; + public static function isEntry($entry){ + return is_int($entry['s'] ?? null) && $entry['s'] >= 0 && is_string($entry['e'] ?? null) && is_string($entry['h'] ?? null); } /** - * Validate MultiPoint coordinates [[lon,lat], [lon,lat], ...] + * A reader that loaded a secondary index and replayed a long tail writes a new checkpoint of it (and of + * its shard's global entry), so the next process doesn't replay the same lines again. Under the shared + * lock only: never inside a write session (whose appends a rollback may still cut), not for + * checkIntegrity(), explain() or a snapshot. Best effort, no generation bump: the content is a function + * of the data. + * @return bool True when one is due but was put off (checkIntegrity(), explain(), a join): the caller + * marks the cached index ("due") and the next plain read of this process writes it */ - private function validateGeoJSONMultiPoint(array $coords): array { - if (!is_array($coords) || count($coords) < 1) { - return ['valid' => false, 'error' => 'MultiPoint must have at least one point', 'type' => null]; + public function maybeSecondaryReaderCheckpoint($dbname, $shardId, $kind, $field, array $index){ + $lock = $this->locks->getLockPath($dbname); + if($index['e'] === null + || !$this->secondaryTailIsLong((int)$index['ck'], (int)$index['s'], (int)$index['bytes'])){ + return false; } - foreach ($coords as $i => $position) { - $result = $this->validateGeoJSONPoint($position); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid point at index $i: " . $result['error'], 'type' => null]; + if($this->noWrites > 0 || $this->multiLock > 0){ + return true; // explain(), checkIntegrity(), a join: due, the next plain read of this process writes it + } + if(!isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_SH){ + return false; // a write session: checkpointSegmentSecondaries() at its end + } + try { + if($kind === 'fx'){ + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->fieldIndex->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } else { + $this->rtree->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; } + // never needed for correctness: the next load replays the lines again } - return ['valid' => true, 'error' => null, 'type' => 'MultiPoint']; + return false; } /** - * Validate MultiLineString coordinates [[[lon,lat], ...], ...] + * End of a write session: write new checkpoints of the field and spatial indexes of a changed segment + * that this process has loaded and whose tail is long, and their global shard summaries (exclusive + * lock, best effort). An index nobody here uses isn't loaded for it: a writer that only writes keeps + * no secondary index in memory, and the next reader that loads it writes the checkpoint. */ - private function validateGeoJSONMultiLineString(array $coords): array { - if (!is_array($coords) || count($coords) < 1) { - return ['valid' => false, 'error' => 'MultiLineString must have at least one linestring', 'type' => null]; + public function checkpointSegmentSecondaries($dbname, $shardId){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; } - foreach ($coords as $i => $lineString) { - $result = $this->validateGeoJSONLineString($lineString); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid linestring at index $i: " . $result['error'], 'type' => null]; + $key = $this->keyIndexWithEpoch($dbname, $shardId); + if($key === null || $key['e'] === null){ + return; + } + foreach(['fx' => $manifest['fields'], 'rx' => $manifest['spatial']] as $kind => $fields){ + foreach($fields as $field){ + $path = $this->secondaryPath($dbname, $kind, $field, $shardId); + $cached = $kind === 'fx' ? ($this->fieldIndexCache[$path] ?? null) : ($this->spatialIndexCache[$path] ?? null); + if($cached === null && $kind === 'fx' && isset($this->fieldIndexCache[$path . '#pg'])){ + // used here through point lookups (pages of its file): in use like a loaded one + $paged = $this->fieldIndexCache[$path . '#pg']; + $cached = ['e' => $paged['h']['e'] ?? null, 'ck' => (int)$paged['h']['s'], 'bytes' => $paged['size']]; + } + if($cached === null || $cached['e'] !== $key['e']){ + continue; // not used here (or of another epoch): the reader that uses it checkpoints it + } + $covered = (int)$cached['ck']; + $bytes = (int)$cached['bytes']; + if(!$this->secondaryTailIsLong($covered, (int)$key['s'], $bytes)){ + continue; + } + try { + if($kind === 'fx'){ + $index = $this->fieldIndex->ensureSegmentFieldIndex($dbname, $shardId, $field); + if($index !== null){ + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->fieldIndex->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + } else { + $index = $this->rtree->ensureSegmentSpatialIndex($dbname, $field, $shardId); + $this->rtree->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: the next load replays a longer tail + } } } - return ['valid' => true, 'error' => null, 'type' => 'MultiLineString']; } + // ========================================== + // SECONDARY INDEX FILES + // ========================================== + // A field index (.fx), an R-tree (.rx) and the global shard summaries (.gfx, .grx) are checkpoints + // of their segment's data, like the key index: line 1 of the file is a header with the field, the + // data bytes "s" it covers, the crc32b "h" of the bytes before s and the epoch "e" of the key index it + // was made with; the body follows. An index is a function of the data alone: loading it replays the + // data lines after s (in memory), so writes never touch it, and a crash, a rollback or another + // process's writes can't leave it stale. Once its tail reached 64 KiB (or a quarter of the index file), + // the process that uses it writes a new checkpoint: a reader that loaded it, or a write session that + // has it loaded (a process that only writes never loads it). An index whose epoch or bytes don't fit + // the segment is rebuilt from the data (exclusive lock). + // Names (in the active generation): indexes/{seg}.{token}.fx, indexes/{token}.gfx, + // spatial/{seg}.{token}.rx, spatial/{token}.grx; the token is the field when it is [a-z0-9_]{1,40}, + // else a lower-case prefix and 12 hex digits of its sha256: at most 40 bytes, and fields that differ + // only in case ("city", "City") never share a file. + + public const SECONDARY_TAIL_BYTES = 65536; +} +} +namespace noneDB\Internal\Index { + +use function array_chunk, array_fill, chr, chunk_split, crc32, hash, intdiv, max, min, ord, pack, str_repeat, strlen, strspn, + strtr, substr, unpack; + +/** + * @internal + */ +final class ShardFilters { + /** Per bit value (1 << (shard & 7)): byte b of a filter => its eight rows, one byte each ($bit or 0) */ + private static $columnBytes = []; /** - * Validate MultiPolygon coordinates [[[[lon,lat], ...], ...], ...] + * The six probe rows of a bucket id in filters of $bits bits: from its xxh3, x the first four bytes and y the + * last four (lowest bit set), big-endian; row (x + i·y) mod 2³² mod $bits + * @return int[] */ - private function validateGeoJSONMultiPolygon(array $coords): array { - if (!is_array($coords) || count($coords) < 1) { - return ['valid' => false, 'error' => 'MultiPolygon must have at least one polygon', 'type' => null]; - } - foreach ($coords as $i => $polygon) { - $result = $this->validateGeoJSONPolygon($polygon); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid polygon at index $i: " . $result['error'], 'type' => null]; - } + public static function probes(string $id, int $bits): array { + $d = unpack('N2', hash('xxh3', $id, true)); + $y = $d[2] | 1; + $rows = []; + for($i = 0; $i < 6; $i++){ + $rows[] = (($d[1] + $i * $y) & 0xFFFFFFFF) & ($bits - 1); } - return ['valid' => true, 'error' => null, 'type' => 'MultiPolygon']; + return $rows; } /** - * Validate GeometryCollection + * The filter of a shard whose field index holds these bucket ids: $bits bits (bit r: byte r >> 3, value 1 << (r & 7)). + * The probes of probes(), set in 64-bit words (bit r: word r >> 6, bit r & 63), packed little-endian. */ - private function validateGeometryCollection(array $geometry): array { - if (!isset($geometry['geometries']) || !is_array($geometry['geometries'])) { - return ['valid' => false, 'error' => 'GeometryCollection must have geometries array', 'type' => null]; - } - foreach ($geometry['geometries'] as $i => $geom) { - if (!is_array($geom)) { - return ['valid' => false, 'error' => "Invalid geometry at index $i: must be an object", 'type' => null]; - } - $result = $this->validateGeoJSON($geom); - if (!$result['valid']) { - return ['valid' => false, 'error' => "Invalid geometry at index $i: " . $result['error'], 'type' => null]; + public static function filter(array $ids, int $bits): string { + $words = array_fill(0, $bits >> 6, 0); + $mask = ($bits - 1) & 0xFFFFFFFF; // mod 2³² mod $bits (a power of two) + foreach($ids as $id){ + $d = unpack('N2', hash('xxh3', (string)$id, true)); + $y = $d[2] | 1; + for($i = 0, $v = $d[1]; $i < 6; $i++, $v += $y){ + $r = $v & $mask; + $words[$r >> 6] |= 1 << ($r & 63); } } - return ['valid' => true, 'error' => null, 'type' => 'GeometryCollection']; + $filter = ''; + foreach(array_chunk($words, 4096) as $chunk){ + $filter .= pack('P*', ...$chunk); + } + return $filter; } - // ========================================== - // SPATIAL INDEX - GEOMETRY OPERATIONS - // ========================================== - /** - * Calculate Minimum Bounding Rectangle (MBR) for any GeoJSON geometry - * @param array $geometry GeoJSON geometry object - * @return array [minLon, minLat, maxLon, maxLat] + * Rows [$first, $first + n) of $rowBytes bytes each (a page: n a power of two, $first a multiple of it), with + * $shard's column taken from its filter: the shard's bit of row r (byte $shard >> 3, value 1 << ($shard & 7)) + * becomes bit $first + r of the filter; every other bit stays */ - private function calculateMBR(array $geometry): array { - $type = $geometry['type'] ?? ''; - $coords = $geometry['coordinates'] ?? []; - - switch ($type) { - case 'Point': - return [$coords[0], $coords[1], $coords[0], $coords[1]]; - - case 'LineString': - case 'MultiPoint': - return $this->mbrFromPoints($coords); + public static function withColumn(string $rows, int $first, int $rowBytes, int $shard, string $filter): string { + $n = intdiv(strlen($rows), $rowBytes); + $bit = 1 << ($shard & 7); + if(!isset(self::$columnBytes[$bit])){ + for($b = 0; $b < 256; $b++){ + $eight = ''; + for($k = 0; $k < 8; $k++){ + $eight .= ($b >> $k) & 1 ? chr($bit) : "\0"; + } + self::$columnBytes[$bit][chr($b)] = $eight; + } + } + // The filter's bits of these rows, one byte per row; then each at its place in its row + $column = substr(strtr(substr($filter, $first >> 3, ($n + 7) >> 3), self::$columnBytes[$bit]), $first & 7, $n); + $at = $shard >> 3; + $column = substr(str_repeat("\0", $at) . chunk_split($column, 1, str_repeat("\0", $rowBytes - 1)), 0, $n * $rowBytes); + $keep = str_repeat(str_repeat("\xFF", $at) . chr(0xFF ^ $bit) . str_repeat("\xFF", $rowBytes - $at - 1), $n); + return ($rows & $keep) | $column; + } - case 'Polygon': - // Use outer ring (first ring) - return $this->mbrFromPoints($coords[0]); + /** True when a filter has every probe bit (the shard may hold the bucket id) */ + public static function mayHold(string $filter, array $probes): bool { + foreach($probes as $r){ + if((ord($filter[$r >> 3]) & (1 << ($r & 7))) === 0){ + return false; + } + } + return true; + } - case 'MultiLineString': - $allPoints = []; - foreach ($coords as $lineString) { - $allPoints = array_merge($allPoints, $lineString); - } - return $this->mbrFromPoints($allPoints); + /** The smallest power of two of at least max($min, $n): a number of bits or the width */ + public static function size(int $n, int $min): int { + $size = $min; + while($size < $n){ + $size <<= 1; + } + return $size; + } - case 'MultiPolygon': - $allPoints = []; - foreach ($coords as $polygon) { - $allPoints = array_merge($allPoints, $polygon[0]); // outer ring - } - return $this->mbrFromPoints($allPoints); + /** Rows per page for rows of $width bits (a page holds about 4096 bytes of rows) */ + public static function pageRows(int $bits, int $width): int { + return min($bits, max(1, intdiv(4096, $width >> 3))); + } - case 'GeometryCollection': - $mbr = null; - foreach ($geometry['geometries'] as $geom) { - $geomMBR = $this->calculateMBR($geom); - $mbr = $mbr === null ? $geomMBR : $this->mbrUnion($mbr, $geomMBR); - } - return $mbr ?? [0, 0, 0, 0]; + /** A page as it is stored: its rows, then their crc32 (u32 little-endian) */ + public static function seal(string $rows): string { + return $rows . pack('V', crc32($rows)); + } - default: - return [0, 0, 0, 0]; + /** The rows of a stored page of $length bytes of rows; null when it is short or its crc doesn't match */ + public static function open(string $stored, int $length): ?string { + if(strlen($stored) !== $length + 4){ + return null; } + $rows = substr($stored, 0, $length); + return unpack('V', $stored, $length)[1] === crc32($rows) ? $rows : null; } /** - * Calculate MBR from array of points - * @param array $points Array of [lon, lat] positions - * @return array [minLon, minLat, maxLon, maxLat] + * The shards (bit positions) set in every row: the AND of the rows, its set bits + * @param string[] $rows + * @return int[] ascending */ - private function mbrFromPoints(array $points): array { - if (empty($points)) { - return [0, 0, 0, 0]; + public static function holders(array $rows): array { + $and = null; + foreach($rows as $row){ + $and = $and === null ? $row : $and & $row; } + return self::setBits($and); + } - $minLon = $maxLon = $points[0][0]; - $minLat = $maxLat = $points[0][1]; - - foreach ($points as $point) { - $minLon = min($minLon, $point[0]); - $maxLon = max($maxLon, $point[0]); - $minLat = min($minLat, $point[1]); - $maxLat = max($maxLat, $point[1]); + /** + * The shards that may hold a bucket id, from a file's body in memory: the rows of probes() (x + i·y, the mod 2³² of + * a power of two is its mask) read in place, the AND given up at the first row that leaves no shard. A warm lookup. + * @param string $body The stored pages + * @param int $rb Bytes of a row + * @return int[] ascending + */ + public static function holdersIn(string $body, int $bits, int $rb, string $id): array { + $d = unpack('N2', hash('xxh3', $id, true)); + $x = $d[1]; + $y = $d[2] | 1; + $mask = $bits - 1; + $per = self::pageRows($bits, $rb << 3); + $stored = $per * $rb + 4; + $and = null; + for($i = 0; $i < 6; $i++, $x += $y){ + $r = $x & $mask; + $row = substr($body, intdiv($r, $per) * $stored + ($r % $per) * $rb, $rb); + $and = $and === null ? $row : $and & $row; + if(strspn($and, "\0") === $rb){ + return []; + } } + return self::setBits($and); + } - return [$minLon, $minLat, $maxLon, $maxLat]; + /** @return int[] The bit positions set in a row, ascending */ + private static function setBits(string $and): array { + $shards = []; + $length = strlen($and); + for($o = strspn($and, "\0"); $o < $length; $o += 1 + strspn($and, "\0", $o + 1)){ + $byte = ord($and[$o]); + for($bit = 0; $bit < 8; $bit++){ + if($byte & (1 << $bit)){ + $shards[] = 8 * $o + $bit; + } + } + } + return $shards; } +} +} +namespace noneDB\Internal\Index\Spatial { - /** - * Union two MBRs - * @param array $mbr1 [minLon, minLat, maxLon, maxLat] - * @param array $mbr2 [minLon, minLat, maxLon, maxLat] - * @return array [minLon, minLat, maxLon, maxLat] - */ - private function mbrUnion(array $mbr1, array $mbr2): array { - return [ - min($mbr1[0], $mbr2[0]), // minLon - min($mbr1[1], $mbr2[1]), // minLat - max($mbr1[2], $mbr2[2]), // maxLon - max($mbr1[3], $mbr2[3]) // maxLat - ]; +use function array_fill, array_is_list, array_keys, array_multisort, asort, count, is_array, is_finite, + is_numeric, is_string, ksort, log, pack; +use const SORT_NUMERIC; + +/** + * @internal + */ +final class GeoJson { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $geometry; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->geometry); } /** - * Check if two MBRs overlap - * @param array $mbr1 [minLon, minLat, maxLon, maxLat] - * @param array $mbr2 [minLon, minLat, maxLon, maxLat] - * @return bool + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function mbrOverlaps(array $mbr1, array $mbr2): bool { - // Check if one MBR is completely to the left, right, above, or below the other - if ($mbr1[2] < $mbr2[0] || $mbr2[2] < $mbr1[0]) return false; // No horizontal overlap - if ($mbr1[3] < $mbr2[1] || $mbr2[3] < $mbr1[1]) return false; // No vertical overlap - return true; + public function bind(): void { } /** - * Calculate area of an MBR (in degrees squared) - * @param array $mbr [minLon, minLat, maxLon, maxLat] - * @return float + * A handle to another service, bound on first use */ - private function mbrArea(array $mbr): float { - $width = $mbr[2] - $mbr[0]; - $height = $mbr[3] - $mbr[1]; - return $width * $height; + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Calculate enlargement needed if entry is added to an MBR - * @param array $mbr Existing MBR - * @param array $entryMBR New entry's MBR - * @return float Area increase + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function mbrEnlargement(array $mbr, array $entryMBR): float { - $unionMBR = $this->mbrUnion($mbr, $entryMBR); - return $this->mbrArea($unionMBR) - $this->mbrArea($mbr); + public function unbind(): void { + $this->geometry = null; + $this->k = null; + $this->rt = null; } + // ========================================== + // SPATIAL INDEX - GEOJSON SUPPORT + // ========================================== + /** - * Calculate Haversine distance between two points (in meters) - * Uses memoization cache for repeated calculations - * @param float $lon1 Longitude of point 1 - * @param float $lat1 Latitude of point 1 - * @param float $lon2 Longitude of point 2 - * @param float $lat2 Latitude of point 2 - * @return float Distance in meters + * Validate a GeoJSON geometry object + * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection + * + * @param array $geometry GeoJSON geometry object + * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] */ - public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float { - // Check cache first (memoization) - $key = "$lon1,$lat1,$lon2,$lat2"; - if (isset($this->distanceCache[$key])) { - return $this->distanceCache[$key]; + public function validateGeoJSON($geometry): array { + // Never throws: a stored record that fails here is simply not indexed (a malformed geometry must + // never make later operations on the database fail) + if (!is_array($geometry)) { + return ['valid' => false, 'error' => 'Geometry must be an array', 'type' => null]; + } + if (!isset($geometry['type'])) { + return ['valid' => false, 'error' => 'Missing type property', 'type' => null]; } - $earthRadius = 6371000.0; // meters - - $latDelta = deg2rad($lat2 - $lat1); - $lonDelta = deg2rad($lon2 - $lon1); + $type = $geometry['type']; + if (!is_string($type)) { + return ['valid' => false, 'error' => 'Geometry type must be a string', 'type' => null]; + } - $a = sin($latDelta / 2) * sin($latDelta / 2) + - cos(deg2rad($lat1)) * cos(deg2rad($lat2)) * - sin($lonDelta / 2) * sin($lonDelta / 2); + // GeometryCollection has different structure + if ($type === 'GeometryCollection') { + return $this->validateGeometryCollection($geometry); + } - $c = 2 * atan2(sqrt($a), sqrt(1 - $a)); + if (!isset($geometry['coordinates'])) { + return ['valid' => false, 'error' => 'Missing coordinates property', 'type' => null]; + } - $distance = $earthRadius * $c; + $coords = $geometry['coordinates']; - // Cache result; when full, the older half makes room (the cache keeps working) - if (count($this->distanceCache) >= 10000) { - $this->distanceCache = array_slice($this->distanceCache, 5000, null, true); + // Each validator checks the lists it needs (a Point needs none: values at 0 and 1) + if (!is_array($coords)) { + return ['valid' => false, 'error' => 'Coordinates must be an array', 'type' => null]; } - $this->distanceCache[$key] = $distance; - return $distance; + // Strict comparison (a switch compared loosely: 0 matched 'Point' before PHP 8) + if ($type === 'Point') { + return $this->validateGeoJSONPoint($coords); + } + if ($type === 'LineString') { + return $this->validateGeoJSONLineString($coords); + } + if ($type === 'Polygon') { + return $this->validateGeoJSONPolygon($coords); + } + if ($type === 'MultiPoint') { + return $this->validateGeoJSONMultiPoint($coords); + } + if ($type === 'MultiLineString') { + return $this->validateGeoJSONMultiLineString($coords); + } + if ($type === 'MultiPolygon') { + return $this->validateGeoJSONMultiPolygon($coords); + } + return ['valid' => false, 'error' => "Unknown geometry type: $type", 'type' => null]; } /** - * Clear distance cache (call after each query for memory management) + * An array with keys 0..n-1 in order (a JSON array) */ - public function clearDistanceCache(): void { - $this->distanceCache = []; + private static function isJsonList($value): bool { + return is_array($value) && array_is_list($value); } /** - * Convert a circle (center + radius) to a bounding box - * @param float $lon Center longitude - * @param float $lat Center latitude - * @param float $radiusMeters Radius in meters - * @return array [minLon, minLat, maxLon, maxLat] + * Validate Point coordinates [lon, lat] or [lon, lat, altitude] */ - private function circleToBBox(float $lon, float $lat, float $radiusMeters): array { - // Approximate degrees per meter at this latitude - $latDeg = $radiusMeters / 111000.0; // 1 degree lat ~ 111 km = 111000 m - $lonDeg = $radiusMeters / (111000.0 * cos(deg2rad($lat))); - - // Near or over a pole every longitude is in range - if ($lat + $latDeg >= 90 || $lat - $latDeg <= -90 || $lonDeg >= 180 || !is_finite($lonDeg)) { - $lonDeg = 360; + private function validateGeoJSONPoint($coords): array { + // A position: values at 0 and 1 (checked below) are enough, no list check per position + if (!is_array($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'Point must have at least 2 coordinates', 'type' => null]; } - - return [ - max(-180, $lon - $lonDeg), // minLon - max(-90, $lat - $latDeg), // minLat - min(180, $lon + $lonDeg), // maxLon - min(90, $lat + $latDeg) // maxLat - ]; + if (!isset($coords[0], $coords[1]) || !is_numeric($coords[0]) || !is_numeric($coords[1]) + || is_string($coords[0]) || is_string($coords[1]) || !is_finite((float)$coords[0]) || !is_finite((float)$coords[1])) { + return ['valid' => false, 'error' => 'Point coordinates must be finite numbers', 'type' => null]; + } + $lon = (float)$coords[0]; + $lat = (float)$coords[1]; + if ($lon < -180 || $lon > 180) { + return ['valid' => false, 'error' => 'Longitude must be between -180 and 180', 'type' => null]; + } + if ($lat < -90 || $lat > 90) { + return ['valid' => false, 'error' => 'Latitude must be between -90 and 90', 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Point']; } /** - * Point-in-Polygon test using Winding Number algorithm - * More robust than ray casting for edge cases - * - * @param float $lon Point longitude - * @param float $lat Point latitude - * @param array $polygon GeoJSON Polygon coordinates [[[lon,lat], ...], ...] - * @return bool + * Validate LineString coordinates [[lon,lat], [lon,lat], ...] */ - private function pointInPolygon(float $lon, float $lat, array $polygon): bool { - // Handle GeoJSON Polygon object or raw coordinates - $coords = isset($polygon['coordinates']) ? $polygon['coordinates'] : $polygon; - - // Check outer ring - if (!$this->pointInRing($lon, $lat, $coords[0])) { - return false; + private function validateGeoJSONLineString($coords): array { + if (!self::isJsonList($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'LineString must have at least 2 positions', 'type' => null]; } - - // Check holes (if any) - for ($i = 1; $i < count($coords); $i++) { - if ($this->pointInRing($lon, $lat, $coords[$i])) { - return false; // Inside a hole + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid position at index $i: " . $result['error'], 'type' => null]; } } - - return true; + return ['valid' => true, 'error' => null, 'type' => 'LineString']; } /** - * Check if point is inside a ring using Winding Number algorithm + * Validate Polygon coordinates [[[lon,lat], ...], ...] + * First ring is outer boundary, subsequent rings are holes */ - private function pointInRing(float $lon, float $lat, array $ring): bool { - $windingNumber = 0; - $n = count($ring); - - for ($i = 0; $i < $n - 1; $i++) { - $x1 = $ring[$i][0]; - $y1 = $ring[$i][1]; - $x2 = $ring[$i + 1][0]; - $y2 = $ring[$i + 1][1]; - - if ($y1 <= $lat) { - if ($y2 > $lat) { - // Upward crossing - if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) > 0) { - $windingNumber++; - } + private function validateGeoJSONPolygon($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'Polygon must have at least one ring', 'type' => null]; + } + // Rings in order. The holes before the first failing ring are checked against the outer ring + // together (one sweep for all their vertices); the first error is the same as ring by ring. + $failure = null; + $holes = []; + foreach ($coords as $ringIndex => $ring) { + if (!self::isJsonList($ring) || count($ring) < 4) { + $failure = "Ring $ringIndex must have at least 4 positions"; + break; + } + // Validate each position + foreach ($ring as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + $failure = "Invalid position at ring $ringIndex, index $i: " . $result['error']; + break 2; } - } else { - if ($y2 <= $lat) { - // Downward crossing - if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) < 0) { - $windingNumber--; - } + } + // Check if ring is closed (first == last, compared numerically: 29 and 29.0 are the same) + $first = $ring[0]; + $last = $ring[count($ring) - 1]; + if ((float)$first[0] != (float)$last[0] || (float)$first[1] != (float)$last[1]) { + $failure = "Ring $ringIndex must be closed (first and last positions must be identical)"; + break; + } + if ($this->ringSelfIntersects($ring)) { + $failure = "Ring $ringIndex must not intersect itself"; + break; + } + if ($ringIndex > 0) { + $holes[$ringIndex] = $ring; + } + } + if ($holes) { + // Every vertex of a hole inside the outer ring or on it + $points = []; + $owner = []; + foreach ($holes as $ringIndex => $ring) { + foreach ($ring as $position) { + $points[] = $position; + $owner[] = $ringIndex; + } + } + foreach ($this->locatePointsInRing($coords[0], $points) as $k => $where) { + if ($where < 0) { + return ['valid' => false, 'error' => "Hole {$owner[$k]} must lie inside the outer ring", 'type' => null]; } } } - - return $windingNumber !== 0; - } - - /** - * Determine if point is left of a line (using cross product) - * Returns: >0 if left, <0 if right, =0 if on line - */ - private function isLeftOfLine(float $x1, float $y1, float $x2, float $y2, float $px, float $py): float { - return (($x2 - $x1) * ($py - $y1)) - (($px - $x1) * ($y2 - $y1)); + if ($failure !== null) { + return ['valid' => false, 'error' => $failure, 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Polygon']; } /** - * Check if two line segments intersect - * @param array $p1 First point of segment 1 [lon, lat] - * @param array $p2 Second point of segment 1 [lon, lat] - * @param array $p3 First point of segment 2 [lon, lat] - * @param array $p4 Second point of segment 2 [lon, lat] - * @return bool + * True when two non-adjacent edges of a closed ring share a point (they cross, touch or overlap; + * exact arithmetic). First a sort and sweep by x that compares the edges whose x ranges overlap: + * fast for usual rings. When that work + * passes 16 edge pairs per edge (rings built against it, e.g. a comb whose edges all overlap in x), + * a sweep line over a balanced tree finishes the check: O(n log n) for every ring. */ - private function lineSegmentsIntersect(array $p1, array $p2, array $p3, array $p4): bool { - $d1 = $this->crossProductDirection($p3, $p4, $p1); - $d2 = $this->crossProductDirection($p3, $p4, $p2); - $d3 = $this->crossProductDirection($p1, $p2, $p3); - $d4 = $this->crossProductDirection($p1, $p2, $p4); - - if ((($d1 > 0 && $d2 < 0) || ($d1 < 0 && $d2 > 0)) && - (($d3 > 0 && $d4 < 0) || ($d3 < 0 && $d4 > 0))) { - return true; + private function ringSelfIntersects(array $ring): bool { + $n = count($ring) - 1; // edges; edge i runs from vertex i to vertex i + 1 + if ($n < 4) { + return false; // every pair of a triangle's edges is adjacent + } + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; + } + $budget = 16 * $n + 256; + if ($budget === 0) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + $minX = []; + $maxX = []; + for ($i = 0; $i < $n; $i++) { + $a = $xs[$i]; + $b = $xs[$i + 1]; + $minX[$i] = $a < $b ? $a : $b; + $maxX[$i] = $a < $b ? $b : $a; + } + asort($minX, SORT_NUMERIC); + $active = []; // edge => max x, edges whose x range may still overlap the next ones + $work = 0; + foreach ($minX as $i => $from) { + foreach ($active as $j => $to) { + if ($to < $from) { + unset($active[$j]); + continue; + } + $d = $i > $j ? $i - $j : $j - $i; + if ($d === 1 || $d === $n - 1) { + continue; // neighbours share a vertex (the first and the last edge share the closing one) + } + if (++$work > $budget) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + if (self::segmentsMeet($xs[$i], $ys[$i], $xs[$i + 1], $ys[$i + 1], $xs[$j], $ys[$j], $xs[$j + 1], $ys[$j + 1])) { + return true; + } + } + $active[$i] = $maxX[$i]; } - - // Check collinear cases - if ($d1 == 0 && $this->onSegment($p3, $p4, $p1)) return true; - if ($d2 == 0 && $this->onSegment($p3, $p4, $p2)) return true; - if ($d3 == 0 && $this->onSegment($p1, $p2, $p3)) return true; - if ($d4 == 0 && $this->onSegment($p1, $p2, $p4)) return true; - return false; } /** - * Cross product direction calculation + * ringSelfIntersects() as a sweep line (Shamos-Hoey) over an AVL tree of the edges that cross it, + * ordered from bottom to top: O(n log n). Events are the vertices in (x, y) order. A ring that doesn't + * intersect itself has at every event point exactly one vertex, so exactly its two edges end or + * start there and no other edge passes through it; between events two edges can only cross after + * they became neighbours in the tree, and every new pair of neighbours is tested. */ - private function crossProductDirection(array $p1, array $p2, array $p3): float { - return ($p3[0] - $p1[0]) * ($p2[1] - $p1[1]) - ($p2[0] - $p1[0]) * ($p3[1] - $p1[1]); + private function ringSelfIntersectsSweep(array $xs, array $ys, int $n): bool { + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $tree = self::avlTree(array_fill(0, $n, 0)); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi); + $meet = function (int $a, int $b) use ($xs, $ys, $n): bool { + if ($a < 0 || $b < 0) { + return false; + } + $d = $a > $b ? $a - $b : $b - $a; + if ($d === 1 || $d === $n - 1) { + return false; + } + return self::segmentsMeet($xs[$a], $ys[$a], $xs[$a + 1], $ys[$a + 1], $xs[$b], $ys[$b], $xs[$b + 1], $ys[$b + 1]); + }; + $count = count($ee); + for ($k = 0; $k < $count;) { + $px = $ex[$k]; + $py = $ey[$k]; + $edges = []; + $starts = []; + $ends = []; + for (; $k < $count && $ex[$k] == $px && $ey[$k] == $py; $k++) { + $edge = $ee[$k] >> 1; + $edges[$edge] = true; + if ($ee[$k] & 1) { + $ends[] = $edge; + } else { + $starts[] = $edge; + } + } + // One vertex: its two edges, adjacent (a repeated vertex or a zero-length edge gives more) + if (count($edges) !== 2) { + return true; + } + [$e1, $e2] = array_keys($edges); + $d = $e1 > $e2 ? $e1 - $e2 : $e2 - $e1; + if ($d !== 1 && $d !== $n - 1) { + return true; + } + foreach ($ends as $edge) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $edge, $cmp); + } + // No other edge through the vertex; the edges below and above it become neighbours + $below = -1; + $above = -1; + for ($node = $tree['root']; $node >= 0;) { + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $px, $py); + if ($c === 0) { + return true; // the vertex lies on another edge + } + if ($c > 0) { + $below = $node; + $node = $tree['r'][$node]; + } else { + $above = $node; + $node = $tree['l'][$node]; + } + } + if (!$starts) { + if ($meet($below, $above)) { + return true; + } + continue; + } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + if ($meet($pred, $edge) || $meet($edge, $succ)) { + return true; + } + } + } + return false; } /** - * Check if point p is on segment p1-p2 + * Sweep events of a ring's edges: [lo, hi, x, y, code] with lo/hi the vertex numbers of each edge's + * smaller and larger end in (x, y) order, and the events (both ends of every edge, code edge * 2 + * for the start, edge * 2 + 1 for the end) sorted by x, then y */ - private function onSegment(array $p1, array $p2, array $p): bool { - return min($p1[0], $p2[0]) <= $p[0] && $p[0] <= max($p1[0], $p2[0]) && - min($p1[1], $p2[1]) <= $p[1] && $p[1] <= max($p1[1], $p2[1]); + private static function sweepEvents(array $xs, array $ys, int $n): array { + $lo = []; + $hi = []; + $ex = []; + $ey = []; + $ee = []; + for ($i = 0; $i < $n; $i++) { + $a = $i; + $b = $i + 1; + if ($xs[$b] < $xs[$a] || ($xs[$b] == $xs[$a] && $ys[$b] < $ys[$a])) { + $a = $i + 1; + $b = $i; + } + $lo[$i] = $a; + $hi[$i] = $b; + $ex[] = $xs[$a]; + $ey[] = $ys[$a]; + $ee[] = $i * 2; + $ex[] = $xs[$b]; + $ey[] = $ys[$b]; + $ee[] = $i * 2 + 1; + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + return [$lo, $hi, $ex, $ey, $ee]; + } + + /** + * Order of two edges that cross the sweep line at the same time and don't cross each other (bottom + * to top): compared where the later one starts, then by direction; equal edges by number (the test + * build counts the comparisons) + */ + private static function sweepOrder(array $xs, array $ys, array $lo, array $hi): \Closure { + return function (int $s, int $t) use ($xs, $ys, $lo, $hi): int { + $sx = $xs[$lo[$s]]; + $sy = $ys[$lo[$s]]; + $tx = $xs[$lo[$t]]; + $ty = $ys[$lo[$t]]; + if ($sx > $tx || ($sx == $tx && $sy > $ty)) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $sx, $sy); + if ($c === 0) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } + } elseif ($sx == $tx && $sy == $ty) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } else { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $tx, $ty); + if ($c === 0) { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $xs[$hi[$t]], $ys[$hi[$t]]); + } + } + return $c !== 0 ? $c : ($s <=> $t); + }; } /** - * Check if two polygons intersect - * @param array $poly1 First polygon coordinates - * @param array $poly2 Second polygon coordinates - * @return bool + * Where points are against a ring that doesn't intersect itself: 1 inside, 0 on its boundary, -1 + * outside (exact). Every point against every edge when that is cheap; otherwise one sweep line over + * the edges with the points as events, O((n + m) log n): the edges below a point, each counted +1 or + * -1 by its direction, give its winding number (a ray straight down). */ - private function polygonsIntersect(array $poly1, array $poly2): bool { - // Handle GeoJSON Polygon objects or raw coordinates - $coords1 = isset($poly1['coordinates']) ? $poly1['coordinates'] : $poly1; - $coords2 = isset($poly2['coordinates']) ? $poly2['coordinates'] : $poly2; - - // 1. Quick MBR rejection test - $mbr1 = $this->mbrFromPoints($coords1[0]); - $mbr2 = $this->mbrFromPoints($coords2[0]); - if (!$this->mbrOverlaps($mbr1, $mbr2)) { - return false; + private function locatePointsInRing(array $ring, array $points): array { + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; } - - // 2. Check if any vertex of poly1 is inside poly2 - foreach ($coords1[0] as $point) { - if ($this->pointInPolygon($point[0], $point[1], $coords2)) { - return true; + $n = count($xs) - 1; + $m = count($points); + $budget = (int)(16 * ($n + $m) * log($n + $m + 2, 2)); + $out = []; + if ($n < 4 || ($budget > 0 && $n * $m <= $budget)) { + foreach ($points as $point) { + $out[] = self::locatePointInRing((float)$point[0], (float)$point[1], $xs, $ys, $n); } + return $out; } - // 3. Check if any vertex of poly2 is inside poly1 - foreach ($coords2[0] as $point) { - if ($this->pointInPolygon($point[0], $point[1], $coords1)) { - return true; - } + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $vertices = []; + for ($i = 0; $i < $n; $i++) { + $vertices[pack('dd', $xs[$i] + 0.0, $ys[$i] + 0.0)] = true; } - - // 4. Check edge intersections - $ring1 = $coords1[0]; - $ring2 = $coords2[0]; - - for ($i = 0; $i < count($ring1) - 1; $i++) { - for ($j = 0; $j < count($ring2) - 1; $j++) { - if ($this->lineSegmentsIntersect($ring1[$i], $ring1[$i + 1], $ring2[$j], $ring2[$j + 1])) { - return true; + $weights = []; + for ($i = 0; $i < $n; $i++) { + $weights[$i] = $lo[$i] === $i ? 1 : -1; // run from its smaller end to its larger one + } + foreach ($points as $k => $point) { + $x = (float)$point[0]; + $y = (float)$point[1]; + if (isset($vertices[pack('dd', $x + 0.0, $y + 0.0)])) { + $out[$k] = 0; + continue; + } + $out[$k] = -1; + $ex[] = $x; + $ey[] = $y; + $ee[] = -1 - $k; // point events: negative codes + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + $tree = self::avlTree($weights); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi); + $count = count($ee); + for ($k = 0; $k < $count;) { + if ($ee[$k] < 0) { + // A point (never at a vertex): the edges below it + $x = $ex[$k]; + $y = $ey[$k]; + $winding = 0; + for ($node = $tree['root']; $node >= 0;) { + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $x, $y); + if ($c === 0) { + $winding = null; // on this edge + break; + } + if ($c > 0) { + $winding += $tree['s'][$tree['l'][$node]] + $weights[$node]; + $node = $tree['r'][$node]; + } else { + $node = $tree['l'][$node]; + } + } + $out[-1 - $ee[$k]] = $winding === null ? 0 : ($winding !== 0 ? 1 : -1); + $k++; + continue; + } + // A vertex: its edges that end here leave the tree, then the ones that start here join it + $px = $ex[$k]; + $py = $ey[$k]; + $starts = []; + for (; $k < $count && $ee[$k] >= 0 && $ex[$k] == $px && $ey[$k] == $py; $k++) { + if ($ee[$k] & 1) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $ee[$k] >> 1, $cmp); + } else { + $starts[] = $ee[$k] >> 1; } } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + } } - - return false; + ksort($out); + return $out; } /** - * Get centroid of a geometry (for distance calculations) - * Uses caching for repeated geometry lookups - * @param array $geometry GeoJSON geometry - * @return array [lon, lat] + * One point against a ring, exact: 1 inside, 0 on its boundary, -1 outside (winding number) */ - private function getGeometryCentroid(array $geometry): array { - if (($geometry['type'] ?? '') === 'Point' && isset($geometry['coordinates'][0], $geometry['coordinates'][1])) { - return [$geometry['coordinates'][0], $geometry['coordinates'][1]]; - } - // Check cache first (use JSON hash as key) - $cacheKey = md5((string)json_encode($geometry)); - if (isset($this->centroidCache[$cacheKey])) { - return $this->centroidCache[$cacheKey]; - } - - $type = $geometry['type'] ?? ''; - $coords = $geometry['coordinates'] ?? []; - $result = null; - - switch ($type) { - case 'Point': - $result = [$coords[0], $coords[1]]; - break; - - case 'LineString': - case 'MultiPoint': - $sumLon = $sumLat = 0; - foreach ($coords as $point) { - $sumLon += $point[0]; - $sumLat += $point[1]; + private static function locatePointInRing(float $x, float $y, array $xs, array $ys, int $n): int { + $winding = 0; + for ($i = 0; $i < $n; $i++) { + $x1 = $xs[$i]; + $y1 = $ys[$i]; + $x2 = $xs[$i + 1]; + $y2 = $ys[$i + 1]; + $o = null; + if (($x1 < $x2 ? $x1 <= $x && $x <= $x2 : $x2 <= $x && $x <= $x1) + && ($y1 < $y2 ? $y1 <= $y && $y <= $y2 : $y2 <= $y && $y <= $y1)) { + $o = self::orient($x1, $y1, $x2, $y2, $x, $y); + if ($o === 0) { + return 0; } - $n = count($coords); - $result = [$sumLon / $n, $sumLat / $n]; - break; - - case 'Polygon': - // Centroid of outer ring - $ring = $coords[0]; - $sumLon = $sumLat = 0; - $n = count($ring) - 1; // Exclude closing point - for ($i = 0; $i < $n; $i++) { - $sumLon += $ring[$i][0]; - $sumLat += $ring[$i][1]; - } - $result = [$sumLon / $n, $sumLat / $n]; - break; - - default: - // For complex types, use MBR center - $mbr = $this->calculateMBR($geometry); - $result = [($mbr[0] + $mbr[2]) / 2, ($mbr[1] + $mbr[3]) / 2]; - } - - // Cache result; when full, the older half makes room - if (count($this->centroidCache) >= 5000) { - $this->centroidCache = array_slice($this->centroidCache, 2500, null, true); + } + if ($y1 <= $y) { + if ($y2 > $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) > 0) { + $winding++; + } + } elseif ($y2 <= $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) < 0) { + $winding--; + } } - $this->centroidCache[$cacheKey] = $result; - - return $result; + return $winding !== 0 ? 1 : -1; } /** - * Clear centroid cache + * True when two closed segments share a point (exact) */ - public function clearCentroidCache(): void { - $this->centroidCache = []; + private static function segmentsMeet(float $ax, float $ay, float $bx, float $by, float $cx, float $cy, float $dx, float $dy): bool { + if (($ax < $bx ? $ax : $bx) > ($cx > $dx ? $cx : $dx) || ($cx < $dx ? $cx : $dx) > ($ax > $bx ? $ax : $bx) + || ($ay < $by ? $ay : $by) > ($cy > $dy ? $cy : $dy) || ($cy < $dy ? $cy : $dy) > ($ay > $by ? $ay : $by)) { + return false; // bounding boxes apart + } + $o1 = self::orient($ax, $ay, $bx, $by, $cx, $cy); + $o2 = self::orient($ax, $ay, $bx, $by, $dx, $dy); + $o3 = self::orient($cx, $cy, $dx, $dy, $ax, $ay); + $o4 = self::orient($cx, $cy, $dx, $dy, $bx, $by); + if ($o1 * $o2 < 0 && $o3 * $o4 < 0) { + return true; + } + // Touching or collinear: an end of one on the other (inside the boxes, which overlap) + return ($o1 === 0 && self::inBox($ax, $ay, $bx, $by, $cx, $cy)) || ($o2 === 0 && self::inBox($ax, $ay, $bx, $by, $dx, $dy)) + || ($o3 === 0 && self::inBox($cx, $cy, $dx, $dy, $ax, $ay)) || ($o4 === 0 && self::inBox($cx, $cy, $dx, $dy, $bx, $by)); } - // ========================================== - // SPATIAL INDEX - R-TREE ENGINE - // ========================================== - // One R-tree per segment and indexed field: {dbDir}{hash}[_sN].nonedb.sidx.{field}. Sharded databases - // also keep {hash}.nonedb.gsidx.{field} with each shard's bounding box (shard skip). Which fields are - // indexed is recorded in the index manifest. Index files of format v3 are complete; older files - // (which could miss records) are rebuilt on first use. + private static function inBox(float $ax, float $ay, float $bx, float $by, float $px, float $py): bool { + return ($ax < $bx ? $ax <= $px && $px <= $bx : $bx <= $px && $px <= $ax) + && ($ay < $by ? $ay <= $py && $py <= $by : $by <= $py && $py <= $ay); + } /** - * Get path to spatial index file - * @param string $dbname Database name - * @param string $field Field name - * @param int|null $shardId Shard ID (null for non-sharded) - * @return string File path + * Sign of the orientation of c against the directed line a -> b: 1 left, -1 right, 0 on the line. + * Exact for all finite doubles: the floating-point determinant decides when it is clearly away from + * zero (Shewchuk's error bound), otherwise it is summed exactly. */ - private function getSpatialIndexPath(string $dbname, string $field, ?int $shardId = null): string { - $hash = $this->hashDBName($dbname); - $name = $this->encodeIndexFieldName($field); - if ($shardId !== null) { - return $this->dbDir . $hash . '_s' . $shardId . '.nonedb.sidx.' . $name; + private static function orient(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + $detLeft = ($ax - $cx) * ($by - $cy); + $detRight = ($ay - $cy) * ($bx - $cx); + $det = $detLeft - $detRight; + if ($detLeft > 0.0) { + if ($detRight <= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = $detLeft + $detRight; + } elseif ($detLeft < 0.0) { + if ($detRight >= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = -$detLeft - $detRight; + } else { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); } - return $this->dbDir . $hash . '.nonedb.sidx.' . $name; + $bound = 3.3306690738754716e-16 * $detSum; // (3 + 16 eps) eps, eps = 2^-53 + if ($det >= $bound || -$det >= $bound) { + return $det > 0.0 ? 1 : -1; + } + return self::orientExact($ax, $ay, $bx, $by, $cx, $cy); } /** - * Get path to global spatial index file (shard MBR metadata) + * Exact sign of (a - c) x (b - c): the differences and products as exact float sums (Dekker, + * Knuth), added into a nonoverlapping expansion whose largest part has the sign */ - private function getGlobalSpatialIndexPath(string $dbname, string $field): string { - return $this->dbDir . $this->hashDBName($dbname) . '.nonedb.gsidx.' . $this->encodeIndexFieldName($field); + private static function orientExact(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + [$acx, $acxTail] = self::twoDiff($ax, $cx); + [$bcy, $bcyTail] = self::twoDiff($by, $cy); + [$acy, $acyTail] = self::twoDiff($ay, $cy); + [$bcx, $bcxTail] = self::twoDiff($bx, $cx); + $sum = []; + foreach ([[$acx, $bcy, 1.0], [$acx, $bcyTail, 1.0], [$acxTail, $bcy, 1.0], [$acxTail, $bcyTail, 1.0], + [$acy, $bcx, -1.0], [$acy, $bcxTail, -1.0], [$acyTail, $bcx, -1.0], [$acyTail, $bcxTail, -1.0]] as [$p, $q, $sign]) { + if ($p == 0.0 || $q == 0.0) { + continue; + } + [$product, $error] = self::twoProduct($p, $q); + $sum = self::growExpansion(self::growExpansion($sum, $sign * $product), $sign * $error); + } + for ($i = count($sum) - 1; $i >= 0; $i--) { + if ($sum[$i] != 0.0) { + return $sum[$i] > 0.0 ? 1 : -1; + } + } + return 0; } - /** - * Read a segment's R-tree. Null when missing or from a pre-3.2 version. - */ - private function readSpatialIndex(string $dbname, string $field, ?int $shardId = null): ?array { - $path = $this->getSpatialIndexPath($dbname, $field, $shardId); - if (array_key_exists($path, $this->spatialIndexCache)) { - return $this->spatialIndexCache[$path]; + /** Add $b to a nonoverlapping expansion (smallest part first), zero parts dropped */ + private static function growExpansion(array $parts, float $b): array { + if ($b == 0.0) { + return $parts; } - $index = $this->atomicRead($path, null); - if (!is_array($index) || ($index['v'] ?? 0) < 3 || !isset($index['nodes'], $index['rootId'], $index['leafOf'])) { - $index = null; + $out = []; + $q = $b; + foreach ($parts as $part) { + [$q, $error] = self::twoSum($q, $part); + if ($error != 0.0) { + $out[] = $error; + } } - $this->spatialIndexCache[$path] = $index; - return $index; + $out[] = $q; + return $out; } - /** - * Write a segment's R-tree (caller holds the exclusive lock) - */ - private function writeSpatialIndex(string $dbname, string $field, array $index, ?int $shardId = null): bool { - $path = $this->getSpatialIndexPath($dbname, $field, $shardId); - $index['updated'] = time(); - if (!$this->atomicWrite($path, $index)) { - unset($this->spatialIndexCache[$path]); - throw new \RuntimeException("noneDB: failed to write spatial index '{$field}' of database '{$dbname}'"); - } - $this->spatialIndexCache[$path] = $index; - unset($this->dirtySpatialIndexes[$path]); - return true; + /** a + b = x + y exactly */ + private static function twoSum(float $a, float $b): array { + $x = $a + $b; + $bVirtual = $x - $a; + $aVirtual = $x - $bVirtual; + return [$x, ($a - $aVirtual) + ($b - $bVirtual)]; } - /** - * Keep an updated R-tree in memory; it is written before the write lock is released - */ - private function markSpatialIndexDirty(string $dbname, string $field, array $index, ?int $shardId = null): void { - $path = $this->getSpatialIndexPath($dbname, $field, $shardId); - $this->spatialIndexCache[$path] = $index; - $this->dirtySpatialIndexes[$path] = ['dbname' => $dbname, 'field' => $field, 'shardId' => $shardId]; + /** a - b = x + y exactly */ + private static function twoDiff(float $a, float $b): array { + $x = $a - $b; + $bVirtual = $a - $x; + $aVirtual = $x + $bVirtual; + return [$x, ($a - $aVirtual) + ($bVirtual - $b)]; } - /** - * Write all dirty spatial indexes to disk (called automatically when a write lock is released) - * @return int Number of indexes flushed - */ - public function flushSpatialIndexes(): int { - $flushed = 0; - foreach ($this->dirtySpatialIndexes as $path => $info) { - if (isset($this->spatialIndexCache[$path])) { - $this->writeSpatialIndex($info['dbname'], $info['field'], $this->spatialIndexCache[$path], $info['shardId']); - $flushed++; - } - } - $this->dirtySpatialIndexes = []; - return $flushed; + /** a * b = x + y exactly (Dekker's split into 26-bit halves) */ + private static function twoProduct(float $a, float $b): array { + $x = $a * $b; + $c = 134217729.0 * $a; // 2^27 + 1 + $aHigh = $c - ($c - $a); + $aLow = $a - $aHigh; + $c = 134217729.0 * $b; + $bHigh = $c - ($c - $b); + $bLow = $b - $bHigh; + $error = (($x - $aHigh * $bHigh) - $aLow * $bHigh) - $aHigh * $bLow; + return [$x, $aLow * $bLow - $error]; } /** - * Read global spatial index (shard MBR metadata); null when missing or pre-3.2 + * An empty AVL tree for the sweeps; $weights[node] is summed per subtree ('s'). Index -1 is the + * empty child (height 0, sum 0). */ - private function readGlobalSpatialIndex(string $dbname, string $field): ?array { - $path = $this->getGlobalSpatialIndexPath($dbname, $field); - if (array_key_exists($path, $this->globalSpatialIndexCache)) { - return $this->globalSpatialIndexCache[$path]; - } - $index = $this->atomicRead($path, null); - if (!is_array($index) || ($index['v'] ?? 0) < 3 || !isset($index['shardMBRs']) || !is_array($index['shardMBRs'])) { - $index = null; - } - $this->globalSpatialIndexCache[$path] = $index; - return $index; - } - - /** - * Write global spatial index (caller holds the exclusive lock) - */ - private function writeGlobalSpatialIndex(string $dbname, string $field, array $index): bool { - $path = $this->getGlobalSpatialIndexPath($dbname, $field); - $index['v'] = 3; - $index['field'] = $field; - $index['updated'] = time(); - if (!$this->atomicWrite($path, $index)) { - unset($this->globalSpatialIndexCache[$path]); - throw new \RuntimeException("noneDB: failed to write global spatial index '{$field}' of database '{$dbname}'"); - } - $this->globalSpatialIndexCache[$path] = $index; - return true; - } - - /** - * Create a new empty R-tree index structure - * @param string $field Field name - * @return array Empty index structure - */ - private function createEmptyRTreeIndex(string $field): array { - return [ - 'v' => 3, - 'field' => $field, - 'type' => 'rtree', - 'nodeSize' => $this->rtreeNodeSize, - 'created' => time(), - 'updated' => time(), - 'rootId' => 0, - 'nextNodeId' => 1, - 'nodes' => [ - '0' => ['isLeaf' => true, 'mbr' => null, 'entries' => [], 'parent' => null] - ], - 'leafOf' => [] - ]; + private static function avlTree(array $weights): array { + return ['l' => [-1 => -1], 'r' => [-1 => -1], 'h' => [-1 => 0], 's' => [-1 => 0], 'w' => $weights, 'root' => -1]; } - /** - * Add an entry to an in-memory R-tree - */ - private function rtreeAddEntry(array &$index, int $key, array $geometry): void { - if (isset($index['leafOf'][$key])) { - $this->rtreeRemoveEntry($index, $key); + /** Insert $node below $x, return the new subtree root; $pred/$succ get its neighbours */ + private static function avlInsert(array &$t, int $x, int $node, \Closure $cmp, int &$pred, int &$succ): int { + if ($x < 0) { + $t['l'][$node] = -1; + $t['r'][$node] = -1; + $t['h'][$node] = 1; + $t['s'][$node] = $t['w'][$node]; + return $node; } - $mbr = $this->calculateMBR($geometry); - $leafId = $this->rtreeChooseLeaf($index, $mbr); - $index['nodes'][$leafId]['entries'][] = ['key' => $key, 'mbr' => $mbr, 'type' => $geometry['type']]; - $index['nodes'][$leafId]['mbr'] = $index['nodes'][$leafId]['mbr'] === null - ? $mbr - : $this->mbrUnion($index['nodes'][$leafId]['mbr'], $mbr); - $index['leafOf'][$key] = $leafId; - if (count($index['nodes'][$leafId]['entries']) > $index['nodeSize']) { - $this->rtreeSplitNode($index, $leafId); + if ($cmp($node, $x) < 0) { + $succ = $x; + $t['l'][$x] = self::avlInsert($t, $t['l'][$x], $node, $cmp, $pred, $succ); + } else { + $pred = $x; + $t['r'][$x] = self::avlInsert($t, $t['r'][$x], $node, $cmp, $pred, $succ); } - $this->rtreeAdjustTree($index, $leafId); + return self::avlBalance($t, $x); } - /** - * Choose the best leaf node for inserting an entry (least enlargement, then smallest area) - * @return string Leaf node ID - */ - private function rtreeChooseLeaf(array &$index, array $entryMBR): string { - $nodeId = (string)$index['rootId']; - - while (!$index['nodes'][$nodeId]['isLeaf']) { - $minEnlargement = PHP_FLOAT_MAX; - $bestChild = null; - $bestArea = PHP_FLOAT_MAX; - - foreach ($index['nodes'][$nodeId]['children'] as $childId) { - $child = $index['nodes'][$childId] ?? null; - if ($child === null) { - continue; - } - $childMBR = $child['mbr']; - if ($childMBR === null) { - if ($child['isLeaf']) { - $bestChild = $childId; // an empty leaf costs nothing - break; - } - continue; // empty internal node: never descend into it - } - $enlargement = $this->mbrEnlargement($childMBR, $entryMBR); - $area = $this->mbrArea($childMBR); - if ($enlargement < $minEnlargement || ($enlargement == $minEnlargement && $area < $bestArea)) { - $minEnlargement = $enlargement; - $bestArea = $area; - $bestChild = $childId; - } - } - - if ($bestChild === null) { - // Internal node without usable children: turn it into an empty leaf - $index['nodes'][$nodeId] = ['isLeaf' => true, 'mbr' => null, 'entries' => [], 'parent' => $index['nodes'][$nodeId]['parent']]; - return $nodeId; + /** Remove $node (it is in the tree) below $x, return the new subtree root */ + private static function avlDelete(array &$t, int $x, int $node, \Closure $cmp): int { + if ($x < 0) { + return -1; + } + if ($x !== $node) { + if ($cmp($node, $x) < 0) { + $t['l'][$x] = self::avlDelete($t, $t['l'][$x], $node, $cmp); + } else { + $t['r'][$x] = self::avlDelete($t, $t['r'][$x], $node, $cmp); } - $nodeId = (string)$bestChild; + return self::avlBalance($t, $x); } - - return $nodeId; - } - - /** - * Split an overflowing node using Linear Split algorithm (O(n) seed selection) - * @param array &$index R-tree index (modified in place) - * @param string $nodeId Node to split - */ - private function rtreeSplitNode(array &$index, string $nodeId): void { - $node = $index['nodes'][$nodeId]; - $isLeaf = $node['isLeaf']; - $entries = $isLeaf ? $node['entries'] : $node['children']; - $n = count($entries); - - if ($n <= $index['nodeSize']) { - return; + $left = $t['l'][$x]; + $right = $t['r'][$x]; + if ($left < 0) { + return $right; } - - $mbrOf = function ($entry) use (&$index, $isLeaf) { - return $isLeaf ? $entry['mbr'] : ($index['nodes'][$entry]['mbr'] ?? null); - }; - - // Linear split: seeds with the greatest normalized separation - $minLon = $maxLon = $minLat = $maxLat = null; - $minLonIdx = $maxLonIdx = $minLatIdx = $maxLatIdx = 0; - $globalMinLon = $globalMinLat = PHP_FLOAT_MAX; - $globalMaxLon = $globalMaxLat = -PHP_FLOAT_MAX; - - for ($i = 0; $i < $n; $i++) { - $mbr = $mbrOf($entries[$i]); - if ($mbr === null) continue; - $globalMinLon = min($globalMinLon, $mbr[0]); - $globalMaxLon = max($globalMaxLon, $mbr[2]); - $globalMinLat = min($globalMinLat, $mbr[1]); - $globalMaxLat = max($globalMaxLat, $mbr[3]); - if ($minLon === null || $mbr[0] > $minLon) { $minLon = $mbr[0]; $minLonIdx = $i; } - if ($minLat === null || $mbr[1] > $minLat) { $minLat = $mbr[1]; $minLatIdx = $i; } - if ($maxLon === null || $mbr[2] < $maxLon) { $maxLon = $mbr[2]; $maxLonIdx = $i; } - if ($maxLat === null || $mbr[3] < $maxLat) { $maxLat = $mbr[3]; $maxLatIdx = $i; } - } - - $lonWidth = $globalMaxLon - $globalMinLon; - $latWidth = $globalMaxLat - $globalMinLat; - $lonSep = ($lonWidth > 0 && $minLon !== null) ? ($minLon - $maxLon) / $lonWidth : 0; - $latSep = ($latWidth > 0 && $minLat !== null) ? ($minLat - $maxLat) / $latWidth : 0; - - if ($lonSep >= $latSep) { - $seed1 = $minLonIdx; - $seed2 = $maxLonIdx; - } else { - $seed1 = $minLatIdx; - $seed2 = $maxLatIdx; + if ($right < 0) { + return $left; } - if ($seed1 === $seed2) { - $seed2 = ($seed1 + 1) % $n; + $next = $right; + while ($t['l'][$next] >= 0) { + $next = $t['l'][$next]; } + $t['r'][$next] = self::avlDeleteMin($t, $right); + $t['l'][$next] = $left; + return self::avlBalance($t, $next); + } - $group1 = [$entries[$seed1]]; - $group2 = [$entries[$seed2]]; - $mbr1 = $mbrOf($entries[$seed1]); - $mbr2 = $mbrOf($entries[$seed2]); - $minEntries = max(1, (int)($index['nodeSize'] * 0.4)); - - for ($i = 0; $i < $n; $i++) { - if ($i == $seed1 || $i == $seed2) continue; - $entry = $entries[$i]; - $entryMBR = $mbrOf($entry); - $remaining = $n - $i - 1; - - if ($entryMBR === null) { - if (count($group1) <= count($group2)) { $group1[] = $entry; } else { $group2[] = $entry; } - continue; - } - if (count($group1) + $remaining <= $minEntries) { - $group1[] = $entry; - $mbr1 = $mbr1 === null ? $entryMBR : $this->mbrUnion($mbr1, $entryMBR); - continue; - } - if (count($group2) + $remaining <= $minEntries) { - $group2[] = $entry; - $mbr2 = $mbr2 === null ? $entryMBR : $this->mbrUnion($mbr2, $entryMBR); - continue; - } - $enlargement1 = $mbr1 === null ? 0 : $this->mbrEnlargement($mbr1, $entryMBR); - $enlargement2 = $mbr2 === null ? 0 : $this->mbrEnlargement($mbr2, $entryMBR); - if ($enlargement1 < $enlargement2 || ($enlargement1 == $enlargement2 && count($group1) <= count($group2))) { - $group1[] = $entry; - $mbr1 = $mbr1 === null ? $entryMBR : $this->mbrUnion($mbr1, $entryMBR); - } else { - $group2[] = $entry; - $mbr2 = $mbr2 === null ? $entryMBR : $this->mbrUnion($mbr2, $entryMBR); - } + private static function avlDeleteMin(array &$t, int $x): int { + if ($t['l'][$x] < 0) { + return $t['r'][$x]; } + $t['l'][$x] = self::avlDeleteMin($t, $t['l'][$x]); + return self::avlBalance($t, $x); + } - $newNodeId = (string)$index['nextNodeId']++; - $currentParent = $index['nodes'][$nodeId]['parent'] ?? null; - - if ($isLeaf) { - $index['nodes'][$nodeId]['entries'] = $group1; - $index['nodes'][$nodeId]['mbr'] = $mbr1; - $index['nodes'][$newNodeId] = ['isLeaf' => true, 'mbr' => $mbr2, 'entries' => $group2, 'parent' => $currentParent]; - foreach ($group2 as $entry) { - $index['leafOf'][$entry['key']] = $newNodeId; - } - } else { - $index['nodes'][$nodeId]['children'] = $group1; - $index['nodes'][$nodeId]['mbr'] = $mbr1; - $index['nodes'][$newNodeId] = ['isLeaf' => false, 'mbr' => $mbr2, 'children' => $group2, 'parent' => $currentParent]; - foreach ($group2 as $childId) { - $index['nodes'][$childId]['parent'] = $newNodeId; - } - foreach ($group1 as $childId) { - $index['nodes'][$childId]['parent'] = $nodeId; + private static function avlBalance(array &$t, int $x): int { + self::avlUpdate($t, $x); + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $diff = $t['h'][$left] - $t['h'][$right]; + if ($diff > 1) { + if ($t['h'][$t['l'][$left]] < $t['h'][$t['r'][$left]]) { + $t['l'][$x] = self::avlRotate($t, $left, true); } + return self::avlRotate($t, $x, false); } - - if ($nodeId == (string)$index['rootId']) { - $newRootId = (string)$index['nextNodeId']++; - $index['nodes'][$newRootId] = [ - 'isLeaf' => false, - 'mbr' => $mbr1 !== null && $mbr2 !== null ? $this->mbrUnion($mbr1, $mbr2) : ($mbr1 ?? $mbr2), - 'children' => [$nodeId, $newNodeId], - 'parent' => null - ]; - $index['rootId'] = (int)$newRootId; - $index['nodes'][$nodeId]['parent'] = $newRootId; - $index['nodes'][$newNodeId]['parent'] = $newRootId; - } elseif ($currentParent !== null) { - $index['nodes'][$currentParent]['children'][] = $newNodeId; - if (count($index['nodes'][$currentParent]['children']) > $index['nodeSize']) { - $this->rtreeSplitNode($index, (string)$currentParent); + if ($diff < -1) { + if ($t['h'][$t['r'][$right]] < $t['h'][$t['l'][$right]]) { + $t['r'][$x] = self::avlRotate($t, $right, false); } + return self::avlRotate($t, $x, true); } + return $x; } - /** - * Recompute one node's MBR from its entries/children - */ - private function rtreeRecomputeMBR(array &$index, string $nodeId): void { - $node = $index['nodes'][$nodeId]; - $mbr = null; - if ($node['isLeaf']) { - foreach ($node['entries'] as $entry) { - $mbr = $mbr === null ? $entry['mbr'] : $this->mbrUnion($mbr, $entry['mbr']); - } + /** Rotate: $left = the right child goes up, else the left child; returns the new subtree root */ + private static function avlRotate(array &$t, int $x, bool $left): int { + if ($left) { + $y = $t['r'][$x]; + $t['r'][$x] = $t['l'][$y]; + $t['l'][$y] = $x; } else { - foreach ($node['children'] as $childId) { - $childMBR = $index['nodes'][$childId]['mbr'] ?? null; - if ($childMBR !== null) { - $mbr = $mbr === null ? $childMBR : $this->mbrUnion($mbr, $childMBR); - } - } + $y = $t['l'][$x]; + $t['l'][$x] = $t['r'][$y]; + $t['r'][$y] = $x; } - $index['nodes'][$nodeId]['mbr'] = $mbr; + self::avlUpdate($t, $x); + self::avlUpdate($t, $y); + return $y; } - /** - * Adjust MBRs of all ancestors of a node - */ - private function rtreeAdjustTree(array &$index, string $nodeId): void { - $parentId = $index['nodes'][$nodeId]['parent'] ?? null; - while ($parentId !== null && isset($index['nodes'][$parentId])) { - $this->rtreeRecomputeMBR($index, (string)$parentId); - $parentId = $index['nodes'][$parentId]['parent'] ?? null; - } + private static function avlUpdate(array &$t, int $x): void { + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $t['h'][$x] = ($t['h'][$left] > $t['h'][$right] ? $t['h'][$left] : $t['h'][$right]) + 1; + $t['s'][$x] = $t['s'][$left] + $t['s'][$right] + $t['w'][$x]; } /** - * Remove an entry from an in-memory R-tree, removing emptied nodes bottom-up - * @return bool True if the key was indexed + * True when a point lies on a ring's boundary */ - private function rtreeRemoveEntry(array &$index, int $key): bool { - $leafId = $index['leafOf'][$key] ?? null; - if ($leafId === null || !isset($index['nodes'][$leafId])) { - return false; - } - $leafId = (string)$leafId; - $entries = $index['nodes'][$leafId]['entries']; - foreach ($entries as $i => $entry) { - if ($entry['key'] === $key) { - array_splice($entries, $i, 1); - break; + public function pointOnRing(float $lon, float $lat, array $ring): bool { + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + if ($this->geometry->crossProductDirection($ring[$i], $ring[$i + 1], [$lon, $lat]) == 0 + && $this->geometry->onSegment($ring[$i], $ring[$i + 1], [$lon, $lat])) { + return true; } } - $index['nodes'][$leafId]['entries'] = $entries; - unset($index['leafOf'][$key]); - - // Condense: drop empty non-root nodes, walking up while parents become empty too - $nodeId = $leafId; - $rootId = (string)$index['rootId']; - while ($nodeId !== $rootId) { - $node = $index['nodes'][$nodeId]; - $empty = $node['isLeaf'] ? empty($node['entries']) : empty($node['children']); - if (!$empty) { - break; - } - $parentId = (string)$node['parent']; - $index['nodes'][$parentId]['children'] = array_values(array_filter( - $index['nodes'][$parentId]['children'], - function ($c) use ($nodeId) { return (string)$c !== $nodeId; } - )); - unset($index['nodes'][$nodeId]); - $nodeId = $parentId; - } - $this->rtreeRecomputeMBR($index, $nodeId); - $this->rtreeAdjustTree($index, $nodeId); - $this->rtreeCondenseTree($index); - return true; + return false; } /** - * Collapse the root: an internal root with no children becomes an empty leaf, one with a single - * child is replaced by that child + * Validate MultiPoint coordinates [[lon,lat], [lon,lat], ...] */ - private function rtreeCondenseTree(array &$index): void { - while (true) { - $rootId = (string)$index['rootId']; - $root = $index['nodes'][$rootId]; - if ($root['isLeaf']) { - return; - } - $children = $root['children'] ?? []; - if (empty($children)) { - $index['nodes'][$rootId] = ['isLeaf' => true, 'mbr' => null, 'entries' => [], 'parent' => null]; - return; - } - if (count($children) > 1) { - return; - } - $newRootId = (string)$children[0]; - unset($index['nodes'][$rootId]); - $index['rootId'] = (int)$newRootId; - $index['nodes'][$newRootId]['parent'] = null; + private function validateGeoJSONMultiPoint($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPoint must have at least one point', 'type' => null]; } - } - - /** - * Search R-tree for entries overlapping any of the given MBRs - * @param array $searchMBR One MBR [minLon, minLat, maxLon, maxLat] or a list of MBRs - * @return array Matching keys - */ - private function rtreeSearchMBR(string $dbname, string $field, array $searchMBR, ?int $shardId = null): array { - $index = $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); - $boxes = is_array($searchMBR[0] ?? null) ? $searchMBR : [$searchMBR]; - $results = []; - foreach ($boxes as $box) { - $found = []; - $this->rtreeSearchNode($index, (string)$index['rootId'], $box, $found); - foreach ($found as $key) { - $results[$key] = $key; + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid point at index $i: " . $result['error'], 'type' => null]; } } - return array_values($results); + return ['valid' => true, 'error' => null, 'type' => 'MultiPoint']; } /** - * Recursive search helper + * Validate MultiLineString coordinates [[[lon,lat], ...], ...] */ - private function rtreeSearchNode(array &$index, string $nodeId, array $searchMBR, array &$results): void { - $node = $index['nodes'][$nodeId] ?? null; - if ($node === null || $node['mbr'] === null || !$this->mbrOverlaps($node['mbr'], $searchMBR)) { - return; + private function validateGeoJSONMultiLineString($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiLineString must have at least one linestring', 'type' => null]; } - if ($node['isLeaf']) { - foreach ($node['entries'] as $entry) { - if ($this->mbrOverlaps($entry['mbr'], $searchMBR)) { - $results[] = $entry['key']; - } - } - } else { - foreach ($node['children'] as $childId) { - $this->rtreeSearchNode($index, (string)$childId, $searchMBR, $results); + foreach ($coords as $i => $lineString) { + $result = $this->validateGeoJSONLineString($lineString); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid linestring at index $i: " . $result['error'], 'type' => null]; } } + return ['valid' => true, 'error' => null, 'type' => 'MultiLineString']; } /** - * Build a segment's R-tree from its data + * Validate MultiPolygon coordinates [[[[lon,lat], ...], ...], ...] */ - private function buildSegmentSpatialIndex(string $dbname, string $field, ?int $shardId, array $options = []): array { - $index = $this->createEmptyRTreeIndex($field); - if (isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4) { - $index['nodeSize'] = (int)$options['nodeSize']; + private function validateGeoJSONMultiPolygon($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPolygon must have at least one polygon', 'type' => null]; } - foreach ($this->readSegmentRecords($dbname, $shardId) as $key => $record) { - $geometry = $record[$field] ?? null; - if (is_array($geometry) && $this->validateGeoJSON($geometry)['valid']) { - $this->rtreeAddEntry($index, (int)$key, $geometry); + foreach ($coords as $i => $polygon) { + $result = $this->validateGeoJSONPolygon($polygon); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid polygon at index $i: " . $result['error'], 'type' => null]; } } - return $index; + return ['valid' => true, 'error' => null, 'type' => 'MultiPolygon']; } /** - * A segment's R-tree, (re)built when missing or from an older version (exclusive lock) + * Validate GeometryCollection */ - private function ensureSegmentSpatialIndex(string $dbname, string $field, ?int $shardId): array { - $index = $this->readSpatialIndex($dbname, $field, $shardId); - if ($index !== null) { - return $index; + private function validateGeometryCollection(array $geometry): array { + if (!isset($geometry['geometries']) || !self::isJsonList($geometry['geometries'])) { + return ['valid' => false, 'error' => 'GeometryCollection must have geometries array', 'type' => null]; } - return $this->withLock($dbname, true, function () use ($dbname, $field, $shardId) { - unset($this->spatialIndexCache[$this->getSpatialIndexPath($dbname, $field, $shardId)]); - $index = $this->readSpatialIndex($dbname, $field, $shardId); - if ($index === null) { - $nodeSize = $this->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null; - $index = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, $nodeSize ? ['nodeSize' => $nodeSize] : []); - $this->writeSpatialIndex($dbname, $field, $index, $shardId); - if ($shardId !== null) { - $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); - } + foreach ($geometry['geometries'] as $i => $geom) { + if (!is_array($geom)) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: must be an object", 'type' => null]; } - return $index; - }); + $result = $this->validateGeoJSON($geom); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'GeometryCollection']; } +} +} +namespace noneDB\Internal\Index\Spatial { - /** - * Record a shard's bounding box in the global spatial index - */ - private function updateGlobalSpatialShard(string $dbname, string $field, int $shardId, array $index): void { - $global = $this->readGlobalSpatialIndex($dbname, $field) ?? ['v' => 3, 'field' => $field, 'shardMBRs' => []]; - $mbr = $index['nodes'][(string)$index['rootId']]['mbr'] ?? null; - $current = $global['shardMBRs'][(string)$shardId] ?? null; - if ($mbr === $current) { - return; - } - if ($mbr === null) { - unset($global['shardMBRs'][(string)$shardId]); - } else { - $global['shardMBRs'][(string)$shardId] = $mbr; - } - $this->writeGlobalSpatialIndex($dbname, $field, $global); +use function array_merge, atan2, cos, count, deg2rad, max, min, sin, sqrt; + +/** + * @internal + */ +final class Geometry { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; } /** - * Rebuild a spatial index for the whole database (all segments + global shard boxes) + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - private function rebuildSpatialIndexForDb($dbname, $field){ - $manifest = $this->getIndexManifest($dbname); - $nodeSize = $manifest['spatialOptions'][$field]['nodeSize'] ?? null; - $options = $nodeSize ? ['nodeSize' => $nodeSize] : []; - @unlink($this->getSpatialIndexPath($dbname, $field, null)); - unset($this->spatialIndexCache[$this->getSpatialIndexPath($dbname, $field, null)]); - $indexed = 0; - $sharded = $this->isSharded($dbname); - if ($sharded) { - @unlink($this->getGlobalSpatialIndexPath($dbname, $field)); - unset($this->globalSpatialIndexCache[$this->getGlobalSpatialIndexPath($dbname, $field)]); - } - foreach ($this->listSegments($dbname) as $segment) { - $index = $this->buildSegmentSpatialIndex($dbname, $field, $segment, $options); - $this->writeSpatialIndex($dbname, $field, $index, $segment); - $indexed += count($index['leafOf']); - if ($sharded) { - $this->updateGlobalSpatialShard($dbname, $field, $segment, $index); - } - } - if ($sharded && $this->readGlobalSpatialIndex($dbname, $field) === null) { - $this->writeGlobalSpatialIndex($dbname, $field, ['v' => 3, 'field' => $field, 'shardMBRs' => []]); - } - return $indexed; + public function bind(): void { } /** - * Delete the spatial index files of one shard + * Drop the handles to the kernel and the other services (the instance is gone) */ - private function removeSpatialShardFiles($dbname, $shardId){ - $prefix = $this->dbDir . $this->hashDBName($dbname) . '_s' . (int)$shardId . '.nonedb.sidx.'; - foreach (glob($prefix . '*') ?: [] as $file) { - @unlink($file); - unset($this->spatialIndexCache[$file], $this->dirtySpatialIndexes[$file]); - } - foreach ($this->getIndexManifest($dbname)['spatial'] as $field) { - $global = $this->readGlobalSpatialIndex($dbname, $field); - if ($global !== null && isset($global['shardMBRs'][(string)$shardId])) { - unset($global['shardMBRs'][(string)$shardId]); - $this->writeGlobalSpatialIndex($dbname, $field, $global); - } - } + public function unbind(): void { + $this->k = null; + $this->rt = null; } + // ========================================== + // SPATIAL INDEX - GEOMETRY OPERATIONS + // ========================================== + /** - * Keep spatial indexes in sync after records of a segment changed (called by the storage core) - * @param array $removed [key => old record] - * @param array $added [key => new record] + * Calculate Minimum Bounding Rectangle (MBR) for any GeoJSON geometry + * @param array $geometry GeoJSON geometry object + * @return array|null [minLon, minLat, maxLon, maxLat]; null for a geometry without any position + * (an empty GeometryCollection): it has no location and is never indexed or matched */ - private function applySpatialIndexChanges($dbname, $shardId, array $removed, array $added){ - // Maintained even when spatialIndexEnabled is off; that flag only affects creating/using indexes - if (empty($removed) && empty($added)) { - return; - } - foreach ($this->getIndexManifest($dbname)['spatial'] as $field) { - $existing = $this->readSpatialIndex($dbname, $field, $shardId); - if ($existing === null) { - // Missing (new shard) or pre-3.2: built from the data, which already holds the change - $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); - continue; - } - $index = $existing; - $changed = false; - foreach ($removed as $key => $old) { - if (isset($added[$key]) && ($old[$field] ?? null) === ($added[$key][$field] ?? null)) { - continue; // geometry unchanged - } - $changed = $this->rtreeRemoveEntry($index, (int)$key) || $changed; - } - foreach ($added as $key => $new) { - if (isset($removed[$key]) && ($removed[$key][$field] ?? null) === ($new[$field] ?? null)) { - continue; + public function calculateMBR(array $geometry): ?array { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + + switch ($type) { + case 'Point': + return [$coords[0], $coords[1], $coords[0], $coords[1]]; + + case 'LineString': + case 'MultiPoint': + return $this->mbrFromPoints($coords); + + case 'Polygon': + // Use outer ring (first ring) + return $this->mbrFromPoints($coords[0]); + + case 'MultiLineString': + $allPoints = []; + foreach ($coords as $lineString) { + $allPoints = array_merge($allPoints, $lineString); } - $geometry = $new[$field] ?? null; - if (is_array($geometry) && $this->validateGeoJSON($geometry)['valid']) { - $this->rtreeAddEntry($index, (int)$key, $geometry); - $changed = true; + return $this->mbrFromPoints($allPoints); + + case 'MultiPolygon': + $allPoints = []; + foreach ($coords as $polygon) { + $allPoints = array_merge($allPoints, $polygon[0]); // outer ring } - } - if ($changed) { - $this->markSpatialIndexDirty($dbname, $field, $index, $shardId); - if ($shardId !== null) { - $this->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + return $this->mbrFromPoints($allPoints); + + case 'GeometryCollection': + $mbr = null; + foreach ($geometry['geometries'] as $geom) { + $geomMBR = $this->calculateMBR($geom); + if ($geomMBR !== null) { + $mbr = $mbr === null ? $geomMBR : $this->mbrUnion($mbr, $geomMBR); + } } - } + return $mbr; + + default: + return null; } } /** - * Check if a spatial index exists - * @param string $dbname Database name - * @param string $field Field name - * @param int|null $shardId Shard ID - * @return bool + * Calculate MBR from array of points + * @param array $points Array of [lon, lat] positions + * @return array [minLon, minLat, maxLon, maxLat] */ - public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $shardId){ - return $this->hasSpatialIndexUnlocked($dbname, $field, $shardId); - }); - } + public function mbrFromPoints(array $points): ?array { + if (empty($points)) { + return null; + } - private function hasSpatialIndexUnlocked(string $dbname, string $field, ?int $shardId = null): bool{ - if (!$this->dbExists($dbname) || !in_array($field, $this->getIndexManifest($dbname)['spatial'], true)) { - return false; + $minLon = $maxLon = $points[0][0]; + $minLat = $maxLat = $points[0][1]; + + foreach ($points as $point) { + $minLon = min($minLon, $point[0]); + $maxLon = max($maxLon, $point[0]); + $minLat = min($minLat, $point[1]); + $maxLat = max($maxLat, $point[1]); } - return $shardId === null || in_array($shardId, $this->listSegments($dbname), true); + + return [$minLon, $minLat, $maxLon, $maxLat]; } - // ========================================== - // SPATIAL INDEX - PUBLIC API - // ========================================== + /** + * Union two MBRs + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return array [minLon, minLat, maxLon, maxLat] + */ + private function mbrUnion(array $mbr1, array $mbr2): array { + return [ + min($mbr1[0], $mbr2[0]), // minLon + min($mbr1[1], $mbr2[1]), // minLat + max($mbr1[2], $mbr2[2]), // maxLon + max($mbr1[3], $mbr2[3]) // maxLat + ]; + } /** - * Create a spatial index on a GeoJSON field - * @param string $dbname Database name - * @param string $field Field name containing GeoJSON - * @param array $options Options ['nodeSize' => 32] - * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + * Check if two MBRs overlap + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return bool */ - public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field, $options){ - return $this->createSpatialIndexUnlocked($dbname, $field, $options); - }); + public function mbrOverlaps(array $mbr1, array $mbr2): bool { + // Check if one MBR is completely to the left, right, above, or below the other + if ($mbr1[2] < $mbr2[0] || $mbr2[2] < $mbr1[0]) return false; // No horizontal overlap + if ($mbr1[3] < $mbr2[1] || $mbr2[3] < $mbr1[1]) return false; // No vertical overlap + return true; } - private function createSpatialIndexUnlocked(string $dbname, string $field, array $options = []): array{ - if (!$this->spatialIndexEnabled) { - return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; - } - if ($field === '' || $field === 'key') { - return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + /** + * Calculate Haversine distance between two points (in meters). Not memoized: a key for two points costs + * more than the formula + * @param float $lon1 Longitude of point 1 + * @param float $lat1 Latitude of point 1 + * @param float $lon2 Longitude of point 2 + * @param float $lat2 Latitude of point 2 + * @return float Distance in meters + */ + public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float { + $earthRadius = 6371000.0; // meters + + $latDelta = deg2rad($lat2 - $lat1); + $lonDelta = deg2rad($lon2 - $lon1); + + $a = sin($latDelta / 2) * sin($latDelta / 2) + + cos(deg2rad($lat1)) * cos(deg2rad($lat2)) * + sin($lonDelta / 2) * sin($lonDelta / 2); + + $c = 2 * atan2(sqrt($a), sqrt(1 - $a)); + + return $earthRadius * $c; + } + + /** + * Point-in-Polygon test using Winding Number algorithm + * More robust than ray casting for edge cases + * + * @param float $lon Point longitude + * @param float $lat Point latitude + * @param array $polygon GeoJSON Polygon coordinates [[[lon,lat], ...], ...] + * @return bool + */ + public function pointInPolygon(float $lon, float $lat, array $polygon): bool { + // Handle GeoJSON Polygon object or raw coordinates + $coords = isset($polygon['coordinates']) ? $polygon['coordinates'] : $polygon; + + // Check outer ring + if (!$this->pointInRing($lon, $lat, $coords[0])) { + return false; } - $this->flushPendingBuffers($dbname); - if (!$this->dbExists($dbname)) { - if (!$this->autoCreateDB) { - return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + + // Check holes (if any) + for ($i = 1; $i < count($coords); $i++) { + if ($this->pointInRing($lon, $lat, $coords[$i])) { + return false; // Inside a hole } - $this->createDBUnlocked($dbname); } - $manifest = $this->getIndexManifest($dbname); - if (in_array($field, $manifest['spatial'], true)) { - return ['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field '$field'"]; - } + return true; + } - $manifest['spatial'][] = $field; - if (isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4) { - $manifest['spatialOptions'][$field] = ['nodeSize' => (int)$options['nodeSize']]; + /** + * Check if point is inside a ring using Winding Number algorithm + */ + public function pointInRing(float $lon, float $lat, array $ring): bool { + $windingNumber = 0; + $n = count($ring); + + for ($i = 0; $i < $n - 1; $i++) { + $x1 = $ring[$i][0]; + $y1 = $ring[$i][1]; + $x2 = $ring[$i + 1][0]; + $y2 = $ring[$i + 1][1]; + + if ($y1 <= $lat) { + if ($y2 > $lat) { + // Upward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) > 0) { + $windingNumber++; + } + } + } else { + if ($y2 <= $lat) { + // Downward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) < 0) { + $windingNumber--; + } + } + } } - $this->saveIndexManifest($dbname, $manifest); - $indexed = $this->rebuildSpatialIndexForDb($dbname, $field); - return ['success' => true, 'indexed' => $indexed, 'error' => null]; + return $windingNumber !== 0; } /** - * Drop a spatial index - * @param string $dbname Database name - * @param string $field Field name - * @return array ['success' => bool, 'error' => ?string] + * Determine if point is left of a line (using cross product) + * Returns: >0 if left, <0 if right, =0 if on line */ - public function dropSpatialIndex(string $dbname, string $field): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field){ - return $this->dropSpatialIndexUnlocked($dbname, $field); - }); + private function isLeftOfLine(float $x1, float $y1, float $x2, float $y2, float $px, float $py): float { + return (($x2 - $x1) * ($py - $y1)) - (($px - $x1) * ($y2 - $y1)); } - private function dropSpatialIndexUnlocked(string $dbname, string $field): array{ - foreach (array_merge([null], $this->isSharded($dbname) ? $this->listSegments($dbname) : []) as $segment) { - $path = $this->getSpatialIndexPath($dbname, $field, $segment); - @unlink($path); - unset($this->spatialIndexCache[$path], $this->dirtySpatialIndexes[$path]); - } - $globalPath = $this->getGlobalSpatialIndexPath($dbname, $field); - @unlink($globalPath); - unset($this->globalSpatialIndexCache[$globalPath]); + /** + * Cross product direction calculation + */ + public function crossProductDirection(array $p1, array $p2, array $p3): float { + return ($p3[0] - $p1[0]) * ($p2[1] - $p1[1]) - ($p2[0] - $p1[0]) * ($p3[1] - $p1[1]); + } - if ($this->dbExists($dbname)) { - $manifest = $this->getIndexManifest($dbname); - $manifest['spatial'] = array_values(array_diff($manifest['spatial'], [$field])); - unset($manifest['spatialOptions'][$field]); - $this->saveIndexManifest($dbname, $manifest); - } - return ['success' => true, 'error' => null]; + /** + * Check if point p is on segment p1-p2 + */ + public function onSegment(array $p1, array $p2, array $p): bool { + return min($p1[0], $p2[0]) <= $p[0] && $p[0] <= max($p1[0], $p2[0]) && + min($p1[1], $p2[1]) <= $p[1] && $p[1] <= max($p1[1], $p2[1]); + } +} +} +namespace noneDB\Internal\Index\Spatial { + +use function array_chunk, array_key_exists, array_map, array_merge, array_multisort, array_pop, array_search, array_slice, + array_values, ceil, count, crc32, floor, in_array, intdiv, is_array, is_int, json_decode, ksort, max, min, pack, sqrt, + strlen, strpos, substr, unpack; +use const INF, SORT_NUMERIC; + +/** + * @internal + */ +final class RTree { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $globalSpatialIndexCache; + private $rtreeNodeSize; + private $spatialIndexCache; + private $cacheGrew; + private $cacheTick; + private $cacheTouch; + private $lockRegistry; + private $metrics; + private $multiLock; + private $noWrites; + private $cache; + private $checkpoints; + private $fieldIndex; + private $geoJson; + private $geometry; + private $io; + private $layout; + private $locks; + private $records; + private $secondary; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->geoJson, $this->geometry, $this->io, $this->layout, $this->locks, $this->records, $this->secondary, $this->segments); } /** - * Get list of spatial indexes for a database - * @param string $dbname Database name - * @return array List of indexed field names + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) */ - public function getSpatialIndexes(string $dbname): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname){ - return $this->getSpatialIndexesUnlocked($dbname); - }); + public function bind(): void { + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->rtreeNodeSize = &$this->k->rtreeNodeSize; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; } - private function getSpatialIndexesUnlocked(string $dbname): array{ - return $this->dbExists($dbname) ? $this->getIndexManifest($dbname)['spatial'] : []; + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; } /** - * Rebuild a spatial index - * @param string $dbname Database name - * @param string $field Field name - * @return array Result from createSpatialIndex + * Drop the handles to the kernel and the other services (the instance is gone) */ - public function rebuildSpatialIndex(string $dbname, string $field): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname, $field){ - return $this->rebuildSpatialIndexUnlocked($dbname, $field); - }); + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->secondary = null; + $this->segments = null; + $this->k = null; + $this->rt = null; } - private function rebuildSpatialIndexUnlocked(string $dbname, string $field): array{ - $options = []; - $nodeSize = $this->dbExists($dbname) ? ($this->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null) : null; - if ($nodeSize) { - $options['nodeSize'] = $nodeSize; - } - $this->dropSpatialIndexUnlocked($dbname, $field); - return $this->createSpatialIndexUnlocked($dbname, $field, $options); + + /** + * Path of a segment's R-tree + */ + public function getSpatialIndexPath(string $dbname, string $field, ?int $shardId = null): string { + return $this->secondary->secondaryPath($dbname, 'rx', $field, $shardId); } - // ========================================== - // SPATIAL QUERIES - // ========================================== + public function getGlobalSpatialIndexPath(string $dbname, string $field): string { + return $this->secondary->secondaryPath($dbname, 'grx', $field); + } /** - * Records whose geometry may overlap the search boxes: [key => record]. - * Uses the R-tree (and shard boxes) when the field is indexed; otherwise every record. - * @param array $boxes List of MBRs + * Integer box of a geometry (outwards), or null when it has no position */ - private function spatialCandidates(string $dbname, string $field, array $boxes): array { - if (!$this->dbExists($dbname)) { - return []; + private function quantizedMBR(array $geometry): ?array { + $mbr = $this->geometry->calculateMBR($geometry); + return $mbr === null ? null : $this->quantizeBox($mbr); + } + + /** + * Integer box of [minLon, minLat, maxLon, maxLat] (clamped to the globe, rounded outwards) + */ + private function quantizeBox(array $box): array { + $minX = max(-180.0, min(180.0, (float)$box[0])); + $minY = max(-90.0, min(90.0, (float)$box[1])); + $maxX = max(-180.0, min(180.0, (float)$box[2])); + $maxY = max(-90.0, min(90.0, (float)$box[3])); + return [(int)floor($minX * self::RTREE_SCALE), (int)floor($minY * self::RTREE_SCALE), + (int)ceil($maxX * self::RTREE_SCALE), (int)ceil($maxY * self::RTREE_SCALE)]; + } + + private static function boxUnion(?array $a, array $b): array { + if($a === null){ + return $b; } - $indexed = $this->spatialIndexEnabled && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); - $records = []; - $sharded = $this->isSharded($dbname); - $global = ($indexed && $sharded) ? $this->readGlobalSpatialIndex($dbname, $field) : null; + return [$a[0] < $b[0] ? $a[0] : $b[0], $a[1] < $b[1] ? $a[1] : $b[1], $a[2] > $b[2] ? $a[2] : $b[2], $a[3] > $b[3] ? $a[3] : $b[3]]; + } - foreach ($this->listSegments($dbname) as $segment) { - if (!$indexed) { - $records += $this->readSegmentRecords($dbname, $segment); - continue; - } - if ($global !== null) { - $shardMBR = $global['shardMBRs'][(string)$segment] ?? null; - $overlaps = false; - foreach ($boxes as $box) { - if ($shardMBR !== null && $this->mbrOverlaps($shardMBR, $box)) { - $overlaps = true; - break; - } + private static function boxOverlaps(?array $a, array $b): bool { + return $a !== null && $a[0] <= $b[2] && $b[0] <= $a[2] && $a[1] <= $b[3] && $b[1] <= $a[3]; + } + + private static function isBox($box): bool { + return is_int($box[0] ?? null) && is_int($box[1] ?? null) && is_int($box[2] ?? null) && is_int($box[3] ?? null); + } + + private static function boxArea(array $a): float { + return ((float)$a[2] - $a[0]) * ((float)$a[3] - $a[1]); + } + + public function newRTree(string $field, int $nodeSize): array { + return ['kind' => 'rx', 'field' => $field, 'ns' => $nodeSize, 's' => 0, 'ck' => -1, 'e' => null, 'bytes' => 0, + 'nodes' => [0 => [0, null, []]], 'root' => 0, 'next' => 1, 'mk' => -1, 'cnt' => 0, 'leafOf' => null, 'parent' => null]; + } + + /** + * Bulk load (sort-tile-recursive): leaves 3/4 full, ties broken by key (the same tree on every PHP). The entries + * come as columns, sorted with array_multisort(): no array per entry until the leaves are made + * @param int[] $ids keys (the leaves), then node ids (the levels above) + * @param int[] $minX, $minY, $maxX, $maxY their boxes, in the same order + */ + private function rtreeBulkLoad(array &$tree, array $ids, array $minX, array $minY, array $maxX, array $maxY): void { + $capacity = max(2, intdiv($tree['ns'] * 3, 4)); + $tree['nodes'] = []; + $tree['next'] = 0; + $tree['cnt'] = count($ids); + $tree['leafOf'] = null; + $tree['parent'] = null; + if(empty($ids)){ + $tree['nodes'][0] = [0, null, []]; + $tree['root'] = 0; + $tree['next'] = 1; + return; + } + for($level = 0; ; $level++){ + $count = count($ids); + $perSlice = (int)ceil(sqrt((int)ceil($count / $capacity))) * $capacity; + // By the x center (doubled: min + max), then the id + $center = []; + for($i = 0; $i < $count; $i++){ + $center[] = $minX[$i] + $maxX[$i]; + } + array_multisort($center, SORT_NUMERIC, $ids, SORT_NUMERIC, $minX, $minY, $maxX, $maxY); + $center = null; + $up = [[], [], [], [], []]; // the nodes made: ids and boxes + for($i = 0; $i < $count; $i += $perSlice){ + // A vertical slice, by the y center, then the id + $sIds = array_slice($ids, $i, $perSlice); + $sMinX = array_slice($minX, $i, $perSlice); + $sMinY = array_slice($minY, $i, $perSlice); + $sMaxX = array_slice($maxX, $i, $perSlice); + $sMaxY = array_slice($maxY, $i, $perSlice); + $m = count($sIds); + $center = []; + for($j = 0; $j < $m; $j++){ + $center[] = $sMinY[$j] + $sMaxY[$j]; } - if (!$overlaps) { - // Verify the shard box is current before skipping: it's rebuilt with the shard index - if ($this->readSpatialIndex($dbname, $field, $segment) !== null) { - continue; + array_multisort($center, SORT_NUMERIC, $sIds, SORT_NUMERIC, $sMinX, $sMinY, $sMaxX, $sMaxY); + for($j = 0; $j < $m; $j += $capacity){ + $id = $tree['next']++; + $box = [$sMinX[$j], $sMinY[$j], $sMaxX[$j], $sMaxY[$j]]; + $payload = []; + for($k = $j, $end = min($m, $j + $capacity); $k < $end; $k++){ + $box = [min($box[0], $sMinX[$k]), min($box[1], $sMinY[$k]), max($box[2], $sMaxX[$k]), max($box[3], $sMaxY[$k])]; + $payload[] = $level === 0 ? [$sIds[$k], $sMinX[$k], $sMinY[$k], $sMaxX[$k], $sMaxY[$k]] : $sIds[$k]; } + $tree['nodes'][$id] = [$level, $box, $payload]; + $up[0][] = $id; + $up[1][] = $box[0]; + $up[2][] = $box[1]; + $up[3][] = $box[2]; + $up[4][] = $box[3]; } } - $keys = $this->rtreeSearchMBR($dbname, $field, $boxes, $segment); - if (!empty($keys)) { - $records += $this->readSegmentRecords($dbname, $segment, $keys); + if(count($up[0]) === 1){ + $tree['root'] = $up[0][0]; + return; } + [$ids, $minX, $minY, $maxX, $maxY] = $up; } - return $records; } /** - * Search boxes for a circle; splits at the antimeridian and covers all longitudes near the poles - * @return array List of [minLon, minLat, maxLon, maxLat] + * Parent and leaf maps (built when the first change needs them) */ - private function circleToBBoxes(float $lon, float $lat, float $radiusMeters): array { - $latDeg = $radiusMeters / 111000.0; - $minLat = $lat - $latDeg; - $maxLat = $lat + $latDeg; - if ($minLat <= -90 || $maxLat >= 90) { - return [[-180.0, max(-90.0, $minLat), 180.0, min(90.0, $maxLat)]]; - } - $cos = cos(deg2rad(max(abs($minLat), abs($maxLat)))); - $lonDeg = $cos > 1e-9 ? $radiusMeters / (111000.0 * $cos) : 360.0; - if ($lonDeg >= 180) { - return [[-180.0, $minLat, 180.0, $maxLat]]; - } - $minLon = $lon - $lonDeg; - $maxLon = $lon + $lonDeg; - if ($minLon < -180) { - return [[-180.0, $minLat, $maxLon, $maxLat], [$minLon + 360.0, $minLat, 180.0, $maxLat]]; + public function rtreeMaps(array &$tree): void { + if($tree['parent'] !== null){ + return; } - if ($maxLon > 180) { - return [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon - 360.0, $maxLat]]; + $parent = []; + $leafOf = []; + foreach($tree['nodes'] as $id => $node){ + if($node[0] === 0){ + foreach($node[2] as $entry){ + $leafOf[$entry[0]] = $id; + } + } else { + foreach($node[2] as $child){ + $parent[$child] = $id; + } + } } - return [[$minLon, $minLat, $maxLon, $maxLat]]; + $tree['parent'] = $parent; + $tree['leafOf'] = $leafOf; } /** - * Distance in meters from a point to a geometry: 0 inside a polygon, otherwise to the nearest - * vertex or edge (edges use a local equirectangular approximation) + * Insert (or move) a key with its box. The way down is remembered for the box updates and splits, + * so a key never seen before needs no parent map (built only when something is removed). */ - private function distanceToGeometry(float $lon, float $lat, array $geometry): float { - $type = $geometry['type'] ?? ''; - $coords = $geometry['coordinates'] ?? []; - switch ($type) { - case 'Point': - return $this->haversineDistance($lon, $lat, (float)$coords[0], (float)$coords[1]); - case 'MultiPoint': - $best = INF; - foreach ($coords as $p) { - $best = min($best, $this->haversineDistance($lon, $lat, (float)$p[0], (float)$p[1])); + private function rtreeInsert(array &$tree, int $key, array $box): void { + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } else { + $tree['mk'] = $key; + } + $path = []; + $id = $tree['root']; + while($tree['nodes'][$id][0] > 0){ + $best = null; + $bestGrowth = INF; + $bestArea = INF; + foreach($tree['nodes'][$id][2] as $child){ + $childBox = $tree['nodes'][$child][1]; + if($childBox === null){ + $best = $best ?? $child; + continue; } - return $best; - case 'LineString': - return $this->distanceToPath($lon, $lat, $coords); - case 'MultiLineString': - $best = INF; - foreach ($coords as $line) { - $best = min($best, $this->distanceToPath($lon, $lat, $line)); - } - return $best; - case 'Polygon': - if ($this->pointInPolygon($lon, $lat, $coords)) { - return 0.0; - } - $best = INF; - foreach ($coords as $ring) { - $best = min($best, $this->distanceToPath($lon, $lat, $ring)); - } - return $best; - case 'MultiPolygon': - $best = INF; - foreach ($coords as $polygon) { - $best = min($best, $this->distanceToGeometry($lon, $lat, ['type' => 'Polygon', 'coordinates' => $polygon])); - } - return $best; - case 'GeometryCollection': - $best = INF; - foreach ($geometry['geometries'] ?? [] as $g) { - $best = min($best, $this->distanceToGeometry($lon, $lat, $g)); + $area = self::boxArea($childBox); + $growth = self::boxArea(self::boxUnion($childBox, $box)) - $area; + if($growth < $bestGrowth || ($growth == $bestGrowth && $area < $bestArea)){ + $best = $child; + $bestGrowth = $growth; + $bestArea = $area; } - return $best; + } + if($best === null){ + $tree['nodes'][$id] = [0, null, []]; // an internal node without children becomes a leaf + break; + } + $path[] = $id; + $id = $best; } - return INF; + $tree['nodes'][$id][2][] = [$key, $box[0], $box[1], $box[2], $box[3]]; + if($tree['leafOf'] !== null){ + $tree['leafOf'][$key] = $id; + } + $tree['cnt']++; + // Widen the boxes on the way up while they change + for($node = $id, $i = count($path); ; $node = $path[--$i]){ + $old = $tree['nodes'][$node][1]; + $new = self::boxUnion($old, $box); + if($old === $new){ + break; + } + $tree['nodes'][$node][1] = $new; + if($i === 0){ + break; + } + } + if(count($tree['nodes'][$id][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $id, $path); + } + } + + private function rtreeBoxOf(array $tree, int $level, $item): ?array { + return $level === 0 ? [$item[1], $item[2], $item[3], $item[4]] : $tree['nodes'][$item][1]; } /** - * Distance from a point to a polyline (meters) + * Split an overflowing node (linear seeds, least enlargement), up to the root + * @param array $path The node's ancestors, root first */ - private function distanceToPath(float $lon, float $lat, array $points): float { - $n = count($points); - if ($n === 0) { - return INF; + private function rtreeSplit(array &$tree, int $id, array $path): void { + [$level, , $items] = $tree['nodes'][$id]; + $n = count($items); + $boxes = []; + foreach($items as $i => $item){ + $boxes[$i] = $this->rtreeBoxOf($tree, $level, $item) ?? [0, 0, 0, 0]; + } + // Seeds: the pair farthest apart along the axis with the greatest normalized separation + $best = null; + foreach([0, 1] as $axis){ + $lowHigh = 0; + $highLow = 0; + $min = INF; + $max = -INF; + foreach($boxes as $i => $b){ + if($b[$axis] > $boxes[$lowHigh][$axis]){ + $lowHigh = $i; + } + if($b[$axis + 2] < $boxes[$highLow][$axis + 2]){ + $highLow = $i; + } + $min = min($min, $b[$axis]); + $max = max($max, $b[$axis + 2]); + } + $width = $max - $min; + $separation = $width > 0 ? ($boxes[$lowHigh][$axis] - $boxes[$highLow][$axis + 2]) / $width : 0; + if($best === null || $separation > $best[0]){ + $best = [$separation, $lowHigh, $highLow]; + } } - $best = $this->haversineDistance($lon, $lat, (float)$points[0][0], (float)$points[0][1]); - $kx = cos(deg2rad($lat)); - for ($i = 0; $i < $n - 1; $i++) { - $ax = ((float)$points[$i][0] - $lon) * $kx; - $ay = (float)$points[$i][1] - $lat; - $bx = ((float)$points[$i + 1][0] - $lon) * $kx; - $by = (float)$points[$i + 1][1] - $lat; - $dx = $bx - $ax; - $dy = $by - $ay; - $len2 = $dx * $dx + $dy * $dy; - $t = $len2 > 0 ? max(0.0, min(1.0, -($ax * $dx + $ay * $dy) / $len2)) : 0.0; - $px = (float)$points[$i][0] + ((float)$points[$i + 1][0] - (float)$points[$i][0]) * $t; - $py = (float)$points[$i][1] + ((float)$points[$i + 1][1] - (float)$points[$i][1]) * $t; - $best = min($best, $this->haversineDistance($lon, $lat, $px, $py)); + [, $seed1, $seed2] = $best; + if($seed1 === $seed2){ + $seed2 = ($seed1 + 1) % $n; + } + $groups = [[$items[$seed1]], [$items[$seed2]]]; + $groupBoxes = [$boxes[$seed1], $boxes[$seed2]]; + $minFill = max(1, (int)($tree['ns'] * 0.4)); + $left = $n - 2; + foreach($items as $i => $item){ + if($i === $seed1 || $i === $seed2){ + continue; + } + if(count($groups[0]) + $left <= $minFill){ + $g = 0; + } elseif(count($groups[1]) + $left <= $minFill){ + $g = 1; + } else { + $grow0 = self::boxArea(self::boxUnion($groupBoxes[0], $boxes[$i])) - self::boxArea($groupBoxes[0]); + $grow1 = self::boxArea(self::boxUnion($groupBoxes[1], $boxes[$i])) - self::boxArea($groupBoxes[1]); + $g = $grow0 < $grow1 || ($grow0 == $grow1 && count($groups[0]) <= count($groups[1])) ? 0 : 1; + } + $groups[$g][] = $item; + $groupBoxes[$g] = self::boxUnion($groupBoxes[$g], $boxes[$i]); + $left--; + } + $newId = $tree['next']++; + $tree['nodes'][$id] = [$level, $groupBoxes[0], $groups[0]]; + $tree['nodes'][$newId] = [$level, $groupBoxes[1], $groups[1]]; + if($tree['parent'] !== null){ + foreach($groups[1] as $item){ + if($level === 0){ + $tree['leafOf'][$item[0]] = $newId; + } else { + $tree['parent'][$item] = $newId; + } + } + } + if(empty($path)){ + $rootId = $tree['next']++; + $tree['nodes'][$rootId] = [$level + 1, self::boxUnion($groupBoxes[0], $groupBoxes[1]), [$id, $newId]]; + $tree['root'] = $rootId; + if($tree['parent'] !== null){ + $tree['parent'][$id] = $rootId; + $tree['parent'][$newId] = $rootId; + } + return; + } + $parentId = array_pop($path); + $tree['nodes'][$parentId][2][] = $newId; + if($tree['parent'] !== null){ + $tree['parent'][$newId] = $parentId; + } + if(count($tree['nodes'][$parentId][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $parentId, $path); } - return $best; } /** - * Find records within a distance from a point (distance to the nearest point of the geometry) - * @param string $dbname Database name - * @param string $field GeoJSON field name - * @param float $lon Center longitude - * @param float $lat Center latitude - * @param float $distanceMeters Distance in meters - * @param array $options ['includeDistance' => false] - * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) + * Remove a key; emptied nodes go, boxes shrink up the tree, a root with one child is replaced by it + * @return bool True when the key was indexed */ - public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $distanceMeters, $options){ - return $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $distanceMeters, $options); - }); - } - - private function withinDistanceUnlocked(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ - $includeDistance = $options['includeDistance'] ?? false; - if ($distanceMeters < 0) { - return []; + private function rtreeRemove(array &$tree, int $key): bool { + $this->rtreeMaps($tree); + if(!isset($tree['leafOf'][$key])){ + return false; } - $results = []; - foreach ($this->spatialCandidates($dbname, $field, $this->circleToBBoxes($lon, $lat, $distanceMeters)) as $record) { - $geometry = $record[$field] ?? null; - if (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']) { - continue; + $id = $tree['leafOf'][$key]; + unset($tree['leafOf'][$key]); + $entries = $tree['nodes'][$id][2]; + foreach($entries as $i => $entry){ + if($entry[0] === $key){ + unset($entries[$i]); + break; } - $distance = $this->distanceToGeometry($lon, $lat, $geometry); - if ($distance <= $distanceMeters) { - $record['_distance'] = round($distance, 2); - $results[] = $record; + } + $tree['nodes'][$id][2] = array_values($entries); + $tree['cnt']--; + // Remove emptied nodes bottom-up + while($id !== $tree['root'] && empty($tree['nodes'][$id][2])){ + $parentId = $tree['parent'][$id]; + $children = $tree['nodes'][$parentId][2]; + $at = array_search($id, $children, true); + if($at !== false){ + unset($children[$at]); + } + $tree['nodes'][$parentId][2] = array_values($children); + unset($tree['nodes'][$id], $tree['parent'][$id]); + $id = $parentId; + } + // Recompute boxes up the tree while they change + while(true){ + [$level, $old, $items] = $tree['nodes'][$id]; + $box = null; + foreach($items as $item){ + $itemBox = $this->rtreeBoxOf($tree, $level, $item); + if($itemBox !== null){ + $box = self::boxUnion($box, $itemBox); + } + } + if($box === $old){ + break; + } + $tree['nodes'][$id][1] = $box; + if(!isset($tree['parent'][$id])){ + break; } + $id = $tree['parent'][$id]; } - usort($results, function ($a, $b) { - return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; - }); - if (!$includeDistance) { - foreach ($results as &$r) { - unset($r['_distance']); + // Collapse the root + while(true){ + [$level, , $items] = $tree['nodes'][$tree['root']]; + if($level === 0){ + break; } - unset($r); + if(empty($items)){ + $tree['nodes'][$tree['root']] = [0, null, []]; + break; + } + if(count($items) > 1){ + break; + } + $child = $items[0]; + unset($tree['nodes'][$tree['root']]); + $tree['root'] = $child; + unset($tree['parent'][$child]); } - $this->clearDistanceCache(); - return $results; + return true; } /** - * Find records within a bounding box (geometries whose bounding box overlaps it). - * minLon > maxLon means the box crosses the antimeridian. - * @return array Matching records + * Keys whose boxes overlap one of the (integer) boxes */ - public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $minLon, $minLat, $maxLon, $maxLat){ - return $this->withinBBoxUnlocked($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); - }); - } - - private function withinBBoxUnlocked(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ - if ($minLat > $maxLat) { - [$minLat, $maxLat] = [$maxLat, $minLat]; - } - $boxes = $minLon <= $maxLon - ? [[$minLon, $minLat, $maxLon, $maxLat]] - : [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon, $maxLat]]; - - $results = []; - foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { - $geometry = $record[$field] ?? null; - if (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']) { + private function rtreeSearch(array $tree, array $boxes): array { + $found = []; + $stack = [$tree['root']]; + while(!empty($stack)){ + $id = array_pop($stack); + [$level, $nodeBox, $items] = $tree['nodes'][$id]; + if($nodeBox === null){ continue; } - $recordMBR = $this->calculateMBR($geometry); - foreach ($boxes as $box) { - if ($this->mbrOverlaps($recordMBR, $box)) { - $results[] = $record; + $hit = false; + foreach($boxes as $box){ + if(self::boxOverlaps($nodeBox, $box)){ + $hit = true; break; } } + if(!$hit){ + continue; + } + foreach($items as $item){ + if($level > 0){ + $stack[] = $item; + continue; + } + foreach($boxes as $box){ + if($item[1] <= $box[2] && $box[0] <= $item[3] && $item[2] <= $box[3] && $box[1] <= $item[4]){ + $found[$item[0]] = $item[0]; + break; + } + } + } } - return $results; + return array_values($found); } /** - * Find K nearest records to a point - * @param string $dbname Database name - * @param string $field GeoJSON field name - * @param float $lon Center longitude - * @param float $lat Center latitude - * @param int $k Number of nearest records - * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] - * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + * Apply one data line to an R-tree: the key now holds $record (null: deleted) */ - public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $k, $options){ - return $this->nearestUnlocked($dbname, $field, $lon, $lat, $k, $options); - }); + private function rtreeApply(array &$tree, int $key, ?array $record): void { + $geometry = $record === null ? null : ($record[$tree['field']] ?? null); + $box = is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid'] ? $this->quantizedMBR($geometry) : null; + if($box === null){ + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } elseif($key > $tree['mk']){ + $tree['mk'] = $key; + } + return; + } + $this->rtreeInsert($tree, $key, $box); } /** - * True when spatial queries on the field use the R-tree (used by the query builder) + * Build a segment's R-tree from its data (covers the key index's bytes) */ - private function usesSpatialIndex(string $dbname, string $field): bool{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field){ - return $this->spatialIndexEnabled && $this->dbExists($dbname) - && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); - }); + public function buildSegmentSpatialIndex(string $dbname, string $field, ?int $shardId, array $options = []): array { + $key = $this->secondary->keyIndexWithEpoch($dbname, $shardId); + $nodeSize = isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4 ? (int)$options['nodeSize'] : $this->rtreeNodeSize; + $tree = $this->newRTree($field, $nodeSize); + $columns = [[], [], [], [], []]; // keys, then their boxes' minX, minY, maxX, maxY + foreach($this->records->eachSegmentBatch($dbname, $shardId) as $batch){ + foreach($batch as $k => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid']){ + $box = $this->quantizedMBR($geometry); + if($box !== null){ + $columns[0][] = (int)$k; + $columns[1][] = $box[0]; + $columns[2][] = $box[1]; + $columns[3][] = $box[2]; + $columns[4][] = $box[3]; + } + } + } + } + $this->rtreeBulkLoad($tree, ...$columns); + $tree['mk'] = $key === null ? -1 : \noneDB\Internal\Storage\KeyIndex::maxKey($key); + $tree['s'] = $key === null ? 0 : (int)$key['s']; + $tree['e'] = $key === null ? null : $key['e']; + return $tree; } - private function nearestUnlocked(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ - $maxDistance = isset($options['maxDistance']) ? (float)$options['maxDistance'] : null; - $includeDistance = $options['includeDistance'] ?? true; - if ($k <= 0 || !$this->dbExists($dbname)) { - return []; + /** + * A segment's R-tree, current (the data lines after the bytes it covers are replayed); null when + * none fits the segment (see ensureSegmentSpatialIndex) + */ + private function readSpatialIndex(string $dbname, string $field, ?int $shardId = null): ?array { + $key = $this->secondary->currentKeyIndex($dbname, $shardId); + if($key === null){ + return null; } - - $indexed = $this->spatialIndexEnabled && in_array($field, $this->getIndexManifest($dbname)['spatial'], true); - $limit = $maxDistance ?? 20040000.0; // half the earth's circumference covers everything - $results = []; - - if ($indexed && $k < $this->countFast($dbname)) { - // Expanding search: nothing outside radius r is closer than r. Results are ordered by the - // rounded distance, so only records rounded below round(r) are certainly in final order - // (a record just outside r can round to the same value and has to win a key tie). - $radius = 1000.0; - while (true) { - $radius = min($radius, $limit); - $found = $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $radius, ['includeDistance' => true]); - if ($radius >= $limit || (count($found) >= $k && $found[$k - 1]['_distance'] < round($radius, 2))) { - $results = $found; - break; - } - $radius *= 4; - } - } else { - foreach ($this->listSegments($dbname) as $segment) { - foreach ($this->readSegmentRecords($dbname, $segment) as $record) { - $geometry = $record[$field] ?? null; - if (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']) { - continue; + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->spatialIndexCache[$path])){ + $tree = $this->spatialIndexCache[$path]; + if($tree['e'] !== null && $tree['e'] === $key['e'] && $tree['s'] <= (int)$key['s']){ + $this->metrics['hits']++; + if($tree['s'] < (int)$key['s']){ + unset($this->spatialIndexCache[$path], $tree['due']); + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; } - $distance = $this->distanceToGeometry($lon, $lat, $geometry); - if ($maxDistance === null || $distance <= $maxDistance) { - $record['_distance'] = round($distance, 2); - $results[] = $record; + } elseif(isset($tree['due']) && $this->noWrites === 0 && $this->multiLock === 0){ + unset($tree['due'], $this->spatialIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; } } + return $this->spatialIndexCache[$path]; } - usort($results, function ($a, $b) { - return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; - }); + unset($this->spatialIndexCache[$path]); + } + $this->metrics['misses']++; + $file = $this->secondary->readSecondaryFile($path); + if($file === null || !$this->secondary->secondaryFits($dbname, $shardId, $file[0], 'rx', $field)){ + return null; + } + $tree = $this->treeOf($field, $file[0], $file[1]); + if($tree === null){ + return null; + } + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + $this->cacheGrew = true; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; } + return $this->spatialIndexCache[$path]; + } - $results = array_slice($results, 0, $k); - if (!$includeDistance) { - foreach ($results as &$r) { - unset($r['_distance']); - } - unset($r); + public function replaySpatialIndex(string $dbname, ?int $shardId, array &$tree, int $to): void { + foreach($this->secondary->segmentTail($dbname, $shardId, $tree['s'], $to) as [$k, $record]){ + $this->rtreeApply($tree, (int)$k, $record); } - $this->clearDistanceCache(); - return $results; + $tree['s'] = $to; } /** - * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) - * @param string $dbname Database name - * @param string $field GeoJSON field name - * @param array $polygon GeoJSON Polygon or MultiPolygon geometry - * @return array Matching records + * Write an R-tree checkpoint */ - public function withinPolygon(string $dbname, string $field, array $polygon): array{ - $dbname = $this->normalizeDbName($dbname); - return $this->withReadLock($dbname, function() use ($dbname, $field, $polygon){ - return $this->withinPolygonUnlocked($dbname, $field, $polygon); - }); + public function writeSpatialIndex(string $dbname, string $field, array &$tree, ?int $shardId = null): bool { + if($tree['e'] === null){ + return false; + } + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$tree['s']); + if($crc === null){ + return false; + } + // One line of nodes (a leaf: its number of entries), then the entries of the leaves in that order + $nodes = []; + $entries = ''; + foreach($tree['nodes'] as $id => $node){ + if($node[0] === 0){ + $entries .= pack('P*', ...array_merge(...$node[2])); + $node[2] = count($node[2]); + } + $nodes[(string)$id] = $node; + } + $body = $this->io->encodeJson($nodes); + if($body === false){ + return false; + } + $body .= "\n" . $entries; + $rootBox = $tree['nodes'][$tree['root']][1]; + $this->secondary->writeSecondaryFile($dbname, $path, ['kind' => 'rx', 'v' => 1, 'field' => $field, 'q' => self::RTREE_SCALE, 'ns' => $tree['ns'], + 's' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'mk' => $tree['mk'], 'cnt' => $tree['cnt'], + 'root' => $tree['root'], 'next' => $tree['next'], 'mbr' => $rootBox, 'bc' => crc32($body)], $body); + $tree['ck'] = (int)$tree['s']; + $tree['bytes'] = strlen($body); + $this->spatialIndexCache[$path] = $tree; + $this->cacheGrew = true; + return true; } - private function withinPolygonUnlocked(string $dbname, string $field, array $polygon): array{ - $validation = $this->validateGeoJSON($polygon); - if (!$validation['valid'] || !in_array($validation['type'], ['Polygon', 'MultiPolygon'], true)) { - return []; + /** + * The tree of an .rx file, or null when it holds none (STORAGE_FORMAT.md §6.5): the crc of the body doesn't fit, or a + * node isn't [level, box, items], where a box is four integers (null only without items), the items of a leaf a + * number of entries the body holds, the items above it nodes of a lower level, named once. So a walk meets every + * node it is sent to and ends. An entry is five integers by its format: no check per entry + */ + public function treeOf(string $field, array $header, string $body): ?array { + [$root, $next, $size] = [$header['root'] ?? null, $header['next'] ?? null, $header['ns'] ?? null]; + if(($header['v'] ?? null) !== 1 || ($header['bc'] ?? null) !== crc32($body) || !is_int($root) || !is_int($next) || !is_int($size) || $size < 4 + || !is_int($header['mk'] ?? null) || !is_int($header['cnt'] ?? null)){ + return null; } - $polygons = $validation['type'] === 'Polygon' ? [$polygon['coordinates']] : $polygon['coordinates']; - $boxes = []; - foreach ($polygons as $coords) { - $boxes[] = $this->mbrFromPoints($coords[0]); + $end = (int)strpos($body, "\n"); + $nodes = json_decode(substr($body, 0, $end), true); + if(!is_array($nodes) || !isset($nodes[$root])){ + return null; } - - $results = []; - foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { - $geometry = $record[$field] ?? null; - if (!is_array($geometry) || !$this->validateGeoJSON($geometry)['valid']) { + $entries = 0; + $named = []; + foreach($nodes as $id => $node){ + $level = $node[0] ?? null; + $box = $node[1] ?? null; + $items = $node[2] ?? null; + if($id >= $next || !is_int($level) || $level < 0 || ($box === null ? !empty($items) : !self::isBox($box))){ + return null; + } + if($level === 0){ + if(!is_int($items) || $items < 0){ + return null; + } + $entries += $items; continue; } - foreach ($polygons as $coords) { - if ($this->geometryWithinPolygon($geometry, $coords)) { - $results[] = $record; - break; + if(!is_array($items)){ + return null; + } + foreach($items as $item){ + if(!is_int($item) || ($nodes[$item][0] ?? $level) >= $level || isset($named[$item])){ + return null; } + $named[$item] = true; } } - return $results; + if($entries * 40 !== strlen($body) - $end - 1){ + return null; + } + $at = $end + 1; + foreach($nodes as $id => $node){ + if($node[0] === 0){ + $nodes[$id][2] = $node[2] === 0 ? [] : array_chunk(unpack('P' . 5 * $node[2], $body, $at), 5); + $at += 40 * $node[2]; + } + } + $tree = $this->newRTree($field, $size); + $tree['nodes'] = $nodes; + $tree['root'] = $root; + $tree['next'] = $next; + $tree['mk'] = $header['mk']; + $tree['cnt'] = $header['cnt']; + $tree['s'] = $header['s']; + $tree['ck'] = $header['s']; + $tree['e'] = $header['e']; + $tree['bytes'] = strlen($body); + return $tree; } /** - * Check if a geometry lies entirely within a polygon: every vertex inside (not in a hole) and no - * edge crossing the polygon's boundary + * Read the global spatial index: ['le', 'sh' => [shard => ['s','e','h','b' => box or null]]] */ - private function geometryWithinPolygon(array $geometry, array $polygonCoords): bool { - $type = $geometry['type'] ?? ''; - $coords = $geometry['coordinates'] ?? []; - - switch ($type) { - case 'Point': - return $this->pointInPolygon((float)$coords[0], (float)$coords[1], $polygonCoords); - case 'MultiPoint': - foreach ($coords as $point) { - if (!$this->pointInPolygon((float)$point[0], (float)$point[1], $polygonCoords)) { - return false; - } + public function readGlobalSpatialIndex(string $dbname, string $field): ?array { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(array_key_exists($path, $this->globalSpatialIndexCache)){ + $this->metrics['hits']++; + return $this->globalSpatialIndexCache[$path]; + } + $this->metrics['misses']++; + $header = $this->secondary->readSecondaryHeader($path); + $global = null; + if(is_array($header) && ($header['kind'] ?? null) === 'grx' && ($header['v'] ?? null) === 1 && ($header['field'] ?? null) === $field && is_array($header['sh'] ?? null)){ + $global = ['le' => $header['le'] ?? null, 'sh' => []]; + foreach($header['sh'] as $shardId => $entry){ + if(\noneDB\Internal\Index\SecondaryFiles::isEntry($entry) && array_key_exists('b', $entry) && ($entry['b'] === null || self::isBox($entry['b']))){ + $global['sh'][(int)$shardId] = $entry; } - return true; - case 'LineString': - return $this->pathWithinPolygon($coords, $polygonCoords); - case 'MultiLineString': - foreach ($coords as $line) { - if (!$this->pathWithinPolygon($line, $polygonCoords)) { - return false; - } - } - return true; - case 'Polygon': - return $this->pathWithinPolygon($coords[0], $polygonCoords); - case 'MultiPolygon': - foreach ($coords as $poly) { - if (!$this->pathWithinPolygon($poly[0], $polygonCoords)) { - return false; - } + } + } + $this->globalSpatialIndexCache[$path] = $global; + $this->cacheGrew = true; + return $global; + } + + public function writeGlobalSpatialIndex(string $dbname, string $field, array $global): void { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $global['le'] = $lock === null ? null : $this->locks->registryEpoch($this->locks->getLockPath($dbname)); + $sh = []; + foreach($global['sh'] as $shardId => $entry){ + $sh[(string)$shardId] = $entry; + } + $this->secondary->writeSecondaryFile($dbname, $path, ['kind' => 'grx', 'v' => 1, 'field' => $field, 'q' => self::RTREE_SCALE, + 'le' => $global['le'], 'sh' => empty($sh) ? new \stdClass() : $sh], ''); + $this->globalSpatialIndexCache[$path] = $global; + unset($this->lockRegistry[$this->locks->getLockPath($dbname)]['globals'][$path]); + } + + /** + * Record a shard's box (at the bytes its R-tree covers) in the global spatial index + */ + public function updateGlobalSpatialShard(string $dbname, string $field, int $shardId, array $tree): void { + $entry = $this->globalSpatialEntry($dbname, $shardId, $tree); + if($entry === null){ + return; + } + $global = $this->secondary->freshGlobalEntries($dbname, $this->readGlobalSpatialIndex($dbname, $field) ?? ['le' => null, 'sh' => []]); + $lockPath = $this->locks->getLockPath($dbname); + if(($global['sh'][$shardId] ?? null) === $entry && $global['le'] === (isset($this->lockRegistry[$lockPath]) ? $this->locks->registryEpoch($lockPath) : null)){ + return; + } + $global['sh'][$shardId] = $entry; + $this->fieldIndex->deferGlobalWrite($dbname, 'grx', $field, $global); + } + + /** + * A shard's entry in the global spatial index: its R-tree's root box at the bytes it covers (null: + * none can be recorded) + */ + private function globalSpatialEntry(string $dbname, int $shardId, array $tree): ?array { + if($tree['e'] === null){ + return null; + } + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$tree['s']); + return $crc === null ? null : ['s' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'b' => $tree['nodes'][$tree['root']][1]]; + } + + /** + * A segment's R-tree: the current one, or built from the data (exclusive lock) + */ + public function ensureSegmentSpatialIndex(string $dbname, string $field, ?int $shardId): array { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if($tree !== null){ + return $tree; + } + if($this->noWrites > 0){ + throw new \RuntimeException("noneDB: no spatial index to read in a read-only check"); + } + return $this->locks->withLock($dbname, true, function () use ($dbname, $field, $shardId) { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if ($tree === null) { + $nodeSize = $this->fieldIndex->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null; + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, $nodeSize ? ['nodeSize' => $nodeSize] : []); + $this->writeSpatialIndex($dbname, $field, $tree, $shardId); + if ($shardId !== null) { + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); } - return true; - case 'GeometryCollection': - foreach ($geometry['geometries'] ?? [] as $g) { - if (!$this->geometryWithinPolygon($g, $polygonCoords)) { - return false; - } + } + return $tree; + }, 'buildSpatialIndex'); + } + + /** + * Keys of a segment whose geometry box may overlap the query boxes (float [minLon, minLat, maxLon, + * maxLat], or a list of them); every caller checks the exact geometry + */ + public function rtreeSearchMBR(string $dbname, string $field, array $searchMBR, ?int $shardId = null): array { + $tree = $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); + $boxes = is_array($searchMBR[0] ?? null) ? $searchMBR : [$searchMBR]; + return $this->rtreeSearch($tree, array_map([$this, 'quantizeBox'], $boxes)); + } + + /** + * Rebuild a spatial index for the whole database (all segments + global shard boxes) + */ + public function rebuildSpatialIndexForDb($dbname, $field, $nodeSize = null){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $nodeSize = $nodeSize ?? $manifest['spatialOptions'][$field]['nodeSize'] ?? null; + $options = $nodeSize ? ['nodeSize' => $nodeSize] : []; + $segments = $this->segments->listSegments($dbname); + if(!in_array(null, $segments, true)){ + $segments[] = null; // leftovers of the non-sharded database + } + foreach($segments as $segment){ + $path = $this->getSpatialIndexPath($dbname, $field, $segment); + $this->io->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + foreach([$this->getGlobalSpatialIndexPath($dbname, $field)] as $path){ + $this->io->fsUnlink($path); + unset($this->globalSpatialIndexCache[$path]); + } + $indexed = 0; + $sharded = $this->segments->isSharded($dbname); + $global = ['le' => null, 'sh' => []]; // written once at the end (once per shard was O(shards²)) + foreach ($this->segments->listSegments($dbname) as $segment) { + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $segment, $options); + $this->writeSpatialIndex($dbname, $field, $tree, $segment); + $indexed += $tree['cnt']; + if ($sharded && ($entry = $this->globalSpatialEntry($dbname, $segment, $tree)) !== null) { + $global['sh'][$segment] = $entry; + } + unset($tree); + $this->cache->evictBetweenSegments(); + } + if ($sharded) { + ksort($global['sh']); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + return $indexed; + } + + /** + * Delete the spatial index files of one shard + */ + public function removeSpatialShardFiles($dbname, $shardId){ + foreach ($this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'spatial/s' . (int)$shardId . '.', '/\.rx$/') as $file) { + $this->io->fsUnlink($file); + unset($this->spatialIndexCache[$file]); + } + foreach ($this->fieldIndex->getIndexManifest($dbname)['spatial'] as $field) { + $global = $this->readGlobalSpatialIndex($dbname, $field); + if ($global !== null && isset($global['sh'][(int)$shardId])) { + unset($global['sh'][(int)$shardId]); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + } + } + + /** + * True when a shard can be skipped for the query boxes: its global entry fits the shard, its box + * misses every box, and the shard's newer lines hold no geometry in them + */ + public function globalSpatialSkips(string $dbname, string $field, int $shardId, ?array $global, array $boxes): bool { + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || !$this->fieldIndex->globalEntryFits($dbname, $shardId, $entry, $global['le'])){ + return false; + } + $quantized = array_map([$this, 'quantizeBox'], $boxes); + foreach($quantized as $box){ + if(self::boxOverlaps($entry['b'], $box)){ + return false; + } + } + return !$this->fieldIndex->shardTailMayMatch($dbname, $shardId, $entry, function($record) use ($field, $quantized){ + $geometry = $record[$field] ?? null; + if(!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid']){ + return false; + } + $box = $this->quantizedMBR($geometry); + foreach($quantized as $q){ + if(self::boxOverlaps($box, $q)){ + return true; } - return true; + } + return false; + }); + } + + /** + * Check if a spatial index exists + * @param string $dbname Database name + * @param string $field Field name + * @param int|null $shardId Shard ID + * @return bool + */ + public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $shardId){ + return $this->hasSpatialIndexUnlocked($dbname, $field, $shardId); + }); + } + + private function hasSpatialIndexUnlocked(string $dbname, string $field, ?int $shardId = null): bool{ + if (!$this->segments->dbExists($dbname) || !in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true)) { + return false; } - return false; + return $shardId === null || in_array($shardId, $this->segments->listSegments($dbname), true); + } + // ========================================== + // SPATIAL INDEX - R-TREE ENGINE + // ========================================== + // One R-tree per segment and indexed field: spatial/{seg}.{token}.rx (see SecondaryFiles), a + // checkpoint like the field indexes: + // {"kind":"rx","v":1,"field":"loc","q":10000000,"ns":32,"s":…,"e":…,"h":…,"mk":…,"cnt":…,"root":3,"next":9} + // {"0":[0,[minX,minY,maxX,maxY],[[key,minX,minY,maxX,maxY],…]],"3":[1,[…],[0,2]],…} + // A node is [level, box, payload]: level 0 holds entries [key, box], a higher level child node ids. + // Boxes are integers: degrees * 1e7 rounded outwards (1e-7° is about 1 cm), so they don't depend on + // serialize_precision, and every query re-checks the exact geometry. Built with sort-tile-recursive + // bulk loading; changes after the checkpoint are replayed from the data when the index is used. + // Sharded databases keep spatial/{token}.grx: each shard's box (null = empty) as of the bytes its + // entry covers; a shard is skipped only when its entry fits, its box misses the query and the shard's + // newer lines hold no geometry in it. + + public const RTREE_SCALE = 10000000; +} +} +namespace noneDB\Internal\Index\Spatial { + +use function array_diff, array_key_exists, array_merge, array_values, in_array, is_int; + +/** + * @internal + */ +final class SpatialIndexApi { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $globalSpatialIndexCache; + private $spatialIndexCache; + private $spatialIndexEnabled; + private $buffer; + private $db; + private $fieldIndex; + private $io; + private $layout; + private $locks; + private $rtree; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->db, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->db = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL INDEX - PUBLIC API + // ========================================== + + /** + * Create a spatial index on a GeoJSON field + * @param string $dbname Database name + * @param string $field Field name containing GeoJSON + * @param array $options Options ['nodeSize' => 32] + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + */ + public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['nodeSize'], 'option', ' of createSpatialIndex()'); + if(array_key_exists('nodeSize', $options) && (!is_int($options['nodeSize']) || $options['nodeSize'] < 4)){ + throw new \InvalidArgumentException("noneDB: 'nodeSize' must be an integer of at least 4"); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field, $options){ + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + }, 'createSpatialIndex'); + } + + private function createSpatialIndexUnlocked(string $dbname, string $field, array $options = []): array{ + if (!$this->spatialIndexEnabled) { + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + if ($field === '' || $field === 'key') { + return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + } + $this->buffer->flushPendingBuffers($dbname); + if (!$this->segments->dbExists($dbname)) { + if (!$this->autoCreateDB) { + return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + } + $this->db->createDBUnlocked($dbname); + } + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if (in_array($field, $manifest['spatial'], true)) { + return ['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field '$field'"]; + } + // Build first, list it last: an index that fails half-way is never listed + $nodeSize = $options['nodeSize'] ?? null; + $indexed = $this->rtree->rebuildSpatialIndexForDb($dbname, $field, $nodeSize); + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['spatial'][] = $field; + if ($nodeSize !== null) { + $manifest['spatialOptions'][$field] = ['nodeSize' => $nodeSize]; + } + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + return ['success' => true, 'indexed' => $indexed, 'error' => null]; + } + + /** + * Drop a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => ?string] + */ + public function dropSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropSpatialIndexUnlocked($dbname, $field); + }, 'dropSpatialIndex'); + } + + private function dropSpatialIndexUnlocked(string $dbname, string $field): array{ + foreach (array_merge([null], $this->segments->isSharded($dbname) ? $this->segments->listSegments($dbname) : []) as $segment) { + foreach ([$this->rtree->getSpatialIndexPath($dbname, $field, $segment)] as $path) { + $this->io->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + } + foreach ([$this->rtree->getGlobalSpatialIndexPath($dbname, $field)] as $globalPath) { + $this->io->fsUnlink($globalPath); + unset($this->globalSpatialIndexCache[$globalPath]); + } + + if ($this->segments->dbExists($dbname)) { + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['spatial'] = array_values(array_diff($manifest['spatial'], [$field])); + unset($manifest['spatialOptions'][$field]); + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + } + return ['success' => true, 'error' => null]; + } + + /** + * Get list of spatial indexes for a database + * @param string $dbname Database name + * @return array List of indexed field names + */ + public function getSpatialIndexes(string $dbname): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getSpatialIndexesUnlocked($dbname); + }); + } + + private function getSpatialIndexesUnlocked(string $dbname): array{ + return $this->segments->dbExists($dbname) ? $this->fieldIndex->getIndexManifest($dbname)['spatial'] : []; + } + + /** + * Rebuild a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createSpatialIndex + */ + public function rebuildSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildSpatialIndexUnlocked($dbname, $field); + }, 'rebuildSpatialIndex'); + } + + private function rebuildSpatialIndexUnlocked(string $dbname, string $field): array{ + if (!$this->spatialIndexEnabled) { + // Checked before dropping anything: a drop must not succeed where the create then fails + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + $options = []; + $nodeSize = $this->segments->dbExists($dbname) ? ($this->fieldIndex->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null) : null; + if ($nodeSize) { + $options['nodeSize'] = $nodeSize; + } + $this->dropSpatialIndexUnlocked($dbname, $field); + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + } +} +} +namespace noneDB\Internal\Maintenance { + +use function array_keys, array_values, asort, basename, bin2hex, count, fclose, fgets, fopen, is_array, json_decode, ksort, + microtime, random_bytes, strlen, substr, time; +use const SORT_NUMERIC; + +/** + * @internal + */ +final class Compaction { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $indexCache; + private $autoCompactThreshold; + private $verifiedIndexes; + private $metrics; + private $checkpoints; + private $io; + private $locks; + private $recovery; + private $recoveryLog; + private $scope; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->indexCache = &$this->k->indexCache; + $this->autoCompactThreshold = &$this->k->autoCompactThreshold; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->metrics = &$this->rt->metrics; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * After a write that left old lines in a segment ($index: the segment's index as the write left it). + * Compaction replaces the data file, so it runs after the write committed (see the rollback scope); whether it + * is due is decided here, with the index in hand: a write over many segments doesn't load each evicted index + * again only to find nothing due + */ + public function deferCompaction($dbname, $shardId, array $index){ + if(!$this->compactionDue($index)){ + return; + } + $this->scope->runIndependent($dbname, 'compact:' . ($shardId === null ? 'main' : $shardId), function() use ($dbname, $shardId){ + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index !== null && $this->compactionDue($index)){ + $this->compactJsonl($dbname, $shardId); + } + }); + } + + /** True once a segment's garbage (old versions, tombstones) exceeds the threshold share of its lines */ + private function compactionDue(array $index){ + $garbage = (int)$index['d']; + return $garbage > 0 && $garbage > (\noneDB\Internal\Storage\KeyIndex::count($index) + $garbage) * $this->autoCompactThreshold; + } + + /** + * Compact a segment: rewrite live records (keys and order preserved), drop old versions and + * tombstones. Crash-safe: the data file is replaced first; if the index rename doesn't happen, + * the shorter data file is detected and the index rebuilt from it. + * @param string $dbname + * @param int|null $shardId + * @return array ['compacted' => int, 'freed' => int] + */ + public function compactJsonl($dbname, $shardId = null){ + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId){ + $started = microtime(true); + $this->scope->refuseUncommittedFile($this->segments->segDataPath($dbname, $shardId), 'compaction'); + $this->recovery->removeStaleTemp($this->segments->segDataPath($dbname, $shardId)); // {seg}.jsonl[.kix].tmp{8 hex} + $plan = null; + for($attempt = 0; $plan === null; $attempt++){ + if($attempt > 1){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + // A record the index points at must verify; if not, rebuild the index once and plan again + $index = $attempt === 0 ? $this->checkpoints->writableSegmentIndex($dbname, $shardId) : $this->checkpoints->repairSegmentIndex($dbname, $shardId, true); + if($index === null){ + return ['compacted' => 0, 'freed' => 0]; + } + $plan = $this->planCompaction($dbname, $shardId, $index); + } + $path = $this->segments->segDataPath($dbname, $shardId); + $live = $plan['live']; + unset($plan['live']); // held once: each record's bytes are released once they are in the buffer + ksort($live, SORT_NUMERIC); + + $newIndex = $this->checkpoints->newJsonlIndex(); + $newIndex['created'] = $index['created'] ?? time(); + $newIndex['n'] = $index['n']; + $buffer = ''; + $offset = 0; + $maxKey = -1; + // A fresh index of distinct keys: its arrays filled directly (KeyIndex::set() per key checks a body) + foreach($live as $key => &$json){ + $newIndex['p'][$key] = $offset; + $newIndex['l'][$key] = strlen($json); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + $maxKey = $key; + $json = null; // in the buffer now: the record's string is freed, the peak stays near one copy + } + unset($json); + $newIndex['live'] = count($live); + $live = null; + // Keep the highest key ever used: an index rebuilt from this file must never hand it out again + $highest = ($shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname)) + (int)$index['n'] - 1; + if($maxKey < $highest){ + $buffer .= '{"__nonedb_deleted":' . $highest . "}\n"; + $newIndex['t'] = 1; + } + // Lines that don't decode are dropped from the data file: their bytes go to quarantine first + $events = []; + foreach($plan['bad'] as $region){ + $quarantine = $this->recovery->quarantineBytes($dbname, $shardId, $path, $region[0], $region[0] + $region[1]); + $events[] = ['action' => 'drop-line', 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]; + } + + // The checkpoint goes first: it must never be taken for the new data file + $this->locks->bumpEpoch($dbname); + $this->io->fsUnlink(\noneDB\Internal\Storage\Checkpoints::ckptPath($path)); + try { + $replaced = $this->io->writeFileAtomic($path, $buffer, true); + } catch(\RuntimeException $e){ + throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data: " . $e->getMessage(), $e->getCode(), $e); + } + if(!$replaced){ + throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data"); + } + $newIndex['s'] = strlen($buffer); + $newIndex['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = $newIndex; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = true; + $this->checkpoints->writeCheckpoint($dbname, $shardId, $newIndex); + foreach($events as $event){ + $this->recoveryLog->logRecoveryEvent($dbname, $event); + } + $this->metrics['compactions']++; + $this->metrics['compactionSeconds'] += microtime(true) - $started; + return ['compacted' => $newIndex['live'], 'freed' => (int)$index['d']]; + }, 'compact'); + } + + /** + * Read a segment line by line for compaction: the JSON of every live record (verified against its + * key) and the byte ranges of lines that are neither records nor tombstones + * @return array|null ['live' => [key => json], 'bad' => [[offset, length], ...]]; null if the index + * doesn't match the data + */ + private function planCompaction($dbname, $shardId, array $index){ + // Every live key's offset and length (a decoded index: its arrays as they are; no array per record) + [$offsets, $lengths] = \noneDB\Internal\Storage\KeyIndex::locate($index, null); + asort($offsets, SORT_NUMERIC); // offsets are distinct: file order (a sort without a callback) + // File order as two lists, no array per record: where each live line starts, and its key + $starts = array_values($offsets); + $keys = array_keys($offsets); + unset($offsets); + $count = count($keys); + // Keys of this segment: a line with another shard's key is damaged, never a superseded record + $size = $shardId === null ? null : $this->segments->shardSizeOf($dbname); + $base = $shardId === null ? 0 : $shardId * $size; + $limit = $shardId === null ? null : $base + $size; + $fp = @fopen($this->segments->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return $count === 0 ? ['live' => [], 'bad' => []] : null; + } + $live = []; + $bad = []; + $next = 0; + $pos = 0; + try { + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + $end = $pos + $length; + $owned = []; // positions in the lists of the records this line holds + while($next < $count && $starts[$next] < $end){ + $owned[] = $next++; + } + if(substr($line, -1) !== "\n"){ + if(!empty($owned)){ + return null; + } + $bad[] = [$pos, $length]; // unterminated tail + break; + } + $json = substr($line, 0, -1); + $whole = count($owned) === 1 && (int)$starts[$owned[0]] === $pos && (int)$lengths[$keys[$owned[0]]] === strlen($json); + foreach($owned as $i){ + $key = (int)$keys[$i]; + $sub = $whole ? $json : substr($line, (int)$starts[$i] - $pos, (int)$lengths[$key]); + $record = json_decode($sub, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== $key){ + return null; + } + $live[$key] = $sub; + } + if(!$whole){ + if(!empty($owned)){ + $bad[] = [$pos, $length]; // a salvaged record inside other bytes: keep the line + } elseif($this->checkpoints->classifyLine($json, $base, $limit) === null){ + $bad[] = [$pos, $length]; // neither a record nor a tombstone of this segment + } + } + $pos = $end; + } + } finally { + fclose($fp); + } + return $next < $count ? null : ['live' => $live, 'bad' => $bad]; + } +} +} +namespace noneDB\Internal\Maintenance { + +use function array_column, array_filter, array_flip, array_is_list, array_key_exists, array_keys, array_map, + array_reverse, array_sum, array_unique, basename, bin2hex, chmod, count, ctype_digit, dirname, fclose, fflush, + fgets, file_exists, file_get_contents, filesize, fopen, fread, fwrite, hash, hash_final, hash_init, hash_update, + hash_update_stream, implode, in_array, intdiv, is_array, is_bool, is_dir, is_file, is_int, is_link, is_string, + json_decode, json_encode, max, min, mkdir, preg_match, random_bytes, realpath, rmdir, rtrim, scandir, sort, + str_replace, strcmp, stream_get_contents, strlen, strncmp, substr, time, trim, unlink, usort; +use const DIRECTORY_SEPARATOR, LOCK_SH, SORT_STRING; + +/** + * @internal + */ +final class BackupRestore { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + // The files of a backup (§10.1): those of a generation by §5.1, the recovery files by name + private const SEGMENT = '(?:main|s(?:0|[1-9][0-9]*))'; + private const TOKEN = '(?:[a-z0-9_]{1,40}|[a-z0-9_]{0,24}-[0-9a-f]{12})'; + private const GENERATION_FILE = '#^(?:metadata|shards\.json|indexes\.json|segments/' . self::SEGMENT . '\.(?:jsonl|kix)' + . '|indexes/' . self::SEGMENT . '\.' . self::TOKEN . '\.fx|indexes/' . self::TOKEN . '\.gfx' + . '|spatial/' . self::SEGMENT . '\.' . self::TOKEN . '\.rx|spatial/' . self::TOKEN . '\.grx' + . '|buffer/pending\.jsonl|buffer/flushing\.(?:0|[1-9][0-9]*)\.jsonl)$#'; + private const RECOVERY_FILE = '#^recovery/(?!\.\.?$)[A-Za-z0-9._-]+$#'; + private const COMPLETE_LINES = '#^generations/[0-9]+/(?:segments|buffer)/[^/]+\.jsonl$#'; + private const CHUNK = 1048576; + + private $dbDir; + private $noWrites; + private $buffer; + private $cache; + private $checkpoints; + private $integrity; + private $io; + private $layout; + private $locks; + private $recovery; + private $recoveryLog; + private $scope; + private $shardInfo; + private $store; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->integrity, $this->io, $this->layout, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->shardInfo, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->integrity = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->shardInfo = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ---------------------------------------------------------------- backup (§10.2) + + /** + * Copy a database, as it is at one moment, into $dir (STORAGE_FORMAT.md §10.2) + * @return array ['database', 'dir', 'databaseId', 'generation', 'nextKey', 'records', 'bufferedRecords', 'files', 'bytes'] + * @throws \InvalidArgumentException $dir inside dbDir, or not an empty directory + */ + public function backup($dbname, $dir){ + $dbname = $this->layout->normalizeDbName($dbname); + $target = $this->backupTarget($dir); + if(!$this->store->openStore(false)){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist"); + } + // STORE.LOCK, then the database's LOCK, both shared until the copy is complete: one moment of the + // database and of the store record + return $this->store->withStoreLock(LOCK_SH, function() use ($dbname, $target){ + $this->store->checkMaintenanceJournal(); + $store = $this->store->loadStoreRecord(); + if($store === null || !in_array($this->store->keyCheck(), $store['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + return $this->locks->withLock($dbname, false, function() use ($dbname, $target, $store){ + $this->scope->refuseInUncommittedWrite($dbname, 'backup'); + if(!$this->layout->dbPresent($dbname)){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist"); + } + $this->noWrites++; // nothing of the database is written: no flush, no checkpoint + try { + return $this->copyDatabase($dbname, $target, $store); + } finally { + $this->noWrites--; + } + }); + }); + } + + /** + * The backup directory resolved (§10.2 step 1): outside dbDir once links are resolved, absent or empty + * @throws \InvalidArgumentException + */ + private function backupTarget($dir){ + if(!is_string($dir) || trim($dir) === ''){ + throw new \InvalidArgumentException('noneDB: backup() needs a directory'); + } + $resolved = \noneDB\Internal\Config\Configuration::canonicalPath($dir); + $root = rtrim($this->dbDir, '/\\'); // canonical (Configuration::resolveDbDir()) + if($resolved === $root || strncmp($resolved . DIRECTORY_SEPARATOR, $root . DIRECTORY_SEPARATOR, strlen($root) + 1) === 0){ + throw new \InvalidArgumentException("noneDB: the backup directory {$dir} must lie outside dbDir"); + } + if(file_exists($resolved) && (!is_dir($resolved) || count(scandir($resolved) ?: []) > 2)){ + throw new \InvalidArgumentException("noneDB: the backup directory {$dir} must not exist or be an empty directory"); + } + return $resolved; + } + + /** + * The copy (§10.2 steps 3-5) under both locks; what it created is removed when it fails + */ + private function copyDatabase($dbname, $target, array $store){ + $takenAt = time(); + $home = $this->layout->dbHome($dbname); + $generation = $this->layout->generationNumber($dbname); + $genDir = $this->layout->genDir($dbname); + $format = $this->layout->readFormatRecord($dbname); + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + if(!is_array($info)){ + throw new \RuntimeException("noneDB: the counters of database '{$dbname}' can't be read"); + } + $sources = ['CURRENT' => $home . 'CURRENT', 'FORMAT' => $home . 'FORMAT']; + foreach($this->filesUnder($genDir) as $file){ + if(preg_match(self::GENERATION_FILE, $file) === 1){ + $sources['generations/' . $generation . '/' . $file] = $genDir . $file; + } + } + foreach($this->filesUnder($home . 'recovery/') as $file){ + if(preg_match(self::RECOVERY_FILE, 'recovery/' . $file) === 1 && preg_match('/tmp[0-9a-f]{8}$/', $file) !== 1){ + $sources['recovery/' . $file] = $home . 'recovery/' . $file; + } + } + $buffered = 0; + foreach($sources as $path => $source){ + if(preg_match('#^generations/[0-9]+/buffer/#', $path) === 1){ + $buffered += count($this->buffer->readBufferRecords($source)); + } + } + $created = []; + try { + $this->makeBackupDirectory($target, $created); + $files = []; + foreach($sources as $path => $source){ + // A data or buffer file up to its last "\n": an unterminated tail is never a record (§6.1) + clearstatcache(true, $source); + $length = preg_match(self::COMPLETE_LINES, $path) === 1 ? $this->checkpoints->lastNewlineIn($source, 0, (int)@filesize($source)) + 1 + : (int)@filesize($source); + $files[$path] = $this->writeBackupFile($target . '/' . $path, $this->chunks($source, $length, null, ''), $created); + } + $files['store.json'] = $this->writeBackupFile($target . '/store.json', [\noneDB\Internal\Storage\StoreRecord::sealRecord($store)], $created); + $paths = array_keys($files); + sort($paths, SORT_STRING); + $entries = []; + foreach($paths as $path){ + [$size, $sha256] = $files[$path]; + if(!$this->fileHolds($target . '/' . $path, $size, $sha256, '')){ + throw new \RuntimeException("noneDB: the backup copy of {$path} in {$target} does not read back as it was written"); + } + $entries[] = ['path' => $path, 'size' => $size, 'sha256' => $sha256]; + } + foreach(array_unique(array_map('dirname', array_map(function($path) use ($target){ return $target . '/' . $path; }, $paths))) as $directory){ + $this->sync->requireSync(null, $directory, 'dir'); + } + $manifest = ['magic' => 'NONEDB4', 'kind' => 'backup', 'storageFormat' => \noneDB::STORAGE_FORMAT, + 'databaseId' => $format['databaseId'], 'name' => $dbname, 'generation' => $generation, + 'nextKey' => (int)$info['nextKey'], 'records' => (int)$info['totalRecords'], 'bufferedRecords' => $buffered, + 'takenAt' => $takenAt, 'files' => $entries]; + $tmp = $target . '/manifest.json.tmp' . bin2hex(random_bytes(4)); + $this->writeBackupFile($tmp, [\noneDB\Internal\Storage\StoreRecord::sealRecord($manifest)], $created); + if(!$this->sync->renameWithRetry($tmp, $target . '/manifest.json')){ + throw new \RuntimeException("noneDB: cannot write the manifest of the backup in {$target}"); + } + $created[] = $target . '/manifest.json'; + $this->sync->requireSync(null, $target, 'dir'); + } catch(\Throwable $e){ + $this->removeBackupFiles($created); + throw $e; + } + return ['database' => $dbname, 'dir' => $target, 'databaseId' => $format['databaseId'], 'generation' => $generation, + 'nextKey' => $manifest['nextKey'], 'records' => $manifest['records'], 'bufferedRecords' => $buffered, + 'files' => count($entries), 'bytes' => array_sum(array_column($entries, 'size'))]; + } + + /** + * Regular files under a database directory, relative, sorted; a symbolic link is an error + * @return string[] + */ + private function filesUnder($dir){ + $files = []; + $walk = function($at, $prefix) use (&$walk, &$files){ + foreach(@scandir($at) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(is_link($at . $entry)){ + throw new \RuntimeException("noneDB: {$at}{$entry} is a symbolic link; a database holds none"); + } + if(is_dir($at . $entry)){ + $walk($at . $entry . '/', $prefix . $entry . '/'); + } elseif(is_file($at . $entry)){ + $files[] = $prefix . $entry; + } + } + }; + $walk($dir, ''); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + sort($files, SORT_STRING); + return $files; + } + + // The backup directory belongs to no database: its files are written here (0700 directories, 0600 + // files), synced in durable mode; what a failed backup created is removed + + /** + * Create a directory of a backup and its missing parents (0700), recording the ones created + */ + private function makeBackupDirectory($dir, array &$created){ + $missing = []; + for($at = $dir; !is_dir($at); $at = dirname($at)){ + $missing[] = $at; + } + foreach(array_reverse($missing) as $new){ + if(!@mkdir($new, 0700) && !is_dir($new)){ + throw new \RuntimeException("noneDB: cannot create the backup directory {$new}"); + } + $created[] = $new; + $this->sync->requireSync(null, dirname($new), 'dir'); + } + @chmod($dir, 0700); + } + + /** + * Write a new file of a backup from $chunks + * @return array [size, SHA-256] of the bytes written + */ + private function writeBackupFile($path, iterable $chunks, array &$created){ + if(!is_dir(dirname($path))){ + $this->makeBackupDirectory(dirname($path), $created); + } + $fp = @fopen($path, 'xb'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot create the backup file {$path}"); + } + $created[] = $path; + @chmod($path, 0600); + try { + $hash = hash_init('sha256'); + $size = 0; + foreach($chunks as $chunk){ + if(fwrite($fp, $chunk) !== strlen($chunk)){ + throw new \RuntimeException("noneDB: cannot write the backup file {$path}"); + } + hash_update($hash, $chunk); + $size += strlen($chunk); + } + if(!fflush($fp)){ + throw new \RuntimeException("noneDB: cannot write the backup file {$path}"); + } + $this->sync->requireSync($fp, $path, 'file'); + } finally { + fclose($fp); + } + return [$size, hash_final($hash)]; + } + + private function removeBackupFiles(array $created){ + foreach(array_reverse($created) as $path){ + if(is_dir($path) && !is_link($path)){ + @rmdir($path); + } else { + @unlink($path); + } + } + } + + // ---------------------------------------------------------------- restore (§10.3) + + /** + * Make a backup the content of a database, with one commit (STORAGE_FORMAT.md §10.3) + * @param array $options ['force' => bool] + * @return array ['database', 'generation', 'nextKey', 'records', 'bufferedRecords', 'target' => 'present'|'absent'|'damaged', 'foreign' => bool] + * @throws \RuntimeException RESTORE_REFUSED (nothing changed) + */ + public function restore($dir, $dbname, array $options){ + $dbname = $this->layout->normalizeDbName($dbname); + \noneDB\Internal\Config\Configuration::checkNames($options, ['force'], 'option', ' of restore()'); + $force = array_key_exists('force', $options) && \noneDB\Internal\Config\Configuration::optionBool('force', $options['force']); + $backup = $this->readBackup($dir); + // A refusal is decided before anything is written: first under the read lock (a missing database is + // answered without creating anything), then again under the write lock + $this->locks->withLock($dbname, false, function() use ($dbname, $backup, $force){ + $this->restoreIdentity($dbname, $backup, $force); + }, 'restore'); + return $this->locks->withLock($dbname, true, function() use ($dbname, $backup, $force){ + $this->scope->refuseInUncommittedWrite($dbname, 'restore'); + [$state, $identity, $foreign] = $this->restoreIdentity($dbname, $backup, $force); + if($state === 'present'){ + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + $nextKey = is_array($info) ? (int)$info['nextKey'] : 0; + } else { + $nextKey = $state === 'absent' ? 0 : $this->damagedNextKey($dbname); + } + $prepare = function($dir, $generation) use ($dbname, $backup, $identity, &$nextKey){ + $nextKey = $this->prepareRestored($dbname, $dir, $generation, $backup, $identity, $nextKey); + }; + if($state === 'present'){ + $generation = $this->layout->switchGeneration($dbname, $prepare); + } elseif($state === 'absent'){ + $this->layout->createDatabaseFiles($dbname, $identity, $prepare); + $generation = 1; + } else { + $generation = $this->layout->replaceDamagedGeneration($dbname, $identity, $prepare); + } + $this->cache->invalidateDbCaches($dbname); + $manifest = $backup['manifest']; + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => $foreign ? 'foreign-restore' : 'restore', 'generation' => $generation, + 'databaseId' => $manifest['databaseId'], 'takenAt' => $manifest['takenAt'], 'records' => $manifest['records']]); + return ['database' => $dbname, 'generation' => $generation, 'nextKey' => $nextKey, 'records' => $manifest['records'], + 'bufferedRecords' => $manifest['bufferedRecords'], 'target' => $state, 'foreign' => $foreign]; + }, 'restore'); + } + + /** + * A valid backup (§10.1): ['dir', 'manifest', 'format', 'metadata'] + * @throws \RuntimeException RESTORE_REFUSED + */ + private function readBackup($dir){ + $root = is_string($dir) && $dir !== '' ? realpath($dir) : false; + $manifest = $root === false ? null : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/manifest.json'), 'backup'); + if($manifest === null || !$this->validManifest($manifest)){ + throw $this->refused($dir, 'it has no valid manifest.json (an interrupted backup has none)'); + } + $generation = $manifest['generation']; + $previous = null; + foreach($manifest['files'] as $entry){ + $path = $entry['path']; + if(!$this->backupFile($path, $generation) || ($previous !== null && strcmp($previous, $path) >= 0)){ + throw $this->refused($dir, "its manifest lists {$path}, which is not a file of a backup in its place"); + } + $previous = $path; + $file = $root . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $path); + if(realpath($file) !== $file || !$this->fileHolds($file, $entry['size'], $entry['sha256'], '')){ + throw $this->refused($dir, "{$path} does not match its manifest"); + } + } + $listed = array_flip(array_column($manifest['files'], 'path')); + foreach(['FORMAT', 'CURRENT', 'store.json', "generations/{$generation}/metadata"] as $required){ + if(!isset($listed[$required])){ + throw $this->refused($dir, "it has no {$required}"); + } + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/FORMAT'), 'database'); + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/CURRENT'), 'current'); + $metadata = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents("{$root}/generations/{$generation}/metadata"), 'generation'); + if($format === null || !$this->layout->validIdentity($format, $manifest['name']) || $format['databaseId'] !== $manifest['databaseId'] + || $current === null || ($current['databaseId'] ?? null) !== $format['databaseId'] || ($current['generation'] ?? null) !== $generation + || $metadata === null || ($metadata['generation'] ?? null) !== $generation || !$this->layout->validIdentity($metadata, $manifest['name']) + || json_encode($this->layout->identityOf($metadata)) !== json_encode($this->layout->identityOf($format)) + || !in_array($metadata['layout'] ?? null, ['single', 'sharded'], true)){ + throw $this->refused($dir, 'its FORMAT, CURRENT and metadata do not agree with its manifest'); + } + return ['dir' => $root, 'manifest' => $manifest, 'format' => $format, 'metadata' => $metadata]; + } + + private function validManifest(array $manifest){ + foreach(['generation' => 1, 'nextKey' => 0, 'records' => 0, 'bufferedRecords' => 0, 'takenAt' => 0] as $member => $least){ + if(!is_int($manifest[$member] ?? null) || $manifest[$member] < $least){ + return false; + } + } + if(($manifest['storageFormat'] ?? null) !== \noneDB::STORAGE_FORMAT || !is_string($manifest['databaseId'] ?? null) + || !is_string($manifest['name'] ?? null) || !is_array($manifest['files'] ?? null) || !array_is_list($manifest['files'])){ + return false; + } + foreach($manifest['files'] as $entry){ + if(!is_array($entry) || array_keys($entry) !== ['path', 'size', 'sha256'] || !is_string($entry['path']) + || !is_int($entry['size']) || $entry['size'] < 0 || !is_string($entry['sha256']) || preg_match('/^[0-9a-f]{64}$/', $entry['sha256']) !== 1){ + return false; + } + } + return true; + } + + /** + * A path a backup of generation $generation may hold (§10.1) + */ + private function backupFile($path, $generation){ + $prefix = 'generations/' . $generation . '/'; + return in_array($path, ['FORMAT', 'CURRENT', 'store.json'], true) || preg_match(self::RECOVERY_FILE, $path) === 1 + || (strncmp($path, $prefix, strlen($prefix)) === 0 && preg_match(self::GENERATION_FILE, substr($path, strlen($prefix))) === 1); + } + + private function refused($dir, $why){ + return new \RuntimeException('noneDB: the backup in ' . (is_string($dir) ? $dir : '(no directory)') . " can't be restored: {$why}", \noneDB::RESTORE_REFUSED); + } + + /** + * The identity a restore gives the database (§10.3 step 3), decided before anything is written + * @return array [target state, identity, foreign] + * @throws \RuntimeException RESTORE_REFUSED + */ + private function restoreIdentity($dbname, array $backup, $force){ + $state = $this->layout->restoreTargetState($dbname); + $theirs = $backup['format']; + $format = $state === 'absent' ? null : $this->layout->readFormatRecord($dbname); + if($format !== null){ + $foreign = $format['databaseId'] !== $theirs['databaseId']; + $identity = $this->layout->identityOf($format); + } else { + $foreign = $theirs['name'] !== $dbname; + $identity = $this->layout->identityOf($theirs); + if($foreign){ + $identity['databaseId'] = \noneDB\Internal\Storage\StoreRecord::uuid4(); + $identity['name'] = $dbname; + $identity['createdAt'] = time(); + } + } + if($foreign && !$force){ + throw new \RuntimeException("noneDB: the backup in {$backup['dir']} is of another database ('{$theirs['name']}'); " + . "restore it with ['force' => true] to load it as database '{$dbname}'", \noneDB::RESTORE_REFUSED); + } + if($backup['metadata']['layout'] === 'sharded' && $identity['shardSize'] !== $theirs['shardSize']){ + throw new \RuntimeException("noneDB: the backup in {$backup['dir']} has shards of {$theirs['shardSize']} records, database " + . "'{$dbname}' of {$identity['shardSize']}; restore it into a database of its own", \noneDB::RESTORE_REFUSED); + } + return [$state, $identity, $foreign]; + } + + /** + * Next key of a damaged database: 1 + the highest key of any complete line in any of its segment files + * (the keys it handed out are not known otherwise, §10.3 step 4.3) + */ + private function damagedNextKey($dbname){ + $next = 0; + $root = $this->layout->dbHome($dbname) . 'generations/'; + foreach(@scandir($root) ?: [] as $generation){ + if(!ctype_digit((string)$generation)){ + continue; + } + foreach(@scandir($root . $generation . '/segments') ?: [] as $file){ + if(preg_match('/^' . self::SEGMENT . '\.jsonl$/', $file) !== 1){ + continue; + } + $fp = @fopen($root . $generation . '/segments/' . $file, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), 0, null)) !== null){ + $next = max($next, $entry[1] + 1); + } + } + if($fp !== false){ + fclose($fp); + } + } + } + return $next; + } + + /** + * PREPARE of a restore (§10.3 step 4): generation $generation in $dir as the backup's copy with its + * metadata, the next key kept, then verified with checkIntegrity() over it + * @return int the next key of the restored database + * @throws \RuntimeException RESTORE_REFUSED when the generation does not verify + */ + private function prepareRestored($dbname, $dir, $generation, array $backup, array $identity, $nextKey){ + $manifest = $backup['manifest']; + $layout = $backup['metadata']['layout']; + $prefix = 'generations/' . $manifest['generation'] . '/'; + $top = max($nextKey, $manifest['nextKey']); + // Keys the database handed out after the backup stay used: a top tombstone of the highest one + $tombstone = ''; + $tombstoned = null; + if($nextKey > $manifest['nextKey']){ + $tombstone = '{"__nonedb_deleted":' . ($nextKey - 1) . "}\n"; + $tombstoned = 'segments/' . ($layout === 'single' ? 'main' : 's' . intdiv($nextKey - 1, $identity['shardSize'])) . '.jsonl'; + } + $copied = []; + foreach($manifest['files'] as $entry){ + if(strncmp($entry['path'], $prefix, strlen($prefix)) !== 0 || $entry['path'] === $prefix . 'metadata'){ + continue; + } + $file = substr($entry['path'], strlen($prefix)); + $this->copyIntoGeneration($backup['dir'] . '/' . $entry['path'], $entry['size'], $entry['sha256'], $dir . $file, $file === $tombstoned ? $tombstone : ''); + $copied[$file] = true; + } + if($tombstoned !== null && !isset($copied[$tombstoned])){ + $this->copyIntoGeneration(null, 0, hash('sha256', ''), $dir . $tombstoned, $tombstone); // a new shard holding it + } + $this->layout->writeGenerationMetadata($dbname, $generation, $identity, $layout); + if($layout === 'sharded' && $tombstoned !== null){ + $meta = json_decode((string)@file_get_contents($dir . 'shards.json'), true); + if(!is_array($meta) || !is_array($meta['shards'] ?? null)){ + throw new \RuntimeException("noneDB: the shard list of the backup in {$backup['dir']} is unreadable", \noneDB::RESTORE_REFUSED); + } + $shard = intdiv($nextKey - 1, $identity['shardSize']); + if(!in_array($shard, array_map('intval', array_column($meta['shards'], 'id')), true)){ + $meta['shards'][] = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shard); + usort($meta['shards'], function($a, $b){ return (int)$a['id'] <=> (int)$b['id']; }); + } + $meta['nextKey'] = $nextKey; + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($dir . 'shards.json', $json . "\n", 'durable')){ + throw new \RuntimeException("noneDB: cannot write the shard list of the restored generation of database '{$dbname}'"); + } + } + [$report, $info] = $this->layout->withGenerationView($dbname, $generation, $identity, function() use ($dbname){ + return [$this->integrity->integrityReport($dbname), $this->shardInfo->getShardInfoUnlocked($dbname)]; + }); + $meta = array_filter($report['indexMismatches'], function($mismatch){ return $mismatch['index'] === 'meta'; }); + $restoredKey = is_array($info) ? (int)$info['nextKey'] : -1; + if($report['records'] !== $manifest['records'] || $report['bufferedRecords'] !== $manifest['bufferedRecords'] || !empty($meta) || $restoredKey !== $top){ + throw new \RuntimeException("noneDB: the restored generation of database '{$dbname}' does not verify: {$report['records']} records of " + . "{$manifest['records']}, {$report['bufferedRecords']} buffered of {$manifest['bufferedRecords']}, next key {$restoredKey} of {$top}" + . (empty($meta) ? '' : ', the shard list does not fit the data') . '; nothing was restored', \noneDB::RESTORE_REFUSED); + } + return $top; + } + + /** + * Write a file of the restored generation from the backup's bytes (and $suffix) with the durability of + * its kind (DURABILITY.md §3), then read it back + */ + private function copyIntoGeneration($source, $size, $sha256, $target, $suffix){ + $file = basename(dirname($target)) . '/' . basename($target); + $mode = preg_match('#(?:^|/)(?:segments|buffer)/[^/]+\.jsonl$#', $file) === 1 ? 'replace' + : (in_array(basename($target), ['shards.json', 'indexes.json'], true) ? 'durable' : 'derived'); + if(!$this->io->writeFileAtomic($target, $this->chunks($source, $size, $sha256, $suffix), $mode)){ + throw new \RuntimeException("noneDB: cannot write {$file} of the restored generation"); + } + if(!$this->fileHolds($target, $size, $sha256, $suffix)){ + throw new \RuntimeException("noneDB: {$file} of the restored generation does not read back as it was copied"); + } + } + + // ---------------------------------------------------------------- bytes + + /** + * The first $length bytes of $path in chunks (checked against $sha256 when one is given), then $suffix + */ + private function chunks($path, $length, $sha256, $suffix){ + if($path !== null){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot read {$path}"); + } + try { + $hash = hash_init('sha256'); + for($left = $length; $left > 0; $left -= strlen($chunk)){ + $chunk = (string)fread($fp, min(self::CHUNK, $left)); + if($chunk === ''){ + throw new \RuntimeException("noneDB: {$path} changed while it was copied"); + } + hash_update($hash, $chunk); + yield $chunk; + } + if($sha256 !== null && hash_final($hash) !== $sha256){ + throw new \RuntimeException("noneDB: {$path} changed while it was copied", \noneDB::RESTORE_REFUSED); + } + } finally { + fclose($fp); + } + } + if($suffix !== ''){ + yield $suffix; + } + } + + /** + * A regular file of $size bytes with SHA-256 $sha256, followed by $suffix exactly + */ + private function fileHolds($path, $size, $sha256, $suffix){ + clearstatcache(true, $path); + if(!is_file($path) || is_link($path) || filesize($path) !== $size + strlen($suffix)){ + return false; + } + $fp = @fopen($path, 'rb'); + if($fp === false){ + return false; + } + $hash = hash_init('sha256'); + $read = $size > 0 ? hash_update_stream($hash, $fp, $size) : 0; + $rest = (string)stream_get_contents($fp); + fclose($fp); + return $read === $size && hash_final($hash) === $sha256 && $rest === $suffix; + } +} +} +namespace noneDB\Internal\Maintenance { + +use function array_column, array_diff, array_filter, array_key_exists, array_keys, array_map, array_merge, + array_reverse, array_shift, array_unique, array_values, bin2hex, clearstatcache, count, dirname, explode, fclose, + fflush, file_exists, file_get_contents, flock, fopen, fread, fseek, fwrite, implode, in_array, is_array, is_dir, + is_file, is_int, is_link, is_string, json_decode, json_encode, preg_match, random_bytes, rtrim, scandir, sort, + str_pad, trim, unlink; +use const JSON_PRESERVE_ZERO_FRACTION, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE, LOCK_EX, LOCK_NB, + LOCK_UN, SORT_STRING; + +/** + * @internal + */ +final class KeyMaintenance { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dbDir; + private $maintenanceDepth; + private $storeState; + private $io; + private $locks; + private $recovery; + private $store; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->io, $this->locks, $this->recovery, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->storeState = &$this->rt->storeState; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // KEY MAINTENANCE (docs/STORAGE_FORMAT.md §9) + // ========================================== + // + // Every operation follows the maintenance order: STORE.LOCK (exclusive, non-blocking), then every + // target database's LOCK (non-blocking; one busy lock: nothing written, MAINTENANCE_BUSY), then the + // journal .nonedb-maintenance/journal.json, then "m" and a new epoch in every target's state record. + // The logical commit point is the journal in state COMMITTED with the complete target STORE record; + // the STORE copies, clearing "m" and closing the journal follow and are repeated by a re-run. + + /** + * Rotate the secret key of every database in $dbDir that belongs to $oldKey: their directories are + * renamed to the new key's locators (contents unchanged: FORMAT and metadata hold no key) and STORE + * registers $newKey instead of $oldKey. Application processes must be stopped. Re-running after an + * interruption finishes the rotation; ['abort' => true] undoes one that did not commit yet. + * @return array ['success' => bool, 'databases' => string[], 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function migrateKey(string $dbDir, string $oldKey, string $newKey, array $options = []): array { + self::checkOptions($options, ['abort', 'durability'], 'migrateKey()'); + $result = ['success' => false, 'databases' => [], 'storeManifest' => null, 'error' => null, 'code' => null]; + if(trim($oldKey) === '' || trim($newKey) === ''){ + $result['error'] = 'Both keys must be non-empty strings'; + return $result; + } + if(in_array($newKey, \noneDB\Internal\Cache\Runtime::get()->placeholderKeys, true)){ + $result['error'] = 'The new key is a placeholder; generate a random one'; + return $result; + } + if($oldKey === $newKey){ + $result['error'] = 'The new key is the same as the old key'; + return $result; + } + return self::maintenanceCall($dbDir, $oldKey, $result, $options, function(self $db) use ($oldKey, $newKey, $options){ + return $db->rotateKey($oldKey, $newKey, $options['abort'] ?? false); + }); + } + + /** + * Register a further key for the store in $dbDir; $registeredKey must already be registered + * @return array ['success' => bool, 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function registerKey(string $dbDir, string $registeredKey, string $newKey, array $options = []): array { + self::checkOptions($options, ['durability'], 'registerKey()'); + $result = ['success' => false, 'storeManifest' => null, 'error' => null, 'code' => null]; + if(trim($newKey) === '' || in_array($newKey, \noneDB\Internal\Cache\Runtime::get()->placeholderKeys, true)){ + $result['error'] = 'The new key must be a non-empty string and not a placeholder'; + return $result; + } + return self::maintenanceCall($dbDir, $registeredKey, $result, $options, function(self $db) use ($newKey){ + return $db->addStoreKey(\noneDB\Internal\Storage\StoreRecord::keyCheckOf($newKey)); + }); + } + + /** + * Rebuild a store record whose two copies are both invalid (STORAGE_FORMAT.md §9.4). Every database + * directory must belong to one of $keys. ['manifest' => path or array]: a verified STORE record (a + * backup's store.json, a maintenance result's storeManifest) is restored (complete). Without one, only + * ['confirmManualRebuild' => true] rebuilds the record from $keys (not complete: keys without + * databases that were registered before are lost). + * @return array ['success' => bool, 'mode' => 'manifest'|'manual'|null, 'complete' => bool, 'storeManifest' => array|null, + * 'error' => ?string (lists databases no key owns), 'code' => ?int] + */ + public static function repairStore(string $dbDir, array $keys, array $options = []): array { + self::checkOptions($options, ['manifest', 'confirmManualRebuild', 'durability'], 'repairStore()'); + $result = ['success' => false, 'mode' => null, 'complete' => false, 'storeManifest' => null, 'error' => null, 'code' => null]; + $keys = array_values(array_filter($keys, 'is_string')); + if(empty($keys)){ + $result['error'] = 'At least one key is needed'; + return $result; + } + return self::maintenanceCall($dbDir, $keys[0], $result, $options, function(self $db) use ($keys, $options){ + return $db->rebuildStoreRecord($keys, $options); + }); + } + + /** + * The options of a maintenance method are the ones it names, of their types (API.md "Key maintenance"), checked + * before anything is done + * @throws \InvalidArgumentException + */ + private static function checkOptions(array $options, array $known, string $method): void { + \noneDB\Internal\Config\Configuration::checkNames($options, $known, 'option', " of {$method}"); + foreach(['abort', 'confirmManualRebuild'] as $name){ + if(array_key_exists($name, $options)){ + \noneDB\Internal\Config\Configuration::optionBool($name, $options[$name]); + } + } + if(array_key_exists('manifest', $options) && !is_string($options['manifest']) && !is_array($options['manifest'])){ + throw new \InvalidArgumentException("noneDB: 'manifest' must be a file path or a store record (array)"); + } + if(array_key_exists('durability', $options)){ + \noneDB\Internal\Config\Configuration::durabilityValue($options['durability']); // the platform: as the setting's (UNSUPPORTED_DURABILITY) + } + } + + /** + * Run a maintenance operation with an instance for $dbDir and $key (no config file involved). + * $options['durability']: 'performance' (default) or 'durable', as the setting (docs/DURABILITY.md). + * @throws \InvalidArgumentException an invalid durability mode + */ + private static function maintenanceCall(string $dbDir, string $key, array $result, array $options, callable $fn): array { + if(!empty(\noneDB\Internal\Cache\Runtime::get()->lockRegistry)){ + $result['error'] = 'Maintenance can\'t run while this process holds a database lock (inside a database operation)'; + return $result; + } + $dbDir = rtrim($dbDir, '/\\') . '/'; + if(!is_dir($dbDir)){ + $result['error'] = "Directory not found: {$dbDir}"; + return $result; + } + try { + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + // The instance stays referenced while its kernel works: its destructor forgets the key + $instance = new \noneDB(['secretKey' => $key, 'dbDir' => $dbDir, 'autoCreateDB' => false, + 'durability' => $options['durability'] ?? 'performance']); + $out = $fn(\noneDB\Internal\Core\Kernel::of($instance)->keyMaintenance); + unset($instance); + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + if(is_array($out['storeManifest'] ?? null)){ + // The complete STORE record with its checksum: usable as repairStore()'s manifest as it is + $out['storeManifest'] = json_decode(\noneDB\Internal\Storage\StoreRecord::sealRecord($out['storeManifest']), true); + } + return array_merge($result, $out, ['success' => true]); + } catch(\RuntimeException $e){ + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + return array_merge($result, ['error' => $e->getMessage(), 'code' => $e->getCode()]); + } + } + + // ---------------------------------------------------------------- the maintenance order + + private function journalPath(){ + return $this->dbDir . '.nonedb-maintenance/journal.json'; + } + + private function readJournal(){ + $path = $this->journalPath(); + clearstatcache(true, $path); + if(!is_file($path)){ + return null; + } + $journal = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($path), 'journal'); + if($journal === null){ + throw new \RuntimeException("noneDB: the maintenance journal in {$this->dbDir} is unreadable; nothing is changed", \noneDB::FORMAT_DAMAGED); + } + return $journal; + } + + private function writeJournal(array $journal){ + $dir = $this->dbDir . '.nonedb-maintenance'; + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + throw new \RuntimeException("noneDB: cannot create {$dir}"); + } + if(!$this->store->writeStoreFile($this->journalPath(), \noneDB\Internal\Storage\StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'journal'] + $journal))){ + throw new \RuntimeException("noneDB: cannot write the maintenance journal in {$this->dbDir}"); + } + } + + /** + * Remove the journal: the maintenance is complete. In durable mode a removal that may not survive a power + * loss is an unknown outcome (the journal could come back and close the store until a re-run). + * @throws \RuntimeException OUTCOME_UNKNOWN + */ + private function closeJournal(){ + @unlink($this->journalPath()); + $this->recovery->removeStaleTemp($this->journalPath()); // a crashed journal write's temp (an abort writes none) + $this->sync->requireSync(null, $this->dbDir . '.nonedb-maintenance', 'dir', true); + } + + /** + * Lock every target database (locator order, non-blocking). All or nothing: a busy one releases the + * others and fails with MAINTENANCE_BUSY before anything was written. + * @param string[] $locators + * @return array locator => handle + */ + /** + * Lock every target database exclusively without waiting, in locator order (§9.2 step 2); one busy: + * all released, MAINTENANCE_BUSY, nothing written + * @param array $names locator => database name (for the message) + */ + private function lockMaintenanceTargets(array $names){ + $locators = array_keys($names); + sort($locators, SORT_STRING); + $handles = []; + foreach($locators as $locator){ + $path = $this->dbDir . $locator . '.ndb4/LOCK'; + $fp = is_dir(dirname($path)) ? @fopen($path, 'c+be') : false; + if($fp === false || !flock($fp, LOCK_EX | LOCK_NB)){ + if($fp !== false){ + fclose($fp); + } + foreach($handles as $held){ + flock($held, LOCK_UN); + fclose($held); + } + throw new \RuntimeException("noneDB: database '{$names[$locator]}' ({$locator}.ndb4) is in use; stop the application processes and run the maintenance again", \noneDB::MAINTENANCE_BUSY); + } + $handles[$locator] = $fp; + } + return $handles; + } + + private static function releaseMaintenanceTargets(array $handles){ + foreach($handles as $fp){ + flock($fp, LOCK_UN); + fclose($fp); + } + } + + /** + * Set or clear "m" in a held database lock's state record, always with a new epoch (synced) + */ + private function markMaintenance($fp, $on, $path){ + fseek($fp, 0); + $raw = trim((string)fread($fp, \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE), " \0"); + $tokens = $raw === '' ? [] : explode(' ', $raw); + $state = (string)array_shift($tokens); + $gen = preg_match('/^([0-9]{1,18}):[0-9a-f]{8}$/', $state, $m) === 1 ? (int)$m[1] : 0; + $tokens = array_values(array_filter($tokens, function($t){ return $t !== '' && $t !== 'm'; })); + if($on){ + $tokens[] = 'm'; + } + $record = str_pad(trim(($gen + 1) . ':' . bin2hex(random_bytes(4)) . ' ' . implode(' ', $tokens)), \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE); + fseek($fp, 0); + if(fwrite($fp, $record) !== \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE || !fflush($fp)){ + throw new \RuntimeException("noneDB: cannot write a database state record in {$this->dbDir}"); + } + // Durable mode: "m" and its epoch are synced before the work, the cleared record before the journal goes + $this->sync->requireSync($fp, $path, 'file'); + } + + /** + * Clear "m" of databases after the commit (their locks are taken again, waiting up to lockTimeout) + */ + private function clearMaintenanceMarks(array $locators){ + foreach($locators as $locator){ + $path = $this->dbDir . $locator . '.ndb4/LOCK'; + if(!is_file($path)){ + continue; + } + $fp = @fopen($path, 'c+be'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot open the lock of database {$locator}"); + } + try { + $this->locks->acquireFlock($fp, LOCK_EX, $locator); + $this->markMaintenance($fp, false, $path); + flock($fp, LOCK_UN); + } finally { + fclose($fp); + } + } + } + + /** + * Apply a committed journal: STORE copies, "m" cleared, journal closed + */ + private function applyCommittedJournal(array $journal){ + try { + $this->store->writeStoreCopies($journal['store']); + $this->clearMaintenanceMarks($journal['clear'] ?? []); + $this->closeJournal(); + } catch(\RuntimeException $e){ + if($e->getCode() === \noneDB::OUTCOME_UNKNOWN){ + throw $e; // the journal is gone in this process; whether that is durable is unknown + } + // Committed but not completed: never reported as success; the journal stays open (the store stays + // closed) until a re-run applies it (STORAGE_FORMAT.md §9.2) + throw new \RuntimeException("noneDB: the maintenance of the store in {$this->dbDir} is committed but not completed (" + . $e->getMessage() . "); run it again", \noneDB::MAINTENANCE_IN_PROGRESS, $e); + } + unset($this->storeState[$this->dbDir]); + return $journal['store']; + } + + // ---------------------------------------------------------------- operations + + private function rotateKey($oldKey, $newKey, $abort){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($oldKey, $newKey, $abort){ + $journal = $this->readJournal(); + if($journal !== null){ + return $this->resumeRotation($journal, $abort); + } + $store = $this->store->loadStoreRecord(); + $oldCheck = \noneDB\Internal\Storage\StoreRecord::keyCheckOf($oldKey); + if($store === null || !in_array($oldCheck, $store['keys'], true)){ + throw new \RuntimeException("noneDB: the old key is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + // The databases of the old key and where they go + $targets = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + if(!is_array($format) || !is_string($format['name'] ?? null) || \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $oldKey) !== $m[1]){ + continue; + } + $to = \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $newKey); + if(file_exists($this->dbDir . $to . '.ndb4')){ + throw new \RuntimeException("noneDB: the new key's directory for database '{$format['name']}' already exists", \noneDB::MAINTENANCE_BUSY); + } + $targets[] = ['name' => $format['name'], 'from' => $m[1], 'to' => $to, 'databaseId' => $format['databaseId'], 'done' => false]; + } + $handles = $this->lockMaintenanceTargets(array_column($targets, 'name', 'from')); + $target = $store; + $target['keys'] = array_values(array_unique(array_merge(array_diff($store['keys'], [$oldCheck]), [\noneDB\Internal\Storage\StoreRecord::keyCheckOf($newKey)]))); + sort($target['keys'], SORT_STRING); + $target['seq'] = $store['seq'] + 1; + $journal = ['op' => 'rotate', 'id' => bin2hex(random_bytes(8)), 'state' => 'STARTED', 'targets' => $targets, + 'store' => $target, 'clear' => array_column($targets, 'to')]; + try { + $this->writeJournal($journal); + foreach($targets as $t){ + $this->markMaintenance($handles[$t['from']], true, $this->dbDir . $t['from'] . '.ndb4/LOCK'); + } + } finally { + self::releaseMaintenanceTargets($handles); // "m" keeps every process out from here on + } + return $this->resumeRotation($journal, $abort); + }, true); + } finally { + $this->maintenanceDepth--; + } + } + + /** + * Continue a rotation from its journal: rename what is left, verify, commit, apply + */ + private function resumeRotation(array $journal, $abort){ + if($journal['op'] !== 'rotate'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished; run it again", \noneDB::MAINTENANCE_IN_PROGRESS); + } + if($journal['state'] === 'COMMITTED'){ + $store = $this->applyCommittedJournal($journal); + return ['databases' => array_column($journal['targets'], 'name'), 'storeManifest' => $store]; + } + if($abort){ + $this->undoRotationRenames($journal); + return ['databases' => [], 'storeManifest' => $this->store->loadStoreRecord(), 'aborted' => true]; + } + foreach($journal['targets'] as $i => $t){ + $from = $this->dbDir . $t['from'] . '.ndb4'; + $to = $this->dbDir . $t['to'] . '.ndb4'; + if(is_dir($from) && !file_exists($to)){ + if(!$this->sync->renameWithRetry($from, $to)){ + throw new \RuntimeException("noneDB: could not rename the directory of database '{$t['name']}'; run migrateKey() again"); + } + } + $journal['targets'][$i]['done'] = true; + // Verified before the commit: FORMAT, CURRENT and its generation belong to this database + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/FORMAT'), 'database'); + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/CURRENT'), 'current'); + $metadata = $current === null || !is_int($current['generation'] ?? null) ? null + : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/generations/' . $current['generation'] . '/metadata'), 'generation'); + if($format === null || ($format['databaseId'] ?? null) !== $t['databaseId'] || ($current['databaseId'] ?? null) !== $t['databaseId'] + || ($metadata['databaseId'] ?? null) !== $t['databaseId']){ + $this->undoRotationRenames($journal); + throw new \RuntimeException("noneDB: database '{$t['name']}' failed verification during the key rotation; nothing was changed", \noneDB::FORMAT_DAMAGED); + } + } + // The renames are durable before the commit (DURABILITY.md §3) + $this->sync->requireSync(null, $this->dbDir, 'dir'); + $journal['state'] = 'COMMITTED'; + $this->writeJournal($journal); + $store = $this->applyCommittedJournal($journal); + return ['databases' => array_column($journal['targets'], 'name'), 'storeManifest' => $store]; + } + + /** + * Move every renamed directory of an uncommitted rotation back (abort, failed verification), then clear + * "m" and close the journal. A rename that fails keeps the journal open: the store stays closed until + * a re-run with ['abort' => true] finishes it (§9.3). + */ + private function undoRotationRenames(array $journal){ + foreach(array_reverse($journal['targets']) as $t){ + $from = $this->dbDir . $t['from'] . '.ndb4'; + $to = $this->dbDir . $t['to'] . '.ndb4'; + clearstatcache(true, $from); + clearstatcache(true, $to); + if(is_dir($to) && !file_exists($from) && !$this->sync->renameWithRetry($to, $from)){ + throw new \RuntimeException("noneDB: could not move database '{$t['name']}' back; the key rotation stays open: run migrateKey() with ['abort' => true] again", \noneDB::MAINTENANCE_IN_PROGRESS); + } + } + $this->sync->requireSync(null, $this->dbDir, 'dir'); + $this->clearMaintenanceMarks(array_column($journal['targets'], 'from')); + $this->closeJournal(); + } + + private function addStoreKey($keyCheck){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($keyCheck){ + $journal = $this->readJournal(); + if($journal !== null){ + if($journal['op'] !== 'register' || $journal['state'] !== 'COMMITTED'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished", \noneDB::MAINTENANCE_IN_PROGRESS); + } + return ['storeManifest' => $this->applyCommittedJournal($journal)]; + } + $store = $this->store->loadStoreRecord(); + if($store === null || !in_array($this->store->keyCheck(), $store['keys'], true)){ + throw new \RuntimeException("noneDB: the registered key is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + if(in_array($keyCheck, $store['keys'], true)){ + return ['storeManifest' => $store]; + } + $target = $store; + $target['keys'][] = $keyCheck; + sort($target['keys'], SORT_STRING); + $target['seq'] = $store['seq'] + 1; + $journal = ['op' => 'register', 'id' => bin2hex(random_bytes(8)), 'state' => 'COMMITTED', 'store' => $target, 'clear' => []]; + $this->writeJournal($journal); + return ['storeManifest' => $this->applyCommittedJournal($journal)]; + }, true); + } finally { + $this->maintenanceDepth--; + } + } + + private function rebuildStoreRecord(array $keys, array $options){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($keys, $options){ + $journal = $this->readJournal(); + if($journal !== null){ + if($journal['op'] !== 'repair-store' || $journal['state'] !== 'COMMITTED'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished", \noneDB::MAINTENANCE_IN_PROGRESS); + } + $store = $this->applyCommittedJournal($journal); + return ['mode' => $journal['mode'], 'complete' => $journal['mode'] === 'manifest', 'storeManifest' => $store]; + } + [, $copy1] = $this->store->readStoreCopy('STORE'); + [, $copy2] = $this->store->readStoreCopy('STORE.2'); + if($copy1 !== null || $copy2 !== null){ + throw new \RuntimeException("noneDB: a valid store record exists in {$this->dbDir}; it is repaired automatically, repairStore() is only for two invalid copies", \noneDB::FORMAT_DAMAGED); + } + // Ownership: every database directory must belong to one of the keys + $unattributed = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + $owned = false; + foreach($keys as $key){ + if(is_array($format) && is_string($format['name'] ?? null) && \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $key) === $m[1]){ + $owned = true; + break; + } + } + if(!$owned){ + $unattributed[] = $entry; + } + } + if(!empty($unattributed)){ + throw new \RuntimeException("noneDB: these databases belong to none of the given keys: " . implode(', ', $unattributed) . "; give every key of the store", \noneDB::FORMAT_DAMAGED); + } + $keyChecks = array_map([\noneDB\Internal\Storage\StoreRecord::class, 'keyCheckOf'], $keys); + if(isset($options['manifest'])){ + // A path (store.json, a saved STORE copy) or the storeManifest array of a maintenance result; + // either way its checksum must verify + $source = $options['manifest']; + $raw = is_array($source) ? json_encode($source, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION) . "\n" + : (is_string($source) ? @file_get_contents($source) : false); + $manifest = \noneDB\Internal\Storage\StoreRecord::openRecord($raw, 'store'); + if($manifest === null || ($manifest['storageFormat'] ?? null) !== \noneDB::STORAGE_FORMAT || !is_array($manifest['keys'] ?? null) + || !is_int($manifest['seq'] ?? null) || !is_string($manifest['storeId'] ?? null)){ + throw new \RuntimeException('noneDB: the store manifest' . (is_string($source) ? " {$source}" : '') . ' is not a valid store record', \noneDB::FORMAT_DAMAGED); + } + $mode = 'manifest'; + $store = $manifest; + $store['keys'] = array_values(array_unique(array_merge($manifest['keys'], $keyChecks))); + $store['seq'] = (int)$manifest['seq'] + 1; + } elseif(!empty($options['confirmManualRebuild'])){ + $mode = 'manual'; + $store = ['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT, 'storeId' => \noneDB\Internal\Storage\StoreRecord::uuid4(), + 'seq' => 1, 'keys' => array_values(array_unique($keyChecks))]; + } else { + throw new \RuntimeException("noneDB: without a verified store manifest the record can only be rebuilt by hand: pass ['confirmManualRebuild' => true] (keys without databases that were registered are not restored)", \noneDB::FORMAT_DAMAGED); + } + sort($store['keys'], SORT_STRING); + unset($store['checksum']); + $journal = ['op' => 'repair-store', 'id' => bin2hex(random_bytes(8)), 'state' => 'COMMITTED', 'mode' => $mode, 'store' => $store, 'clear' => []]; + $this->writeJournal($journal); + $store = $this->applyCommittedJournal($journal); + $this->store->logStoreEvent(['action' => $mode === 'manual' ? 'manual-store-rebuild' : 'store-restored-from-manifest', 'keys' => count($store['keys'])]); + return ['mode' => $mode, 'complete' => $mode === 'manifest', 'storeManifest' => $store]; + }, true); + } finally { + $this->maintenanceDepth--; + } + } +} +} +namespace noneDB\Internal\Maintenance { + +use function array_keys, clearstatcache, count, file_exists, filesize, intdiv, max; + +/** + * @internal + */ +final class Maintenance { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $bufferCountLimit; + private $bufferEnabled; + private $bufferFlushInterval; + private $bufferSizeLimit; + private $indexCache; + private $shardSize; + private $shardingEnabled; + private $lockRegistry; + private $buffer; + private $cache; + private $checkpoints; + private $compaction; + private $db; + private $fieldIndex; + private $io; + private $layout; + private $layoutSwitch; + private $locks; + private $rtree; + private $scope; + private $segments; + private $shardMeta; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->compaction, $this->db, $this->fieldIndex, $this->io, $this->layout, $this->layoutSwitch, $this->locks, $this->rtree, $this->scope, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->bufferCountLimit = &$this->k->bufferCountLimit; + $this->bufferEnabled = &$this->k->bufferEnabled; + $this->bufferFlushInterval = &$this->k->bufferFlushInterval; + $this->bufferSizeLimit = &$this->k->bufferSizeLimit; + $this->indexCache = &$this->k->indexCache; + $this->shardSize = &$this->k->shardSize; + $this->shardingEnabled = &$this->k->shardingEnabled; + $this->lockRegistry = &$this->rt->lockRegistry; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->compaction = null; + $this->db = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->layoutSwitch = null; + $this->locks = null; + $this->rtree = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE BUFFER PUBLIC API + // ========================================== + + /** + * Manually flush buffer for a database + * @param string $dbname + * @return array ['success' => bool, 'flushed' => int, 'error' => string|null] + */ + public function flush($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->flushUnlocked($dbname); + }, 'flush'); + } + + private function flushUnlocked($dbname){ + $flushed = $this->buffer->flushPendingBuffers($dbname); + return ['success' => true, 'flushed' => $flushed, 'error' => null]; + } + + /** + * Flush the write buffers of every database (of this secretKey) in dbDir + * @return array ['databases' => int, 'flushed' => int] + */ + public function flushAllBuffers(){ + $names = []; + if(!empty($this->lockRegistry)){ + return ['databases' => 0, 'flushed' => 0]; // inside an interrupted operation (see releaseInterruptedSessions) + } + foreach($this->db->getDBs(false) as $dbname){ + $buffered = $this->locks->withLock($dbname, false, function() use ($dbname){ + return $this->buffer->hasBuffer($this->buffer->getBufferPath($dbname)) || !empty($this->buffer->flushingFiles($dbname)); + }); + if($buffered){ + $names[$dbname] = true; + } + } + $total = 0; + foreach(array_keys($names) as $dbname){ + $result = $this->flush((string)$dbname); + $total += $result['flushed']; + } + return ['databases' => count($names), 'flushed' => $total]; + } + + /** + * Get buffer information for a database + * @param string $dbname + * @return array + */ + public function getBufferInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + $fn = function() use ($dbname){ + return $this->getBufferInfoUnlocked($dbname); + }; + return $this->locks->withLock($dbname, false, $fn); + } + + private function getBufferInfoUnlocked($dbname){ + $info = [ + 'enabled' => $this->bufferEnabled, + 'sizeLimit' => $this->bufferSizeLimit, + 'countLimit' => $this->bufferCountLimit, + 'flushInterval' => $this->bufferFlushInterval, + 'buffers' => [] + ]; + $describe = function($path){ + clearstatcache(true, $path); + $exists = file_exists($path) && filesize($path) > 0; + return [ + 'exists' => $exists, + 'size' => $exists ? (int)filesize($path) : 0, + 'records' => $exists ? $this->buffer->getBufferRecordCount($path) : 0 + ]; + }; + $info['buffers']['main'] = $describe($this->buffer->getBufferPath($dbname)); + return $info; + } + + /** + * Enable or disable write buffering + * @param bool $enable + */ + public function enableBuffering($enable = true){ + $this->bufferEnabled = (bool)$enable; + } + + /** + * Check if buffering is enabled + * @return bool + */ + public function isBufferingEnabled(){ + return $this->bufferEnabled; + } + + /** + * Set buffer size limit (in bytes) + * @param int $bytes + */ + public function setBufferSizeLimit($bytes){ + $this->bufferSizeLimit = max(1024, (int)$bytes); // Minimum 1KB + } + + /** + * Set buffer flush interval (in seconds) + * @param int $seconds 0 to disable time-based flush + */ + public function setBufferFlushInterval($seconds){ + $this->bufferFlushInterval = max(0, (int)$seconds); + } + + /** + * Set buffer count limit + * @param int $count + */ + public function setBufferCountLimit($count){ + $this->bufferCountLimit = max(10, (int)$count); // Minimum 10 records + } + + /** + * Compact a database: drop old record versions and deleted records from the data files. + * Keys are preserved (sharded and non-sharded alike); indexes stay valid. + * @param string $dbname + * @return array ['success' => bool, 'freedSlots' => int, 'totalRecords' => int, 'sharded' => bool] + */ + public function compact($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $this->scope->refuseInUncommittedWrite($dbname, 'compact'); + return $this->compactUnlocked($dbname); + }, 'compact'); + } + + private function compactUnlocked($dbname){ + $result = array("success" => false, "freedSlots" => 0); + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + $result['status'] = 'database_not_found'; + return $result; + } + + $sharded = $this->segments->isSharded($dbname); + $freed = 0; + if($sharded){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + $freed = (int)($meta['deletedCount'] ?? 0); + } + $total = 0; + foreach($this->segments->listSegments($dbname) as $segment){ + $compacted = $this->compaction->compactJsonl($dbname, $segment); + if(!$sharded){ + $freed += $compacted['freed']; + } + $total += $compacted['compacted']; + $this->cache->evictBetweenSegments(); + } + if($sharded){ + $this->removeEmptyShards($dbname); + $meta = $this->shardMeta->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + $meta['shards'][$i]['deleted'] = 0; + } + $meta['deletedCount'] = 0; + $meta['totalRecords'] = $total; + $this->shardMeta->writeMeta($dbname, $meta); + } + + $result['success'] = true; + $result['freedSlots'] = $freed; + $result['totalRecords'] = $total; + $result['sharded'] = $sharded; + if($sharded){ + $result['newShardCount'] = count($this->segments->listSegments($dbname)); + } + return $result; + } + + /** + * Remove shards without records (keys never move, so nothing else changes). The shard that holds + * the newest key is kept: new records go there. + */ + private function removeEmptyShards($dbname){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + if($meta === null || count($meta['shards']) < 2){ + return; + } + $lastShard = intdiv(max(0, (int)$meta['nextKey'] - 1), (int)$meta['shardSize']); + $kept = []; + $removed = []; + $done = false; // a shard was read: the cache may drop what it loaded before the next one + foreach($meta['shards'] as $shard){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $shardId = (int)$shard['id']; + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($shardId !== $lastShard && ($index === null || \noneDB\Internal\Storage\KeyIndex::count($index) === 0)){ + $removed[] = $shardId; + } else { + $kept[] = $shard; + } + } + if(empty($removed)){ + return; + } + $meta['shards'] = $kept; + // The meta must not lose this change in a power loss after the files are gone (where PHP can sync) + $this->shardMeta->writeMeta($dbname, $meta, true); + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $this->locks->bumpEpoch($dbname); // shard files go away + foreach($removed as $shardId){ + $dataPath = $this->segments->getShardPath($dbname, $shardId); + foreach($manifest['fields'] as $field){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $shardId); + } + $this->rtree->removeSpatialShardFiles($dbname, $shardId); + $this->io->fsUnlink(\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)); + $this->io->fsUnlink($dataPath); + $this->cache->markFileNotExists($dataPath); + unset($this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]); + } + // The shard list no longer names them: their removal only frees space. A power loss may bring them + // back as files no shard entry names (set aside when a shard with that number is created again). + foreach($manifest['fields'] as $field){ + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + } + } + /** + * Switch a database of the single layout to the sharded layout (STORAGE_FORMAT.md §5.2) + * @param string $dbname + * @return array ["success" => bool, "status" => string] + */ + public function convertToSharded($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $this->scope->refuseInUncommittedWrite($dbname, 'convertToSharded'); + return $this->convertToShardedUnlocked($dbname); + }, 'convertToSharded'); + } + + private function convertToShardedUnlocked($dbname){ + if($this->segments->isSharded($dbname)){ + return array("success" => true, "status" => "already_sharded"); + } + $this->buffer->flushPendingBuffers($dbname); + if($this->segments->isSharded($dbname)){ + return array("success" => true, "status" => "converted"); + } + if(!$this->cache->cachedFileExists($this->segments->getDataPath($dbname))){ + return array("success" => false, "status" => "database_not_found"); + } + if($this->layoutSwitch->convertToSharded($dbname)){ + return array("success" => true, "status" => "converted"); + } + return array("success" => false, "status" => "conversion_failed"); + } + + /** + * Check if sharding is enabled + * @return bool + */ + public function isShardingEnabled(){ + return $this->shardingEnabled; + } + + /** + * Get current shard size setting + * @return int + */ + public function getShardSize(){ + return $this->shardSize; + } +} +} +namespace noneDB\Internal\Query { + +/** + * @internal + */ +final class Aggregation { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $layout; + private $locks; + private $planner; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->layout, $this->locks, $this->planner); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Sum numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function sum($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->sumUnlocked($dbname, $field, $filter); + }); + } + + private function sumUnlocked($dbname, $field, $filter = 0){ + $sum = 0; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$sum){ + foreach($records as $record){ + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + } + } + }, true); + return $sum; + } + + /** + * Average of numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function avg($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->avgUnlocked($dbname, $field, $filter); + }); + } + + private function avgUnlocked($dbname, $field, $filter = 0){ + $sum = 0; + $count = 0; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$sum, &$count){ + foreach($records as $record){ + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + $count++; + } + } + }, true); + return $count > 0 ? $sum / $count : 0; + } + + /** + * Get minimum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function min($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->minUnlocked($dbname, $field, $filter); + }); + } + + private function minUnlocked($dbname, $field, $filter = 0){ + // The order of sort() (and of the query builder's min()); loose < depended on the insertion order + $min = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$min){ + foreach($records as $record){ + if(isset($record[$field]) && ($min === null || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $min) < 0)){ + $min = $record[$field]; + } + } + }, true); + return $min; + } + + /** + * Get maximum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function max($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->maxUnlocked($dbname, $field, $filter); + }); + } + + private function maxUnlocked($dbname, $field, $filter = 0){ + $max = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$max){ + foreach($records as $record){ + if(isset($record[$field]) && ($max === null || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $max) > 0)){ + $max = $record[$field]; + } + } + }, true); + return $max; + } +} +} +namespace noneDB\Internal\Query { + +use function array_chunk, array_flip, array_key_exists, array_values, count, in_array, is_array, is_int, is_string, preg_match; + +/** + * @internal + */ +final class FindFilter { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $cache; + private $checkpoints; + private $fieldIndex; + private $records; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * The filter parameter of find(), count(), sum(), ...: 0, null (all records) and arrays are filters, + * anything else is invalid (null) + * @return array|null + */ + public function normalizeFilter($filter){ + if($filter === null || is_int($filter)){ + return []; + } + return is_array($filter) ? $filter : null; + } + + // ========================================== + // FILTER MATCHING (find / update / delete) + // ========================================== + + /** + * Normalize a "key" filter value to a list of int keys (invalid values match nothing) + * @return array + */ + public function normalizeKeyFilter($value){ + $values = is_array($value) ? $value : [$value]; + $keys = []; + foreach($values as $v){ + if(is_int($v)){ + $k = $v; + } elseif(is_string($v) && preg_match('/\A(0|[1-9][0-9]*)\z/', $v)){ // \z: "5\n" is not 5 + $k = (int)$v; + } else { + continue; + } + if($k >= 0 && !isset($keys[$k])){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + + /** + * Strict equality filter used by find(), update() and delete(): + * every field must exist in the record and be identical (===) to the filter value. + * "key" filters compare against the record key. + */ + public function matchesFilters(array $record, array $filters){ + foreach($filters as $field => $value){ + if($field === 'key'){ + if(!isset($record['key']) || !in_array($record['key'], $this->normalizeKeyFilter($value), true)){ + return false; + } + continue; + } + if(!array_key_exists($field, $record) || $record[$field] !== $value){ + return false; + } + } + return true; + } + + /** + * Records matching $filters, in key order (or in the order of a "key" filter) + * @return array [key => record] + */ + public function selectRecords($dbname, array $filters){ + $result = []; + foreach($this->segmentMatches($dbname, $filters) as $records){ + $result += $records; + } + if(count($result) > 1 && array_key_exists('key', $filters)){ + $ordered = []; + foreach($this->normalizeKeyFilter($filters['key']) as $key){ + if(isset($result[$key])){ + $ordered[$key] = $result[$key]; + } + } + return $ordered; + } + return $result; + } + + /** + * The records matching $filters one segment at a time: yields segment id (null: a single-layout database) + * => [key => record], in key order (a "key" filter: in its order within the segment). Once the consumer is + * done with a segment the cache may drop what it loaded (evictBetweenSegments()), so a write by filter + * over a large database holds one segment's records. Caller holds the lock. + * @return \Generator + */ + public function segmentMatches($dbname, array $filters){ + if(array_key_exists('key', $filters)){ + $bySegment = []; + foreach($this->normalizeKeyFilter($filters['key']) as $key){ + $segment = $this->segments->segmentForKey($dbname, $key); + if($segment !== false){ + $bySegment[$segment === null ? '' : $segment][] = $key; + } + } + unset($filters['key']); + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($bySegment as $segment => $keys){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $segment = $segment === '' ? null : (int)$segment; + foreach(array_chunk($keys, \noneDB\Internal\Storage\RecordIO::BATCH) as $chunk){ + // In the order of the filter's keys + $found = $this->records->readSegmentRecords($dbname, $segment, $chunk); + $matches = []; + foreach($chunk as $key){ + if(isset($found[$key]) && $this->matchesFilters($found[$key], $filters)){ + $matches[$key] = $found[$key]; + } + } + unset($found); + if(!empty($matches)){ + yield $segment => $matches; + } + } + } + return; + } + + // Shards the global field indexes allow (null: every segment); none: nothing to read at all + $targets = $this->fieldIndex->targetShardsFromGlobalIndex($dbname, $filters); + if($targets === []){ + return; + } + $segments = $this->segments->listSegments($dbname); + if($targets !== null){ + // The targets that are segments (a global index may still name a removed shard), in order + $present = array_flip($segments); + $segments = []; + foreach($targets as $target){ + if(isset($present[$target])){ + $segments[] = $target; + } + } + } + $usable = $this->fieldIndex->indexedFilters($dbname, $filters); + $done = false; + foreach($segments as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + $done = false; + } + $keyIndex = null; + $candidates = null; + if(!empty($usable)){ + $keyIndex = $this->checkpoints->readJsonlIndex($dbname, $segment); + $candidates = $this->fieldIndex->segmentCandidates($dbname, $segment, $usable, $keyIndex); + if($candidates === []){ + continue; + } + } + foreach($this->records->eachSegmentBatch($dbname, $segment, $candidates, $keyIndex) as &$batch){ + $matches = []; + foreach($batch as $key => $record){ + if(empty($filters) || $this->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + $batch = null; // the records that don't match go before the caller writes the matches + if(!empty($matches)){ + yield $segment => $matches; + } + } + unset($batch); + $done = true; + } + } +} +} +namespace noneDB\Internal\Query { + +use function array_fill_keys, array_filter, array_flip, array_intersect_key, array_key_exists, array_keys, + array_slice, array_values, count, intdiv, is_array, min, sort; +use const PHP_INT_MIN, SORT_NUMERIC; + +/** + * @internal + */ +final class Planner { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $fieldIndexEnabled; + private $noWrites; + private $cache; + private $checkpoints; + private $fieldIndex; + private $findFilter; + private $locks; + private $records; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->findFilter, $this->locks, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->findFilter = null; + $this->locks = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // QUERY PLANS AND CURSORS + // ========================================== + // A plan is a tree of what a query's filters ask for that indexes can answer: + // ['eq', field, value] ['in', field, values] ['range', field, operator, operand] + // ['keys', [key, ...]] ['any'] ['and', [nodes]] ['or', [nodes]] + // For each segment it gives candidate keys (a superset of the matches: the caller checks every record + // with the whole filter) or null (scan the segment); shards it can rule out (global field indexes, + // key ranges) are skipped. Negations (whereNot, $ne, $nin, notLike, ...) never use indexes. + // scanRecords() reads the candidates of every segment in key order, in batches (32, growing to 4096), + // and stops as soon as the visitor says so: first(), exists() and limit() read what they need. + + /** + * Candidate keys of a segment for a plan node; null when the node can't narrow the segment down + * @return int[]|null sorted + */ + private function planCandidates($dbname, $shardId, array $node){ + switch($node[0]){ + case 'eq': + if(!$this->fieldIndexEnabled || !$this->fieldIndex->isIndexableValue($node[2]) || !$this->fieldIndex->isFieldIndexed($dbname, $node[1])){ + return null; + } + return $this->fieldIndex->fieldIndexCandidates($dbname, $shardId, [$node[1] => $node[2]]); + case 'in': + if(!$this->fieldIndexEnabled || !$this->fieldIndex->isFieldIndexed($dbname, $node[1])){ + return null; + } + $keys = []; + foreach($node[2] as $value){ + if(!$this->fieldIndex->isIndexableValue($value)){ + return null; + } + $found = $this->fieldIndex->fieldIndexCandidates($dbname, $shardId, [$node[1] => $value]); + if($found === null){ + return null; + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'range': + return $this->fieldIndex->fieldIndexRangeCandidates($dbname, $shardId, $node[1], [$node[2] => $node[3]]); + case 'keys': + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + $keys = []; + foreach($node[1] as $k){ + if($index !== null && \noneDB\Internal\Storage\KeyIndex::has($index, (int)$k)){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + continue; + } + $result = $result === null ? $found : array_keys(array_intersect_key(array_flip($result), array_flip($found))); + if(empty($result)){ + return []; + } + } + return $result; + case 'or': + $keys = []; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + return null; // one branch needs a scan: the whole segment + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + return null; + } + + /** + * Shards a plan node can match in (null: all). Only skips a shard when it can't hold a match. + * @return array|null shard id => true + */ + private function planShards($dbname, array $node){ + if(!$this->segments->isSharded($dbname)){ + return null; + } + switch($node[0]){ + case 'eq': + if(!$this->fieldIndex->isIndexableValue($node[2])){ + return null; + } + $shards = $this->fieldIndex->targetShardsFromGlobalIndex($dbname, [$node[1] => $node[2]]); + return $shards === null ? null : array_fill_keys($shards, true); + case 'in': + $all = []; + foreach($node[2] as $value){ + $found = $this->planShards($dbname, ['eq', $node[1], $value]); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + case 'keys': + $size = $this->segments->shardSizeOf($dbname); + $shards = []; + foreach($node[1] as $k){ + $shards[intdiv($k, $size)] = true; + } + return $shards; + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found !== null){ + $result = $result === null ? $found : array_intersect_key($result, $found); + } + } + return $result; + case 'or': + $all = []; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + } + return null; + } + + /** + * Read the records a plan allows, segment by segment in key order, in batches: $visit([key => record]) + * gets each batch (never an empty one) in key order and returns false to stop. Caller holds the lock. + * @param bool $whole The visitor never stops (count, top-k): each segment is read in one go (RecordIO::BATCH records at a time when it is larger) + * @return array ['segments', 'skipped', 'indexed', 'scanned', 'recordsRead', 'plan'] + */ + public function scanRecords($dbname, array $plan, callable $visit, $whole = false){ + $stats = ['segments' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0, 'recordsRead' => 0, 'plan' => []]; + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return $stats; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($segments as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + $done = false; + } + $stats['segments']++; + if($shards !== null && !isset($shards[$segment])){ + $stats['skipped']++; + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } + continue; + } + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + $access = $keys === null ? 'scan' : 'index'; + if($keys === null){ + $stats['scanned']++; + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + $count = $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index); + // A scan that can't stop early reads the segment as a whole (RecordIO::eachSegmentBatch()): no key list + $keys = $whole || $count === 0 ? null : \noneDB\Internal\Storage\KeyIndex::keys($index); + } else { + $stats['indexed']++; + $count = count($keys); + } + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => $access, 'candidates' => $count]; + } + if($count === 0){ + continue; + } + if($keys === null){ + foreach($this->records->eachSegmentBatch($dbname, $segment) as $records){ + $stats['recordsRead'] += count($records); + if($visit($records) === false){ + return $stats; + } + } + $done = true; + continue; + } + // Keys come in ascending order almost always (they are assigned in order, an update keeps a + // key's place in the index): sorted only when they aren't + $previous = PHP_INT_MIN; + foreach($keys as $k){ + if($k < $previous){ + sort($keys, SORT_NUMERIC); + break; + } + $previous = $k; + } + if($whole){ + // Index candidates for a visitor that never stops: a batch at a time (RecordIO::BATCH) + foreach($this->records->eachSegmentBatch($dbname, $segment, $keys) as $records){ + $stats['recordsRead'] += count($records); + if($visit($records) === false){ + return $stats; + } + } + $done = true; + continue; + } + // Batches that grow from 32: a visitor that stops early reads little + $n = count($keys); + for($i = 0, $batch = 32; $i < $n; $i += $batch, $batch = min(4096, $batch * 4)){ + $records = $this->records->readSegmentRecords($dbname, $segment, $access === 'scan' && $i === 0 && $batch >= $n ? null : array_slice($keys, $i, $batch)); + $stats['recordsRead'] += count($records); + if(!empty($records) && $visit($records) === false){ + return $stats; + } + } + $done = true; + } + return $stats; + } + + /** + * explain(): what scanRecords() would do, without reading records or building anything (an index that + * a read would build counts as a scan) + */ + public function explainPlan($dbname, array $plan){ + $explain = function() use ($dbname, $plan){ + $out = ['sharded' => $this->segments->dbExists($dbname) && $this->segments->isSharded($dbname), 'segments' => ['total' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0], + 'plan' => [], 'estimatedRead' => 0]; + if(!$this->segments->dbExists($dbname)){ + return $out; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + foreach($this->segments->listSegments($dbname) as $segment){ + $out['segments']['total']++; + if($shards !== null && !isset($shards[$segment])){ + $out['segments']['skipped']++; + $entry = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } else { + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + if($keys === null){ + $out['segments']['scanned']++; + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + $entry = ['segment' => $segment, 'access' => 'scan', 'candidates' => $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index)]; + } else { + $out['segments']['indexed']++; + $entry = ['segment' => $segment, 'access' => 'index', 'candidates' => count($keys)]; + } + } + $out['estimatedRead'] += $entry['candidates']; + if(count($out['plan']) < 50){ + $out['plan'][] = $entry; + } + unset($index, $keys); + $this->cache->evictBetweenSegments(); + } + $out['planTruncated'] = $out['segments']['total'] > 50; + return $out; + }; + $this->noWrites++; + try { + return $this->locks->withLock($dbname, false, $explain); + } finally { + $this->noWrites--; + } + } + + + /** + * Visit the records matching the filter parameter of find(), count(), sum(), ... (FindFilter::normalizeFilter) + * in find()'s order, none kept: $visit([key => record]) gets the matching records of each batch read and + * returns false to stop. One segment's records are held at a time (a "key" filter: the records of its keys, + * in its order, in one batch); indexes plan the scan. False for an invalid filter or a missing database + * without autoCreateDB (where find() returns false). Caller holds the lock. + * @param bool $whole The visitor never stops (count, aggregations): each segment is read in one go (RecordIO::BATCH records at a time when it is larger) + * @return bool + */ + public function eachMatch($dbname, $filter, callable $visit, $whole = false){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null){ + return false; + } + if(!$this->segments->dbExists($dbname)){ + return (bool)$this->autoCreateDB; + } + if(array_key_exists('key', $filters)){ + $records = $this->findFilter->selectRecords($dbname, $filters); + if(!empty($records)){ + $visit($records); + } + return true; + } + $and = []; + foreach($filters as $field => $value){ + $and[] = is_array($value) ? ['any'] : ['eq', (string)$field, $value]; + } + $plan = count($and) === 1 ? $and[0] : ['and', $and]; + if($plan[0] === 'and'){ + $plan[1] = array_values(array_filter($plan[1], function($node){ return $node[0] !== 'any'; })); + $plan = empty($plan[1]) ? ['any'] : (count($plan[1]) === 1 ? $plan[1][0] : $plan); + } + $this->scanRecords($dbname, $plan, empty($filters) ? $visit : function(array $records) use ($filters, $visit){ + $matches = []; + foreach($records as $key => $record){ + if($this->findFilter->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + return empty($matches) || $visit($matches) !== false; + }, $whole); + return true; + } +} +} +namespace noneDB\Internal\Query { + +use function array_column, array_key_exists, array_values, count, is_array, is_bool, is_scalar, strtolower, + usort; + +/** + * @internal + */ +final class QueryMethods { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $checkpoints; + private $findFilter; + private $layout; + private $locks; + private $planner; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->findFilter, $this->layout, $this->locks, $this->planner, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->findFilter = null; + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // QUERY METHODS + // ========================================== + + /** + * Get unique values for a field + * @param string $dbname + * @param string $field + * @return array|false + */ + public function distinct($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->distinctUnlocked($dbname, $field); + }); + } + + private function distinctUnlocked($dbname, $field){ + $seen = []; + $values = []; + $found = $this->planner->eachMatch($dbname, [], function(array $records) use ($field, &$seen, &$values){ + \noneDB\Internal\Query\Values::collectUnique($records, $field, $seen, $values); + }, true); + return $found ? $values : false; + } + + /** + * Sort array by field + * @param array $array Result from find() + * @param string $field Field to sort by + * @param string $order 'asc' or 'desc' + * @return array|false + */ + public function sort($array, $field, $order = 'asc'){ + if(!is_array($array)) return false; + if(count($array) === 0) return []; + $desc = strtolower((string)$order) === 'desc'; + $items = []; + foreach(array_values($array) as $i => $row){ + $items[] = [\noneDB\Internal\Query\Values::sortKey(is_array($row) && array_key_exists($field, $row) ? $row[$field] : null), $i, $row]; + } + usort($items, function($a, $b) use ($desc){ + $result = $a[0] <=> $b[0]; + if($desc){ + $result = -$result; + } + return $result !== 0 ? $result : $a[1] <=> $b[1]; // stable + }); + return array_column($items, 2); + } + + /** + * Count records matching filter + * @param string $dbname + * @param mixed $filter + * @return int + */ + public function count($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->countUnlocked($dbname, $filter); + }); + } + + private function countUnlocked($dbname, $filter = 0){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null){ + return 0; // a filter find() rejects matches nothing + } + // Fast-path: no filter = use index/meta counters + if(empty($filters)){ + return $this->countFast($dbname); + } + $count = 0; + $this->planner->eachMatch($dbname, $filters, function(array $records) use (&$count){ $count += count($records); }, true); + return $count; + } + + /** + * Count without reading records: index entries (non-sharded) or meta totalRecords (sharded) + * @param string $dbname Already normalized + * @return int + */ + public function countFast($dbname){ + if(!$this->segments->dbExists($dbname)){ + return 0; + } + if($this->segments->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + return $meta !== null ? (int)($meta['totalRecords'] ?? 0) : 0; + } + $index = $this->checkpoints->readJsonlIndex($dbname, null); + return $index !== null ? \noneDB\Internal\Storage\KeyIndex::count($index) : 0; + } + + /** + * Pattern matching search (LIKE) + * @param string $dbname + * @param string $field + * @param string $pattern Use ^ for starts with, $ for ends with + * @return array|false + */ + public function like($dbname, $field, $pattern){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $pattern){ + return $this->likeUnlocked($dbname, $field, $pattern); + }); + } + + private function likeUnlocked($dbname, $field, $pattern){ + if(!is_scalar($pattern)){ + return false; // an array is never a pattern subject + } + $result = []; + $like = \noneDB\Internal\Query\Values::compileLike((string)$pattern); + $found = $this->planner->eachMatch($dbname, [], function(array $records) use ($field, $like, &$result){ + foreach($records as $record){ + $value = $record[$field] ?? null; + // Only strings and numbers (true used to match "1", like in the query builder it never does) + if(is_scalar($value) && !is_bool($value) && \noneDB\Internal\Query\Values::likeMatch($like, (string)$value)){ + $result[] = $record; + } + } + }, true); + return $found ? $result : false; + } +} +} +namespace noneDB\Internal\Query { + +use function abs, array_filter, array_key_exists, array_keys, array_map, array_slice, array_unique, array_values, + asort, atan2, cos, count, deg2rad, fmod, in_array, is_array, is_finite, is_float, is_int, is_nan, max, min, round, + sin, sort, sqrt, usort; +use const INF; + +/** + * @internal + */ +final class SpatialQueries { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $indexedCandidates; + private $spatialIndexEnabled; + private $cache; + private $fieldIndex; + private $geoJson; + private $geometry; + private $layout; + private $locks; + private $queryMethods; + private $records; + private $rtree; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->geoJson, $this->geometry, $this->layout, $this->locks, $this->queryMethods, $this->records, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->indexedCandidates = &$this->k->indexedCandidates; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->layout = null; + $this->locks = null; + $this->queryMethods = null; + $this->records = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL QUERIES + // ========================================== + + /** + * Records whose geometry may overlap the search boxes, in key order, one segment at a time (the cache may + * drop what a segment loaded once it is done): yields key => record. Uses the R-tree (and shard boxes) + * when the field is indexed; otherwise every record. + * @param array $boxes List of MBRs + * @return \Generator + */ + private function spatialCandidates(string $dbname, string $field, array $boxes) { + if (!$this->segments->dbExists($dbname)) { + return; + } + $indexed = $this->spatialIndexEnabled && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + $this->indexedCandidates = []; + $sharded = $this->segments->isSharded($dbname); + $global = ($indexed && $sharded) ? $this->rtree->readGlobalSpatialIndex($dbname, $field) : null; + + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach ($this->segments->listSegments($dbname) as $segment) { + if ($done) { + $this->cache->evictBetweenSegments(); + $done = false; + } + $keys = null; // null: every record of the segment + // Shard skip: only with an entry that fits the shard, misses the boxes, and no newer geometry of + // the shard in them. A missing entry (or global file) means unknown: searched. + if ($indexed && $global !== null && $this->rtree->globalSpatialSkips($dbname, $field, $segment, $global, $boxes)) { + continue; + } + if ($indexed) { + try { + $keys = $this->rtree->rtreeSearchMBR($dbname, $field, $boxes, $segment); + foreach ($keys as $key) { + $this->indexedCandidates[$key] = true; + } + } catch (\Throwable $e) { + if ($e->getCode() === \noneDB::LOCK_FAILED) { + throw $e; + } + $keys = null; // the index can't be built: scan + } + } + if ($keys === null || !empty($keys)) { + foreach ($this->records->eachSegmentBatch($dbname, $segment, $keys) as $batch) { + yield from $batch; + } + } + $done = true; + } + } + + /** + * Search boxes for a circle; splits at the antimeridian and covers all longitudes near the poles + * @return array List of [minLon, minLat, maxLon, maxLat] + */ + private function circleToBBoxes(float $lon, float $lat, float $radiusMeters): array { + $latDeg = $radiusMeters / 111000.0; + $minLat = $lat - $latDeg; + $maxLat = $lat + $latDeg; + if ($minLat <= -90 || $maxLat >= 90) { + return [[-180.0, max(-90.0, $minLat), 180.0, min(90.0, $maxLat)]]; + } + $cos = cos(deg2rad(max(abs($minLat), abs($maxLat)))); + $lonDeg = $cos > 1e-9 ? $radiusMeters / (111000.0 * $cos) : 360.0; + if ($lonDeg >= 180) { + return [[-180.0, $minLat, 180.0, $maxLat]]; + } + $minLon = $lon - $lonDeg; + $maxLon = $lon + $lonDeg; + if ($minLon < -180) { + return [[-180.0, $minLat, $maxLon, $maxLat], [$minLon + 360.0, $minLat, 180.0, $maxLat]]; + } + if ($maxLon > 180) { + return [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon - 360.0, $maxLat]]; + } + return [[$minLon, $minLat, $maxLon, $maxLat]]; + } + + /** + * Distance in meters from a point to a geometry: 0 inside a polygon, otherwise to the nearest + * vertex or edge (edges use a local equirectangular approximation) + */ + public function distanceToGeometry(float $lon, float $lat, array $geometry): float { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + switch ($type) { + case 'Point': + return $this->geometry->haversineDistance($lon, $lat, (float)$coords[0], (float)$coords[1]); + case 'MultiPoint': + $best = INF; + foreach ($coords as $p) { + $best = min($best, $this->geometry->haversineDistance($lon, $lat, (float)$p[0], (float)$p[1])); + } + return $best; + case 'LineString': + return $this->distanceToPath($lon, $lat, $coords); + case 'MultiLineString': + $best = INF; + foreach ($coords as $line) { + $best = min($best, $this->distanceToPath($lon, $lat, $line)); + } + return $best; + case 'Polygon': + if ($this->geometry->pointInPolygon($lon, $lat, $coords)) { + return 0.0; + } + $best = INF; + foreach ($coords as $ring) { + $best = min($best, $this->distanceToPath($lon, $lat, $ring)); + } + return $best; + case 'MultiPolygon': + $best = INF; + foreach ($coords as $polygon) { + $best = min($best, $this->distanceToGeometry($lon, $lat, ['type' => 'Polygon', 'coordinates' => $polygon])); + } + return $best; + case 'GeometryCollection': + $best = INF; + foreach ($geometry['geometries'] ?? [] as $g) { + $best = min($best, $this->distanceToGeometry($lon, $lat, $g)); + } + return $best; + } + return INF; + } + + /** + * Distance from a point to a polyline (meters): to its nearest point, an edge being straight in + * longitude/latitude. Exact up to floating point: every vertex is measured, and every edge whose + * bounding box could hold a nearer point is searched (golden section on the haversine of the point at + * t along the edge, which wraps at the antimeridian by itself); the other edges are skipped by a lower + * bound of the haversine over their box. + */ + private function distanceToPath(float $lon, float $lat, array $points): float { + $n = count($points); + if ($n === 0) { + return INF; + } + $cosLat = cos(deg2rad($lat)); + $xs = []; + $ys = []; + $best = INF; // the haversine "a" of the nearest point found (monotonic in the distance) + foreach ($points as $i => $point) { + $xs[$i] = (float)$point[0]; + $ys[$i] = (float)$point[1]; + $best = min($best, $this->haversineA($lon, $lat, $cosLat, $xs[$i], $ys[$i])); + } + $points = array_keys($xs); + $bounds = []; + for ($e = 0; $e < $n - 1; $e++) { + $i = $points[$e]; + $j = $points[$e + 1]; + $bound = $this->edgeBoxLowerBound($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j]); + if ($bound < $best) { + $bounds[$e] = $bound; + } + } + asort($bounds); + foreach ($bounds as $e => $bound) { + if ($bound >= $best) { + break; // every other edge is at least this far + } + $i = $points[$e]; + $j = $points[$e + 1]; + $best = min($best, $this->edgeMinimumA($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j])); + } + return 6371000.0 * (2 * atan2(sqrt($best), sqrt(1 - $best))); + } + + /** + * The "a" of haversineDistance() (the same operations, so a vertex gives the same distance) + */ + private function haversineA(float $lon, float $lat, float $cosLat, float $x, float $y): float { + $latDelta = deg2rad($y - $lat); + $lonDelta = deg2rad($x - $lon); + return sin($latDelta / 2) * sin($latDelta / 2) + $cosLat * cos(deg2rad($y)) * sin($lonDelta / 2) * sin($lonDelta / 2); + } + + /** + * A lower bound of haversineA() over the bounding box of an edge: a = hav(dlat) + cos(lat) cos(lat2) + * hav(dlon), each term bounded by the box's smallest latitude and longitude difference to the point + * and its smallest cosine + */ + private function edgeBoxLowerBound(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $loY = min($ay, $by); + $hiY = max($ay, $by); + $dLat = $lat < $loY ? $loY - $lat : ($lat > $hiY ? $lat - $hiY : 0.0); + $loX = min($ax, $bx); + $width = max($ax, $bx) - $loX; + $dLon = 0.0; + if ($width < 360) { + $offset = fmod($lon - $loX, 360.0); + if ($offset < 0) { + $offset += 360.0; + } + if ($offset > $width) { + $dLon = min($offset - $width, 360.0 - $offset); + } + } + $cosMin = max(0.0, min(cos(deg2rad($loY)), cos(deg2rad($hiY)))); + $sLat = sin(deg2rad($dLat) / 2); + $sLon = sin(deg2rad($dLon) / 2); + return $sLat * $sLat + $cosLat * $cosMin * $sLon * $sLon; + } + + /** + * The smallest haversineA() of the points of an edge: golden-section search over t (60 steps: t to + * about 1e-13), plus the point a local flat projection picks, whichever is nearer + */ + private function edgeMinimumA(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $dx = $bx - $ax; + $dy = $by - $ay; + $at = function (float $t) use ($lon, $lat, $cosLat, $ax, $ay, $dx, $dy) { + return $this->haversineA($lon, $lat, $cosLat, $ax + $dx * $t, $ay + $dy * $t); + }; + // Local projection around the point, the longitude of the edge's start taken on the point's side + $px = fmod($ax - $lon + 540.0, 360.0) - 180.0; + $qx = ($px + $dx) * $cosLat; + $px *= $cosLat; + $py = $ay - $lat; + $qy = $by - $lat; + $ex = $qx - $px; + $ey = $qy - $py; + $len2 = $ex * $ex + $ey * $ey; + $best = $at($len2 > 0 ? max(0.0, min(1.0, -($px * $ex + $py * $ey) / $len2)) : 0.0); + $ratio = 0.6180339887498949; + $lo = 0.0; + $hi = 1.0; + $c = $hi - $ratio * ($hi - $lo); + $d = $lo + $ratio * ($hi - $lo); + $fc = $at($c); + $fd = $at($d); + for ($step = 0; $step < 60; $step++) { + if ($fc < $fd) { + $hi = $d; + $d = $c; + $fd = $fc; + $c = $hi - $ratio * ($hi - $lo); + $fc = $at($c); + } else { + $lo = $c; + $c = $d; + $fc = $fd; + $d = $lo + $ratio * ($hi - $lo); + $fd = $at($d); + } + } + return min($best, $fc, $fd); + } + + /** + * Find records within a distance from a point (distance to the nearest point of the geometry) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @param array $options ['includeDistance' => false] + * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) + */ + public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['includeDistance'], 'option', ' of withinDistance()'); + if(array_key_exists('includeDistance', $options)){ + \noneDB\Internal\Config\Configuration::optionBool('includeDistance', $options['includeDistance']); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $distanceMeters, $options){ + return $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $distanceMeters, $options); + }); + } + + private function withinDistanceUnlocked(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + $includeDistance = $options['includeDistance'] ?? false; + if ($distanceMeters < 0 || is_nan($distanceMeters) || !is_finite($lon) || !is_finite($lat)) { + return []; + } + $distanceMeters = min($distanceMeters, self::EARTH_HALF_CIRCUMFERENCE); // infinite: everything + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $this->circleToBBoxes($lon, $lat, $distanceMeters)) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && $distance <= $distanceMeters) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + } + } + usort($results, function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + return $results; + } + + /** + * Find records within a bounding box (geometries whose bounding box overlaps it). + * minLon > maxLon means the box crosses the antimeridian. + * @return array Matching records + */ + public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $minLon, $minLat, $maxLon, $maxLat){ + return $this->withinBBoxUnlocked($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); + }); + } + + private function withinBBoxUnlocked(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + if (is_nan($minLon) || is_nan($minLat) || is_nan($maxLon) || is_nan($maxLat)) { + return []; + } + // Infinite edges reach the edge of the map + $minLon = max(-180.0, min(180.0, $minLon)); + $maxLon = max(-180.0, min(180.0, $maxLon)); + $minLat = max(-90.0, min(90.0, $minLat)); + $maxLat = max(-90.0, min(90.0, $maxLat)); + if ($minLat > $maxLat) { + [$minLat, $maxLat] = [$maxLat, $minLat]; + } + $boxes = $minLon <= $maxLon + ? [[$minLon, $minLat, $maxLon, $maxLat]] + : [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon, $maxLat]]; + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + $recordMBR = $this->geometry->calculateMBR($geometry); + if ($recordMBR === null) { + continue; + } + foreach ($boxes as $box) { + if ($this->geometry->mbrOverlaps($recordMBR, $box)) { + $results[] = $record; + break; + } + } + } + return $results; + } + + /** + * Find K nearest records to a point + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] + * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + */ + public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['maxDistance', 'includeDistance'], 'option', ' of nearest()'); + if(array_key_exists('maxDistance', $options) && $options['maxDistance'] !== null && !is_int($options['maxDistance']) && !is_float($options['maxDistance'])){ + throw new \InvalidArgumentException("noneDB: 'maxDistance' must be a number of meters or null"); + } + if(array_key_exists('includeDistance', $options)){ + \noneDB\Internal\Config\Configuration::optionBool('includeDistance', $options['includeDistance']); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $k, $options){ + return $this->nearestUnlocked($dbname, $field, $lon, $lat, $k, $options); + }); + } + + /** + * True when spatial queries on the field use the R-tree (used by the query builder) + */ + public function usesSpatialIndex(string $dbname, string $field): bool{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->spatialIndexEnabled && $this->segments->dbExists($dbname) + && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + }); + } + + private function nearestUnlocked(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + $maxDistance = isset($options['maxDistance']) ? (float)$options['maxDistance'] : null; // an int or a float (nearest()) + $includeDistance = $options['includeDistance'] ?? true; + // NaN matches nothing (and NaN radii made the expanding search below loop forever) + if ($k <= 0 || !is_finite($lon) || !is_finite($lat) || ($maxDistance !== null && is_nan($maxDistance)) || !$this->segments->dbExists($dbname)) { + return []; + } + + $indexed = $this->spatialIndexEnabled && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + // Half the earth's circumference covers everything (also an infinite maxDistance) + $limit = min($maxDistance ?? self::EARTH_HALF_CIRCUMFERENCE, self::EARTH_HALF_CIRCUMFERENCE); + $results = []; + + if ($indexed && $k < $this->queryMethods->countFast($dbname)) { + // Expanding search: nothing outside radius r is closer than r. Results are ordered by the + // rounded distance, so only records rounded below round(r) are certainly in final order + // (a record just outside r can round to the same value and has to win a key tie). + $radius = 1000.0; + while (true) { + $radius = min($radius, $limit); + $found = $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $radius, ['includeDistance' => true]); + if ($radius >= $limit || (count($found) >= $k && $found[$k - 1]['_distance'] < round($radius, 2))) { + $results = $found; + break; + } + $radius *= 4; + } + } else { + // Every record by its distance; only the best k are kept (pruned as they come) + $order = function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }; + $prune = max(4 * $k, 1024); + foreach ($this->segments->listSegments($dbname) as $segment) { + foreach ($this->records->eachSegmentBatch($dbname, $segment) as $batch) { + foreach ($batch as $record) { + $geometry = $record[$field] ?? null; + if (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid']) { + continue; + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && ($maxDistance === null || $distance <= $maxDistance)) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + if (count($results) > $prune) { + usort($results, $order); + $results = array_slice($results, 0, $k); + } + } + } + } + $this->cache->evictBetweenSegments(); + } + usort($results, $order); + } + + $results = array_slice($results, 0, $k); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + return $results; + } + + /** + * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon or MultiPolygon geometry + * @return array Matching records + */ + public function withinPolygon(string $dbname, string $field, array $polygon): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $polygon){ + return $this->withinPolygonUnlocked($dbname, $field, $polygon); + }); + } + + private function withinPolygonUnlocked(string $dbname, string $field, array $polygon): array{ + $validation = $this->geoJson->validateGeoJSON($polygon); + if (!$validation['valid'] || !in_array($validation['type'], ['Polygon', 'MultiPolygon'], true)) { + return []; + } + $polygons = $validation['type'] === 'Polygon' ? [$polygon['coordinates']] : $polygon['coordinates']; + $boxes = []; + foreach ($polygons as $coords) { + $boxes[] = $this->geometry->mbrFromPoints($coords[0]); + } + + // One point inside each hole of the query: a record polygon that contains one covers part of the hole + $parts = []; + foreach ($polygons as $coords) { + $holePoints = []; + for ($h = 1; $h < count($coords); $h++) { + $point = $this->pointInsideRing($coords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + $parts[] = ['coords' => $coords, 'holePoints' => $holePoints]; + } + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + if ($this->geometryWithinParts($geometry, $parts)) { + $results[] = $record; + } + } + return $results; + } + + /** + * Within a Polygon or MultiPolygon query (its parts): a single geometry must fit one part; every + * component of a multi-part geometry must fit some part. A component spanning two adjacent + * parts doesn't fit (the union of the parts isn't computed). + */ + private function geometryWithinParts(array $geometry, array $parts): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + $components = null; + if ($type === 'MultiPoint') { + $components = array_map(function ($p) { return ['type' => 'Point', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'MultiLineString') { + $components = array_map(function ($l) { return ['type' => 'LineString', 'coordinates' => $l]; }, $coords); + } elseif ($type === 'MultiPolygon') { + $components = array_map(function ($p) { return ['type' => 'Polygon', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'GeometryCollection') { + // Members without a position don't count; a collection without any position is never within + $components = array_values(array_filter($geometry['geometries'] ?? [], function ($g) { + return is_array($g) && $this->geometry->calculateMBR($g) !== null; + })); + } + if ($components !== null) { + if (empty($components)) { + return false; + } + foreach ($components as $component) { + if (!$this->geometryWithinParts($component, $parts)) { + return false; + } + } + return true; + } + foreach ($parts as $part) { + if ($this->geometryWithinPolygon($geometry, $part['coords'], $part['holePoints'])) { + return true; + } + } + return false; + } + + /** + * A point strictly inside a ring: the middle of the first span a horizontal line between the two + * lowest distinct y values has inside the ring. Null for a degenerate ring. + */ + private function pointInsideRing(array $ring): ?array { + $ys = array_values(array_unique(array_map(function ($p) { return (float)$p[1]; }, $ring))); + sort($ys); + if (count($ys) < 2) { + return null; + } + $y = ($ys[0] + $ys[1]) / 2; + $xs = []; + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + [$x1, $y1] = [(float)$ring[$i][0], (float)$ring[$i][1]]; + [$x2, $y2] = [(float)$ring[$i + 1][0], (float)$ring[$i + 1][1]]; + if (($y1 <= $y && $y2 > $y) || ($y2 <= $y && $y1 > $y)) { + $xs[] = $x1 + ($y - $y1) / ($y2 - $y1) * ($x2 - $x1); + } + } + sort($xs); + return count($xs) >= 2 ? [($xs[0] + $xs[1]) / 2, $y] : null; + } + + /** + * Check if a geometry lies entirely within a polygon. The polygon is closed (its boundary, also a + * hole's boundary, belongs to it, as in withinBBox). Every vertex is inside, no edge crosses a ring of + * the polygon, and every piece of an edge between polygon vertices on it is inside; a record polygon + * must also not contain a point inside one of the polygon's holes ($holePoints). + */ + private function geometryWithinPolygon(array $geometry, array $polygonCoords, ?array $holePoints = null): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + if ($holePoints === null) { + $holePoints = []; + for ($h = 1; $h < count($polygonCoords); $h++) { + $point = $this->pointInsideRing($polygonCoords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + } + + switch ($type) { + case 'Point': + return $this->pointInPolygonClosed((float)$coords[0], (float)$coords[1], $polygonCoords); + case 'MultiPoint': + foreach ($coords as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + return true; + case 'LineString': + return $this->pathWithinPolygon($coords, $polygonCoords); + case 'MultiLineString': + foreach ($coords as $line) { + if (!$this->pathWithinPolygon($line, $polygonCoords)) { + return false; + } + } + return true; + case 'Polygon': + return $this->polygonWithinPolygon($coords, $polygonCoords, $holePoints); + case 'MultiPolygon': + foreach ($coords as $poly) { + if (!$this->polygonWithinPolygon($poly, $polygonCoords, $holePoints)) { + return false; + } + } + return true; + case 'GeometryCollection': + // Members without a position don't count; a collection without any position is never within + $located = false; + foreach ($geometry['geometries'] ?? [] as $g) { + if (is_array($g) && $this->geometry->calculateMBR($g) === null) { + continue; + } + if (!$this->geometryWithinPolygon($g, $polygonCoords)) { + return false; + } + $located = true; + } + return $located; + } + return false; + } + + /** + * A polygon (record) within a polygon (query): all its rings within, and no point inside one of the + * query's holes inside it (a record around a hole has its whole boundary inside the query) + */ + private function polygonWithinPolygon(array $rings, array $polygonCoords, array $holePoints): bool { + foreach ($rings as $ring) { + if (!$this->pathWithinPolygon($ring, $polygonCoords)) { + return false; + } + } + foreach ($holePoints as [$x, $y]) { + if ($this->geometry->pointInPolygon($x, $y, $rings)) { + return false; + } + } + return true; + } + + /** + * Point inside a polygon or on its boundary (holes excluded, their boundary included) + */ + private function pointInPolygonClosed(float $lon, float $lat, array $polygonCoords): bool { + if (!$this->geoJson->pointOnRing($lon, $lat, $polygonCoords[0]) && !$this->geometry->pointInRing($lon, $lat, $polygonCoords[0])) { + return false; + } + for ($i = 1; $i < count($polygonCoords); $i++) { + if ($this->geometry->pointInRing($lon, $lat, $polygonCoords[$i]) && !$this->geoJson->pointOnRing($lon, $lat, $polygonCoords[$i])) { + return false; + } + } + return true; + } + + /** + * A path within the closed polygon: every vertex inside, no segment properly crossing a ring, and + * every piece of a segment between polygon vertices lying on it inside (a segment can run along a + * boundary or through a hole between two of its vertices) + */ + private function pathWithinPolygon(array $points, array $polygonCoords): bool { + foreach ($points as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + $n = count($points); + for ($i = 0; $i < $n - 1; $i++) { + $a = [(float)$points[$i][0], (float)$points[$i][1]]; + $b = [(float)$points[$i + 1][0], (float)$points[$i + 1][1]]; + $ts = [0.0, 1.0]; + $alongX = abs($b[0] - $a[0]) >= abs($b[1] - $a[1]); + foreach ($polygonCoords as $ring) { + $m = count($ring); + for ($j = 0; $j < $m - 1; $j++) { + if ($this->segmentsCrossProperly($a, $b, $ring[$j], $ring[$j + 1])) { + return false; + } + $v = [(float)$ring[$j][0], (float)$ring[$j][1]]; + if ($this->geometry->crossProductDirection($a, $b, $v) == 0 && $this->geometry->onSegment($a, $b, $v) && $a != $b) { + $ts[] = $alongX ? ($v[0] - $a[0]) / ($b[0] - $a[0]) : ($v[1] - $a[1]) / ($b[1] - $a[1]); + } + } + } + if (count($ts) > 2) { + sort($ts); + for ($k = 0; $k < count($ts) - 1; $k++) { + if ($ts[$k + 1] - $ts[$k] <= 0) { + continue; + } + $t = ($ts[$k] + $ts[$k + 1]) / 2; + if (!$this->pointInPolygonClosed($a[0] + $t * ($b[0] - $a[0]), $a[1] + $t * ($b[1] - $a[1]), $polygonCoords)) { + return false; + } + } + } elseif (!$this->pointInPolygonClosed(($a[0] + $b[0]) / 2, ($a[1] + $b[1]) / 2, $polygonCoords)) { + return false; // e.g. a chord between two boundary vertices across a concave notch + } + } + return true; + } + + /** + * True when two segments cross at a single interior point (touching endpoints don't count) + */ + private function segmentsCrossProperly(array $p1, array $p2, array $p3, array $p4): bool { + $d1 = $this->geometry->crossProductDirection($p3, $p4, $p1); + $d2 = $this->geometry->crossProductDirection($p3, $p4, $p2); + $d3 = $this->geometry->crossProductDirection($p1, $p2, $p3); + $d4 = $this->geometry->crossProductDirection($p1, $p2, $p4); + return (($d1 > 0 && $d2 < 0) || ($d1 < 0 && $d2 > 0)) && (($d3 > 0 && $d4 < 0) || ($d3 < 0 && $d4 > 0)); + } + + public const EARTH_HALF_CIRCUMFERENCE = 20040000.0; // meters: no two points are farther apart +} +} +namespace noneDB\Internal\Query { + +use function array_key_exists; + +/** + * @internal + */ +final class Utility { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $fieldIndexEnabled; + private $cache; + private $fieldIndex; + private $findFilter; + private $layout; + private $locks; + private $planner; + private $records; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->findFilter, $this->layout, $this->locks, $this->planner, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->findFilter = null; + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // UTILITY METHODS + // ========================================== + + /** + * Get first matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function first($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->firstUnlocked($dbname, $filter); + }); + } + + private function firstUnlocked($dbname, $filter = 0){ + $first = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use (&$first){ $first = reset($records); return false; }); + return $first; + } + + /** + * Get last matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function last($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->lastUnlocked($dbname, $filter); + }); + } + + private function lastUnlocked($dbname, $filter = 0){ + $last = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use (&$last){ $last = end($records); }, true); + return $last; + } + + /** + * Check if records exist matching filter + * @param string $dbname + * @param mixed $filter + * @return bool + */ + public function exists($dbname, $filter){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->existsUnlocked($dbname, $filter); + }); + } + + private function existsUnlocked($dbname, $filter){ + $found = false; + $this->planner->eachMatch($dbname, $filter, function() use (&$found){ $found = true; return false; }); + return $found; + } + + /** + * Range query (min <= value <= max) + * @param string $dbname + * @param string $field + * @param mixed $min + * @param mixed $max + * @param mixed $filter Additional filter + * @return array|false + */ + public function between($dbname, $field, $min, $max, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $min, $max, $filter){ + return $this->betweenUnlocked($dbname, $field, $min, $max, $filter); + }); + } + + private function betweenUnlocked($dbname, $field, $min, $max, $filter = 0){ + // Same comparison as query()->between(): numbers (and numeric strings) numerically, strings + // lexically; null, booleans, arrays and number-vs-text are never in range + $result = []; + $collect = function(array $records) use ($field, $min, $max, &$result){ + foreach($records as $record){ + if(!array_key_exists($field, $record)){ + continue; + } + $lo = \noneDB\Internal\Query\Values::compareValues($record[$field], $min); + $hi = \noneDB\Internal\Query\Values::compareValues($record[$field], $max); + if($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0){ + $result[] = $record; + } + } + }; + // An indexed field: only the records whose value the field index puts in range are read + if(!$this->eachRangeCandidate($dbname, $field, ['$gte' => $min, '$lte' => $max], $filter, $collect) + && !$this->planner->eachMatch($dbname, $filter, $collect, true)){ + return false; + } + return $result; + } + + /** + * Visit the records matching a filter parameter whose field value may satisfy the range conditions, read + * through the field index of every segment (a superset: the visitor checks the range), in key order, one + * segment at a time: $visit([key => record]) gets a segment's (never none). False when it can't: no index + * of the field, a key filter decides, an invalid filter or no database (the caller scans). + * @return bool + */ + private function eachRangeCandidate($dbname, $field, array $conditions, $filter, callable $visit){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null || array_key_exists('key', $filters) || !$this->segments->dbExists($dbname) || !$this->fieldIndexEnabled + || !$this->fieldIndex->isFieldIndexed($dbname, $field)){ + return false; + } + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($this->segments->listSegments($dbname) as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $keys = $this->fieldIndex->fieldIndexRangeCandidates($dbname, $segment, $field, $conditions); + if($keys === null || !empty($keys)){ + foreach($this->records->eachSegmentBatch($dbname, $segment, $keys) as $batch){ + $matches = []; + foreach($batch as $key => $record){ + if($this->findFilter->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + if(!empty($matches)){ + $visit($matches); + } + } + } + } + return true; + } +} +} +namespace noneDB\Internal\Query { + +use function bin2hex, count, function_exists, is_array, is_bool, is_float, is_int, is_nan, is_numeric, + is_object, is_string, json_encode, mb_convert_case, pack, preg_last_error, preg_last_error_msg, preg_match, + preg_quote, spl_object_id, str_replace, strcmp, strlen, strncmp, strpos, substr; +use const MB_CASE_FOLD, PREG_BAD_UTF8_ERROR, PREG_BAD_UTF8_OFFSET_ERROR, PREG_NO_ERROR; + +/** + * @internal + */ +final class Values { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + + /** + * valueKey() for the query builder + * @return \Closure + */ + public function valueKeyFunction(){ + return \Closure::fromCallable([self::class, 'valueKey']); + } + + /** + * Add the records' values of a field to the distinct values seen so far (null excluded), in first-seen + * order, same identity as in_array($value, $values, true), in linear time ($seen: a hash set of their + * valueKey()s): distinct() batch by batch, the records never held together + */ + public static function collectUnique(array $records, $field, array &$seen, array &$values){ + foreach($records as $record){ + if(isset($record[$field])){ + $id = self::valueKey($record[$field]); + if(!isset($seen[$id])){ + $seen[$id] = true; + $values[] = $record[$field]; + } + } + } + } + + /** + * A string that is equal for two values exactly when they are identical (===): typed, -0.0 is 0.0, + * arrays by their typed keys and values in order (lengths make the encoding unambiguous). + * Floats by their bits, so neither precision nor locale settings matter. + */ + public static function valueKey($v){ + if($v === null) return 'n'; + if($v === true) return 't'; + if($v === false) return 'f'; + if(is_int($v)) return 'i' . $v; + if(is_float($v)) return 'd' . bin2hex(pack('E', $v + 0.0)); + if(is_string($v)) return 's' . $v; + if(is_array($v)){ + $key = 'a' . count($v) . ':'; + foreach($v as $k => $item){ + $kk = (is_int($k) ? 'i' : 's') . $k; + $id = self::valueKey($item); + $key .= strlen($kk) . ':' . $kk . strlen($id) . ':' . $id; + } + return $key; + } + return is_object($v) ? 'o' . spl_object_id($v) : 'r' . (int)$v; // identical only to itself + } + + /** + * Total order used by sort(), min() and max() (and the query builder): missing/null < booleans < + * numbers < numeric strings (by their number) < other strings (byte order) < arrays. + * @return int + */ + public static function compareSortValues($a, $b){ + return self::sortKey($a) <=> self::sortKey($b); + } + + /** + * Sort key of a value: [kind, comparable value]. Two keys compare with <=> in the order above + * (kind first, then the value; arrays by their JSON), so sorts compute each key once instead of + * classifying both values in every comparison. + * @return array + */ + public static function sortKey($v){ + if($v === null) return [0, 0]; + if(is_bool($v)) return [1, $v]; + if(is_int($v) || is_float($v)) return [2, $v]; + if(is_string($v)){ + // Numeric strings (is_numeric()) never meet other strings, so <=> never mixes numeric and byte + // comparison (mixed, "9" < "10" < "1a" < "9" would be a cycle). + return is_numeric($v) ? [3, $v + 0] : [4, $v]; + } + return [5, (string)json_encode($v)]; + } + + /** + * sortKey() for the query builder + * @return \Closure + */ + public function sortKeyFunction(){ + return \Closure::fromCallable([self::class, 'sortKey']); + } + /** + * True when mbstring can be used for Unicode case folding (like(), search()) + * @internal used by noneDBQuery; not part of the public API + */ + public static function hasMbstring(): bool { + return function_exists('mb_convert_case'); + } + /** + * Compare two values for ordering operators ($gt, between, ...). + * Numbers (and numeric strings) compare numerically, other strings lexically; anything else + * (null, booleans, arrays, number vs. non-numeric string) is not comparable. + * @return int|null + */ + public static function compareValues($a, $b): ?int { + if ($a === null || $b === null || is_bool($a) || is_bool($b) || is_array($a) || is_array($b) || is_object($a) || is_object($b)) { + return null; + } + // NAN compares as greater than everything with <=> ($gt NAN matched every number) + if ((is_float($a) && is_nan($a)) || (is_float($b) && is_nan($b))) { + return null; + } + $numA = self::numberOf($a); + $numB = self::numberOf($b); + if ($numA !== null && $numB !== null) { + return $numA <=> $numB; + } + if (is_string($a) && is_string($b)) { + return strcmp($a, $b) <=> 0; + } + return null; + } + /** + * The number of an int, float or numeric string (is_numeric(): leading and trailing whitespace + * allowed), else null. The sort order and range indexes use the same rule. + * @return int|float|null + */ + public static function numberOf($value) { + if (is_int($value) || is_float($value)) { + return $value; + } + if (is_string($value)) { + return is_numeric($value) ? $value + 0 : null; + } + return null; + } + /** + * Run preg_match, turning compile errors and backtrack-limit failures into exceptions + */ + public static function regexMatch(string $regex, string $subject): bool { + $result = @preg_match($regex, $subject); + if ($result === false) { + $error = preg_last_error(); + if ($error === PREG_BAD_UTF8_ERROR || $error === PREG_BAD_UTF8_OFFSET_ERROR) { + return false; // subject isn't UTF-8: can't match a UTF-8 pattern + } + if ($error === PREG_NO_ERROR) { + throw new \InvalidArgumentException('noneDB: invalid regular expression'); + } + throw new \RuntimeException('noneDB: regular expression failed (' . preg_last_error_msg() . ')'); + } + return $result === 1; + } + /** + * Case folding for matching (Unicode full folding: "ß" and "ẞ" are "ss", "ſ" is "s", "ς" is "σ"). + * "İ" folds to "i" + combining dot (U+0307); the dot is dropped so "İzmir", "IZMIR" and "izmir" all + * match each other. Language independent: "I" and "ı" don't match. + */ + public static function foldCase(string $s): string { + return str_replace("\u{0307}", '', mb_convert_case($s, MB_CASE_FOLD, 'UTF-8')); + } + /** + * Compile a like() pattern: a leading "^" anchors the start, a trailing "$" the end, everything else + * is literal and case-insensitive. + * @return array + */ + public static function compileLike(string $pattern): array { + $start = $pattern !== '' && $pattern[0] === '^'; + $end = strlen($pattern) > ($start ? 1 : 0) && substr($pattern, -1) === '$'; + $body = (string)substr($pattern, $start ? 1 : 0, strlen($pattern) - ($start ? 1 : 0) - ($end ? 1 : 0)); + $utf8 = preg_match('//u', $body) === 1; + return [ + 'start' => $start, 'end' => $end, + // Folded body for UTF-8 subjects (null: no mbstring, or the pattern isn't UTF-8) + 'folded' => $utf8 && self::hasMbstring() ? self::foldCase($body) : null, + // Otherwise a regex (PCRE /iu: simple case folding) + // ("D": the end anchor is the end of the value, not also before a final newline) + 'regex' => '~' . ($start ? '^' : '') . preg_quote($body, '~') . ($end ? '$' : '') . '~D' . ($utf8 ? 'iu' : 'i'), + ]; + } + /** + * Match a compiled like() pattern + */ + public static function likeMatch(array $like, string $subject): bool { + if ($like['folded'] === null || preg_match('//u', $subject) !== 1) { + return self::regexMatch($like['regex'], $subject); + } + $t = self::foldCase($subject); + $b = $like['folded']; + if ($like['start'] && $like['end']) { + return $t === $b; + } + if ($like['start']) { + return strncmp($t, $b, strlen($b)) === 0; + } + if ($like['end']) { + return $b === '' || (strlen($t) >= strlen($b) && substr($t, -strlen($b)) === $b); + } + return $b === '' || strpos($t, $b) !== false; + } +} +} +namespace noneDB\Internal\Recovery { + +use function array_filter, array_keys, array_map, array_values, basename, bin2hex, clearstatcache, fclose, + fflush, file_exists, filesize, fopen, fread, fseek, fwrite, hash_file, hash_final, hash_init, hash_update, + in_array, is_dir, is_file, min, random_bytes, rename, rtrim, strlen, substr, unlink; + +/** + * @internal + */ +final class Recovery { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $durable; + private $lockRegistry; + private $noWrites; + private $cache; + private $checkpoints; + private $io; + private $layout; + private $locks; + private $recoveryLog; + private $segments; + private $shardMeta; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->recoveryLog, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->durable = &$this->k->durable; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recoveryLog = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + + /** + * Recovery of a database a writer left half-updated (journal j/J; caller holds the exclusive lock): + * key indexes are checked against the data (field and spatial indexes follow by themselves), temp + * files of interrupted atomic writes are removed and the shard counters recounted from the data + */ + public function recoverPendingIndexes($dbname, $path){ + $pending = $this->lockRegistry[$path]['flags']['j'] ?? []; + $known = $this->segments->listSegments($dbname); + $segments = isset($pending['*']) ? $known : array_map(function($segment){ + return $segment === 'main' ? null : (int)substr($segment, 1); + }, array_keys($pending)); + $segments = array_values(array_filter($segments, function($shardId) use ($known){ + return in_array($shardId, $known, true); + })); + foreach($segments as $shardId){ + $this->checkpoints->readJsonlIndex($dbname, $shardId); + $this->cache->evictBetweenSegments(); + } + $home = $this->layout->dbHome($dbname); + $gen = $this->layout->genDir($dbname); + foreach([$home, $home . 'recovery/', $gen, $gen . 'segments/', $gen . 'indexes/', $gen . 'spatial/', $gen . 'buffer/'] as $dir){ + $this->removeStaleTemp($dir); + } + if($this->segments->isSharded($dbname)){ + // A write may have died between its data and its counters + $this->shardMeta->recountMeta($dbname); + } + unset($this->lockRegistry[$path]['flags']['j']); + $this->locks->saveLockState($path); + } + + /** + * Remove bytes [$from, $to) at the end of a data or buffer file, keeping them in a quarantine file + * first (exclusive lock). Fails closed: if the bytes can't be stored and verified, nothing is cut + * and the write that needed the cut is refused. + * @throws \RuntimeException + */ + public function quarantineAndCut($dbname, $shardId, $path, $from, $to, $action = 'cut-tail'){ + $quarantine = $this->quarantineBytes($dbname, $shardId, $path, $from, $to); + if(!$this->io->fsTruncate($path, $from)){ + throw new \RuntimeException("noneDB: could not cut the torn end of a file of database '{$dbname}'"); + } + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => $action, 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']], $quarantine['id']); + return $quarantine; + } + + /** + * Copy bytes [$from, $to) of a file into recovery/q.{segment}.{offset}.{length}.{sha256}.bin + * (segment: main, s{N}, or buffer for a buffer file: $shardId 'b') + * and verify the copy. An existing file with the same name is reused only if its bytes match. + * @return array ['id' => string, 'file' => string, 'durable' => true|false|'unknown'] + * @throws \RuntimeException + */ + public function quarantineBytes($dbname, $shardId, $path, $from, $to){ + $length = $to - $from; + $seg = $shardId === 'b' ? 'buffer' : \noneDB\Internal\Storage\Layout::segmentName($shardId); + $src = @fopen($path, 'rb'); + if($src === false){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + $dir = $this->layout->dbHome($dbname) . 'recovery/'; + if(!is_dir($dir)){ + $this->io->fsMkdir($dir); + } + $tmp = $dir . 'q.tmp' . bin2hex(random_bytes(4)); + $this->locks->noteWrite($tmp); + $out = null; + try { + $out = @fopen($tmp, 'xb'); + if($out === false){ + throw new \RuntimeException("noneDB: could not create a quarantine file for database '{$dbname}'"); + } + $this->io->setPermissions($tmp); + $hash = hash_init('sha256'); + fseek($src, $from); + $left = $length; + while($left > 0){ + $chunk = (string)fread($src, min(65536, $left)); + if($chunk === ''){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + hash_update($hash, $chunk); + if(fwrite($out, $chunk) !== strlen($chunk)){ + throw new \RuntimeException("noneDB: could not write a quarantine file for database '{$dbname}'"); + } + $left -= strlen($chunk); + } + $sha = hash_final($hash); + // Synced in both modes (DURABILITY.md §3): required in durable mode, else reported in the log + $fileSynced = fflush($out) ? $this->sync->syncHandle($out, $tmp) : false; + fclose($out); + $out = null; + if($this->durable){ + $this->sync->noteRequired('file', $fileSynced === true); + if($fileSynced !== true){ + throw $this->sync->failure('file', $tmp, false); + } + } + + $id = "{$seg}.{$from}.{$length}.{$sha}"; + $name = "q.{$id}.bin"; + $target = $dir . $name; + clearstatcache(true, $target); + if(file_exists($target)){ + @unlink($tmp); // the same event again (e.g. the cut didn't happen last time): keep the first copy + $fileSynced = null; + } elseif(!@rename($tmp, $target)){ + throw new \RuntimeException("noneDB: could not publish a quarantine file for database '{$dbname}'"); + } + $dirSynced = $this->sync->syncDirectory(rtrim($dir, '/')); + if($this->durable){ + // The copy must be durable before the source bytes are cut + $this->sync->noteRequired('dir', $dirSynced === true); + if($dirSynced !== true){ + throw $this->sync->failure('dir', $dir, false); + } + } + if(@filesize($target) !== $length || hash_file('sha256', $target) !== $sha){ + throw new \RuntimeException("noneDB: quarantine file {$name} doesn't match the bytes it should hold"); + } + if($fileSynced === false || $dirSynced === false){ + $durable = false; + } elseif($dirSynced === null){ + $durable = 'unknown'; + } else { + $durable = true; + } + return ['id' => $id, 'file' => $name, 'durable' => $durable]; + } finally { + fclose($src); + if($out !== null && $out !== false){ + fclose($out); + } + if(is_file($tmp)){ + @unlink($tmp); + } + } + } + + /** + * Delete temp files ("*.tmp" + 8 hex digits) starting with $prefix: what a writer that died left + * behind. Exclusive lock only: no other writer can be in the middle of an atomic write. + */ + public function removeStaleTemp($prefix){ + if($this->noWrites > 0){ + return; + } + foreach($this->segments->listFilesWithPrefix($prefix, '/\.tmp[0-9a-f]{8}$/') as $file){ + @unlink($file); + } + } +} +} +namespace noneDB\Internal\Recovery { + +use function array_merge, array_slice, clearstatcache, dirname, fclose, fgets, file_get_contents, filesize, + fopen, is_array, is_dir, json_decode, json_encode, max, strpos, substr, time; +use const LOCK_EX; + +/** + * @internal + */ +final class RecoveryLog { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $lockRegistry; + private $metrics; + private $noWrites; + private $pendingRecoveryEvents; + private $io; + private $layout; + private $locks; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->io, $this->layout, $this->locks); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->pendingRecoveryEvents = &$this->rt->pendingRecoveryEvents; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->k = null; + $this->rt = null; + } + + + private function getRecoveryLogPath($dbname){ + return $this->layout->dbHome($dbname) . 'recovery/log.jsonl'; + } + + /** + * Append an entry to the database's recovery log (caller holds the exclusive lock). Best effort: a + * failure to log never fails the operation. Past 1 MiB the log moves to log.1.jsonl (replacing it). + */ + public function logRecoveryEvent($dbname, array $event, $dedupId = null){ + try { + $path = $this->getRecoveryLogPath($dbname); + if($dedupId !== null){ + $id = '"id":' . json_encode($dedupId); + foreach([substr($path, 0, -6) . '.1.jsonl', $path] as $file){ + $content = @file_get_contents($file); + // the id is followed by the next member or ends the entry + if($content !== false && (strpos($content, $id . ',') !== false || strpos($content, $id . '}') !== false)){ + return; // already logged + } + } + } + clearstatcache(true, $path); + if(!is_dir(dirname($path))){ + $this->io->fsMkdir(dirname($path)); + } + if(@filesize($path) > 1048576){ + $this->io->fsRename($path, substr($path, 0, -6) . '.1.jsonl'); + } + $line = $this->io->encodeJson(['t' => time()] + $event); + if($line !== false && $this->io->appendToFile($path, $line . "\n") !== false){ + $this->metrics['recoveryEvents']++; + $this->metrics['recoveryActions'][$event['action']] = ($this->metrics['recoveryActions'][$event['action']] ?? 0) + 1; + } + } catch(\Throwable $ignored){ + } + } + + /** + * A recovery decision: logged at once under the exclusive lock, else kept in memory and logged by the + * next exclusive session of this process (readers never write); getRecoveryLog() shows both + */ + public function reportRecoveryEvent($dbname, array $event){ + $lock = $this->locks->getLockPath($dbname); + if($this->noWrites === 0 && isset($this->lockRegistry[$lock]) && $this->lockRegistry[$lock]['mode'] === LOCK_EX){ + $this->logRecoveryEvent($dbname, $event, $event['id']); + } else { + $this->pendingRecoveryEvents[$lock][$event['id']] = ['t' => time()] + $event; + } + } + + public function getRecoveryLogEntries($dbname){ + $path = $this->getRecoveryLogPath($dbname); + $entries = []; + foreach([substr($path, 0, -6) . '.1.jsonl', $path] as $file){ + $fp = @fopen($file, 'rb'); + if($fp === false){ + continue; + } + while(($line = fgets($fp)) !== false){ + $entry = substr($line, -1) === "\n" ? json_decode($line, true) : null; + if(is_array($entry)){ + $entries[] = $entry; + } + } + fclose($fp); + } + return $entries; + } + + public function getRecoveryLog($dbname, $limit = 100){ + $dbname = $this->layout->normalizeDbName($dbname); + $read = function() use ($dbname, $limit){ + // What readers of this process found and no write session logged yet: "persisted" => false + $pending = []; + foreach($this->pendingRecoveryEvents[$this->locks->getLockPath($dbname)] ?? [] as $event){ + $pending[] = $event + ['persisted' => false]; + } + $limit = max(0, (int)$limit); + return $limit === 0 ? [] : array_slice(array_merge($this->getRecoveryLogEntries($dbname), $pending), -$limit); + }; + return $this->locks->withLock($dbname, false, $read); + } +} +} +namespace noneDB\Internal\Recovery { + +use function basename, clearstatcache, fclose, fopen, fstat, ftruncate; + +/** + * @internal + */ +final class RollbackScope { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $durable; + private $lockRegistry; + private $noWrites; + private $pendingRecoveryEvents; + private $cache; + private $locks; + private $recovery; + private $recoveryLog; + private $segments; + private $shardMeta; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->locks, $this->recovery, $this->recoveryLog, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->durable = &$this->k->durable; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + $this->pendingRecoveryEvents = &$this->rt->pendingRecoveryEvents; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Run the outermost exclusive call of a database inside a rollback scope (see above), then the jobs + * deferred until its commit, each in a scope of its own. The call is one write operation: how it ended + * goes to the durability report (docs/DURABILITY.md §2): committed when $fn returned (every required + * sync succeeded, or it would have thrown), unknown for OUTCOME_UNKNOWN, rolled back otherwise. + * @param string $op Name of the operation; null for a deferred job (not an operation of its own) + */ + public function runWriteScope($dbname, $path, callable $fn, $op = 'write'){ + if($op !== null){ + if(!empty($this->pendingRecoveryEvents[$path]) && $this->noWrites === 0){ + // What readers of this process found (they don't write): logged once, deduplicated by id + $events = $this->pendingRecoveryEvents[$path]; + unset($this->pendingRecoveryEvents[$path]); + foreach($events as $id => $event){ + $this->recoveryLog->logRecoveryEvent($dbname, $event, $id); + } + } + $this->lockRegistry[$path]['wrote'] = false; + if($this->durable){ + $this->lockRegistry[$path]['syncs'] = []; // the syncs durable mode requires (Sync::noteRequired) + } + } + $this->lockRegistry[$path]['scope'] = ['marks' => [], 'deferred' => [], 'independent' => false]; + try { + $result = $fn(); + } catch(\Throwable $e){ + try { + $this->rollbackScope($dbname, $path, $e); // rethrows + } catch(\Throwable $thrown){ + if($op !== null){ + $this->finishOperation($dbname, $path, $op, $thrown); + } + throw $thrown; + } + } + $deferred = $this->lockRegistry[$path]['scope']['deferred']; + $this->lockRegistry[$path]['scope'] = null; + if($op !== null){ + $this->finishOperation($dbname, $path, $op, null); + } + foreach($deferred as $job => $work){ + try { + $this->runWriteScope($dbname, $path, function() use ($dbname, $job, $work){ + return $this->runIndependent($dbname, $job, $work); + }, null); + } catch(\Throwable $e){ + // The operation itself is committed; the job runs again on a later operation + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'deferred-failed', 'job' => $job, 'error' => $e->getMessage()]); + break; + } + $this->cache->evictBetweenSegments(); // a job works on one segment (compaction of a large write's segments) + } + return $result; + } + + /** + * The outcome of a write operation for getDurability(); an operation that wrote nothing has none + */ + private function finishOperation($dbname, $path, $op, ?\Throwable $e){ + $syncs = []; + if(isset($this->lockRegistry[$path]['syncs'])){ + $syncs = $this->lockRegistry[$path]['syncs']; + unset($this->lockRegistry[$path]['syncs']); + } + if(!$this->lockRegistry[$path]['wrote']){ + return; + } + if($e === null){ + $this->sync->finishOperation($op, $dbname, $syncs, 'committed'); + } else { + $this->sync->finishOperation($op, $dbname, $syncs, $e->getCode() === \noneDB::OUTCOME_UNKNOWN ? 'unknown' : 'rolled_back', $e->getMessage()); + } + } + + /** + * Undo the appends of a failed scope and rethrow; OUTCOME_UNKNOWN when a file can't be cut back + * (in durable mode: or its cut can't be synced), with the journal "J" so the next session recovers + */ + private function rollbackScope($dbname, $path, \Throwable $e){ + $marks = $this->lockRegistry[$path]['scope']['marks']; + $this->lockRegistry[$path]['scope'] = null; + $this->lockRegistry[$path]['rolledBack'] = true; + $failed = []; + foreach($marks as $file => $mark){ + try { + if(!$this->truncateBack($file, $mark)){ + $failed[] = $file; + } + } catch(\Throwable $t){ + $failed[] = $file; + } + } + $this->cache->invalidateDbCaches($dbname); + if(!empty($failed) || $e->getCode() === \noneDB::OUTCOME_UNKNOWN){ + // What is on disk is uncertain: the next session recovers the whole database ("J") + $this->lockRegistry[$path]['flags']['j']['*'] = true; + try { + $this->locks->saveLockState($path); + } catch(\Throwable $ignored){ + } + if(!empty($failed)){ + throw new \RuntimeException("noneDB: a write to database '{$dbname}' failed and could not be undone: the outcome is unknown, the database is recovered on its next access", \noneDB::OUTCOME_UNKNOWN, $e); + } + } + if(!empty($marks) && $this->segments->isSharded($dbname)){ + try { + $this->shardMeta->recountMeta($dbname); // keys reserved by the failed write stay unused + } catch(\Throwable $ignored){ + // the journal entry stays, recovery recounts + } + } + throw $e; + } + + /** + * Cut a file back to the size recorded in a mark, if it is still the same file + */ + private function truncateBack($file, array $mark){ + list($size, $dev, $ino) = $mark; + $this->locks->noteWrite($file); + $fp = @fopen($file, 'r+b'); + if($fp === false){ + return false; + } + try { + $stat = fstat($fp); + if($stat === false || ($ino && ($stat['ino'] !== $ino || $stat['dev'] !== $dev))){ + return false; + } + return $stat['size'] <= $size || (ftruncate($fp, $size) && $this->sync->syncUndo($fp, $file)); + } finally { + fclose($fp); + clearstatcache(true, $file); + } + } + + /** + * Record the size a file had before the scope's first append to it + */ + public function markScopeFile($dbname, $file, $size, $fp){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->lockRegistry[$lock]['scope']) && !isset($this->lockRegistry[$lock]['scope']['marks'][$file])){ + $stat = fstat($fp); + $this->lockRegistry[$lock]['scope']['marks'][$file] = [$size, $stat ? $stat['dev'] : 0, $stat ? $stat['ino'] : 0]; + } + } + + /** + * Run work that must not be undone by its caller's failure (buffer flush, the switch to shards): in place while the + * scope has no appends, else deferred until the scope committed (once per $job). Returns null if deferred. + */ + public function runIndependent($dbname, $job, callable $work){ + $path = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]['scope'])){ + return $work(); + } + if(!empty($this->lockRegistry[$path]['scope']['marks'])){ + if(!isset($this->lockRegistry[$path]['scope']['deferred'][$job])){ + $this->lockRegistry[$path]['scope']['deferred'][$job] = $work; + } + return null; + } + $was = $this->lockRegistry[$path]['scope']['independent']; + $this->lockRegistry[$path]['scope']['independent'] = true; + try { + $result = $work(); + $this->commitIndependentStep($dbname); + return $result; + } finally { + if(isset($this->lockRegistry[$path]['scope'])){ + $this->lockRegistry[$path]['scope']['independent'] = $was; + } + } + } + + /** + * Work that replaces or removes data files (dropDB, compact, convertToSharded) can't be undone by a rollback: + * inside a write of the same database that hasn't committed (a nested call from the caller's own code, which + * a value's jsonSerialize() can make before the first append) it is refused before it touches any file + * @throws \LogicException + */ + public function refuseInUncommittedWrite($dbname, $operation){ + $entry = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $scope = $entry['scope'] ?? null; + if($scope !== null && (!empty($scope['marks']) || $entry['depth'] > 1) && empty($scope['independent'])){ + throw new \LogicException("noneDB: {$operation}() can't run inside a write to database '{$dbname}' that hasn't committed yet (it replaces or deletes data files, which the write's rollback couldn't undo); call it after that write"); + } + } + + /** + * Before a file is replaced, renamed or removed: never one that holds appends a rollback may have to cut + * back (whatever path asks for it) + * @throws \LogicException + */ + public function refuseUncommittedFile($file, $operation){ + foreach($this->lockRegistry as $entry){ + if(isset($entry['scope']['marks'][$file]) && empty($entry['scope']['independent'])){ + throw new \LogicException("noneDB: {$operation} of " . basename($file) . " refused: it holds appends of database '{$entry['dbname']}' that aren't committed yet"); + } + } + } + + /** + * Inside independent work: its appends so far are final (the scope had none before it started) + */ + public function commitIndependentStep($dbname){ + $path = $this->locks->getLockPath($dbname); + if(!empty($this->lockRegistry[$path]['scope']['independent'])){ + $this->lockRegistry[$path]['scope']['marks'] = []; + } + } +} +} +namespace noneDB\Internal\Sharding { + +use function array_flip, array_intersect_key, bin2hex, count, fclose, fgets, fopen, intdiv, max, min, random_bytes, + range, strlen, substr; + +/** + * @internal + */ +final class LayoutSwitch { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $buffer; + private $cache; + private $checkpoints; + private $fieldIndex; + private $io; + private $layout; + private $locks; + private $records; + private $recovery; + private $scope; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->records, $this->recovery, $this->scope, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recovery = null; + $this->scope = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Convert a single-layout database to the sharded layout (keys preserved, indexes rebuilt per shard): + * a generation switch (STORAGE_FORMAT.md §5.2). The new generation is built and verified completely + * before CURRENT names it; an interruption leaves the old generation or the new one, never a mix. + * @param string $dbname + * @return bool + */ + public function convertToSharded($dbname){ + return $this->locks->withLock($dbname, true, function() use ($dbname){ + if($this->segments->isSharded($dbname)){ + return true; + } + $singlePath = $this->segments->getDataPath($dbname); + if(!$this->cache->cachedFileExists($singlePath)){ + return false; + } + $this->scope->refuseUncommittedFile($singlePath, 'The switch to shards'); + $this->buffer->flushPendingBuffers($dbname); + + $index = $this->checkpoints->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + $live = \noneDB\Internal\Storage\KeyIndex::count($index); + $maxKey = \noneDB\Internal\Storage\KeyIndex::maxKey($index); + $size = $this->segments->shardSizeOf($dbname); + $nextKey = max((int)$index['n'], $maxKey + 1); + $numShards = max(1, intdiv(max($nextKey, 1) - 1, $size) + 1); + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $format = $this->layout->readFormatRecord($dbname); + unset($index); + + $this->layout->switchGeneration($dbname, function($dir, $generation) use ($dbname, $numShards, $size, $nextKey, $maxKey, $live, $manifest, $format){ + $this->layout->writeGenerationMetadata($dbname, $generation, $this->layout->identityOf($format), 'sharded'); + $meta = \noneDB\Internal\Sharding\ShardMeta::newShardMeta($size, $nextKey); + $meta['totalRecords'] = $live; + $written = 0; + // One shard at a time: its records read from the single segment, written with its checkpoint + for($shardId = 0; $shardId < $numShards; $shardId++){ + $base = $shardId * $size; + $limit = min($nextKey, $base + $size); + $count = 0; + $shardIndex = $this->checkpoints->newJsonlIndex(); + $shardIndex['n'] = $limit - $base; + $buffer = $this->io->withExactFloats(function() use ($dbname, $base, $limit, &$shardIndex, &$count){ + $buffer = ''; + $offset = 0; + // The shard's records a batch at a time (a large shard size holds one batch, not the shard) + foreach($base < $limit ? $this->records->eachSegmentBatch($dbname, null, range($base, $limit - 1)) : [] as $batch){ + foreach($batch as $key => $record){ + $json = $this->io->encodeRecord($record); + \noneDB\Internal\Storage\KeyIndex::set($shardIndex, $key, $offset, strlen($json)); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + $count++; + } + } + return $buffer; + }); + if($shardId === $numShards - 1 && $nextKey - 1 > $maxKey){ + // Keep the highest key ever used, as compaction does: a key index rebuilt from these + // files must never hand it out again + $buffer .= '{"__nonedb_deleted":' . ($nextKey - 1) . "}\n"; + $shardIndex['t'] = 1; + } + $shardPath = $dir . 'segments/s' . $shardId . '.jsonl'; + if(!$this->io->writeFileAtomic($shardPath, $buffer, 'replace')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards failed while writing shard {$shardId}"); + } + $shardIndex['s'] = strlen($buffer); + $shardIndex['e'] = bin2hex(random_bytes(4)); + $buffer = null; + // Verified from the file: every record of the shard is a line of it + $written += $this->countRecordLines($shardPath, $base, $base + $size); + $this->checkpoints->writeCheckpointFile($shardPath, $shardId, $size, $shardIndex); // derived: best effort + $shardIndex = null; + $entry = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shardId); + $entry['count'] = $count; + $meta['shards'][] = $entry; + } + if($written !== $live){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not be verified"); + } + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($dir . 'shards.json', $json . "\n", 'durable')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not write its shard list"); + } + if(!empty($manifest['fields']) || !empty($manifest['spatial'])){ + $list = ['kind' => 'indexes', 'v' => 1, 'fields' => $manifest['fields'], 'spatial' => $manifest['spatial'], + 'spatialOptions' => array_intersect_key($manifest['spatialOptions'], array_flip($manifest['spatial']))]; + if(!$this->io->atomicWrite($dir . 'indexes.json', $list, false, 'durable')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not write its index list"); + } + } + }); + // The new generation is CURRENT: its shards' field and spatial indexes + $this->fieldIndex->rebuildIndexesAfterSwitch($dbname); + return true; + }, 'convertToSharded'); + } + + /** + * Complete lines of a data file that are records with keys in [$base, $limit) + */ + private function countRecordLines($path, $base, $limit){ + $count = 0; + $fp = @fopen($path, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $base, $limit)) !== null && $entry[0] === 'r'){ + $count++; + } + } + if($fp !== false){ + fclose($fp); + } + return $count; + } +} +} +namespace noneDB\Internal\Sharding { + +use function count; + +/** + * @internal + */ +final class ShardInfo { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $checkpoints; + private $layout; + private $locks; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->layout, $this->locks, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SHARDING PUBLIC METHODS + // ========================================== + + /** + * Get sharding information for a database + * @param string $dbname + * @return array|false + */ + public function getShardInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getShardInfoUnlocked($dbname); + }); + } + + /** + * getShardInfo() of a database whose lock the caller holds (backup() and restore() read the counters) + */ + public function getShardInfoUnlocked($dbname){ + if(!$this->segments->dbExists($dbname)){ + return false; + } + if(!$this->segments->isSharded($dbname)){ + $index = $this->checkpoints->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + return array( + "sharded" => false, + "shards" => 0, + "totalRecords" => \noneDB\Internal\Storage\KeyIndex::count($index), + "shardSize" => $this->segments->shardSizeOf($dbname), // fixed in FORMAT at creation (§4.1) + "nextKey" => (int)$index['n'] + ); + } + + $meta = $this->shardMeta->getCachedMeta($dbname); + if($meta === null){ + return false; + } + return array( + "sharded" => true, + "shards" => count($meta['shards']), + "totalRecords" => (int)$meta['totalRecords'], + "deletedCount" => (int)($meta['deletedCount'] ?? 0), + "shardSize" => (int)$meta['shardSize'], + "nextKey" => (int)$meta['nextKey'] + ); + } +} +} +namespace noneDB\Internal\Sharding { + +use function clearstatcache, count, fclose, fgets, fopen, is_array, is_file, is_int, max, sort, strlen, substr; + +/** + * @internal + */ +final class ShardMeta { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $metaCache; + private $shardedCache; + private $cacheGrew; + private $cacheTick; + private $cacheTouch; + private $noWrites; + private $cache; + private $checkpoints; + private $io; + private $layout; + private $locks; + private $recoveryLog; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->recoveryLog, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->metaCache = &$this->k->metaCache; + $this->shardedCache = &$this->k->shardedCache; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recoveryLog = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Shard list and counters of a sharded database + */ + public function getMetaPath($dbname){ + return $this->layout->genPath($dbname, 'shards.json'); + } + + /** + * Read shard metadata + * @param string $dbname + * @return array|null + */ + public function readMeta($dbname){ + $meta = $this->io->atomicRead($this->getMetaPath($dbname), null); + return is_array($meta) && ($meta['kind'] ?? null) === 'shards' && ($meta['v'] ?? null) === 1 + && is_int($meta['shardSize'] ?? null) && is_int($meta['nextKey'] ?? null) + && isset($meta['shards']) && is_array($meta['shards']) ? $meta : null; + } + + /** + * Get cached meta data. The cache is dropped whenever another process writes the + * database (generation check in withLock), so no TTL is needed. + * @param string $dbname + * @param bool $forceRefresh Force refresh from disk + * @return array|null + */ + public function getCachedMeta($dbname, $forceRefresh = false){ + $metaPath = $this->getMetaPath($dbname); + + $this->cacheTouch[$metaPath] = ++$this->cacheTick; + if(!$forceRefresh && isset($this->metaCache[$metaPath])){ + return $this->metaCache[$metaPath]; + } + + $meta = $this->readMeta($dbname); + if($meta === null){ + clearstatcache(true, $metaPath); + if(is_file($metaPath)){ + // There but unreadable (e.g. zero-length after a power loss): rebuilt from the shards. + // checkIntegrity() writes nothing: it works with a meta derived in memory, not kept + if($this->noWrites > 0){ + try { + return $this->deriveShardMeta($dbname); + } catch(\RuntimeException $e){ + return null; + } + } + $meta = $this->recoverMeta($dbname); + } + } + if($meta !== null){ + $this->metaCache[$metaPath] = $meta; + $this->cacheGrew = true; + return $meta; + } + unset($this->metaCache[$metaPath]); + return null; + } + + /** + * Rebuild an unreadable shards.json from the shard files (exclusive lock): the shard size is FORMAT's, + * nextKey follows the highest key ever used. A missing shards.json is never "recovered": shard files + * without one are leftovers (see setOrphanShardsAside). + * @return array|null + */ + private function recoverMeta($dbname){ + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $metaPath = $this->getMetaPath($dbname); + unset($this->metaCache[$metaPath]); + $meta = $this->readMeta($dbname); + clearstatcache(true, $metaPath); + if($meta !== null || !is_file($metaPath)){ + return $meta; + } + $meta = $this->deriveShardMeta($dbname); + $this->writeMeta($dbname, $meta, true); + $this->recountMeta($dbname); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'meta-rebuilt', 'shards' => count($meta['shards']), 'shardSize' => $meta['shardSize']]); + return $this->getCachedMeta($dbname); + }, 'recovery'); + } + + /** + * A shard list derived from the shard files alone (reads only; counters 0, recountMeta() fills them). + * The shard size is FORMAT's; the next key is above every key a shard file holds (tombstones included). + * @return array + */ + public function deriveShardMeta($dbname){ + $size = $this->segments->shardSizeOf($dbname); + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + $ids = []; + $nextKey = 0; + foreach($this->segments->listFilesWithPrefix($prefix, '/^\d+\.jsonl$/') as $file){ + $shardId = (int)substr($file, strlen($prefix)); + $ids[] = $shardId; + $fp = @fopen($file, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $shardId * $size, ($shardId + 1) * $size)) !== null){ + $nextKey = max($nextKey, $entry[1] + 1); + } + } + if($fp !== false){ + fclose($fp); + } + } + sort($ids); + $meta = self::newShardMeta($size, $nextKey); + foreach($ids as $shardId){ + $meta['shards'][] = self::newShardEntry($shardId); + } + return $meta; + } + + /** + * An empty shards.json record (docs/STORAGE_FORMAT.md §5.1) + */ + public static function newShardMeta($size, $nextKey){ + return ['kind' => 'shards', 'v' => 1, 'shardSize' => (int)$size, 'nextKey' => (int)$nextKey, + 'totalRecords' => 0, 'deletedCount' => 0, 'shards' => []]; + } + + public static function newShardEntry($shardId){ + return ['id' => (int)$shardId, 'count' => 0, 'deleted' => 0]; + } + + /** + * Write shard metadata (caller holds the exclusive lock) + * @param string $dbname + * @param array $meta + * @return bool + * @throws \RuntimeException when shards.json can't be written + */ + public function writeMeta($dbname, $meta, $sync = false){ + $path = $this->getMetaPath($dbname); + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($path, $json . "\n", $sync ? 'replace' : 'durable')){ + throw new \RuntimeException("noneDB: failed to write metadata of database '{$dbname}'"); + } + $this->metaCache[$path] = $meta; + $this->shardedCache[$path] = true; + $this->cache->markFileExists($path); + return true; + } + + /** + * Recompute the per-shard and total counters of shards.json from the shard indexes + */ + public function recountMeta($dbname){ + $meta = $this->getCachedMeta($dbname, true); + if($meta === null){ + return; + } + $size = (int)$meta['shardSize']; + $total = 0; + $nextKey = 0; + foreach($meta['shards'] as $i => $shard){ + $index = $this->checkpoints->readJsonlIndex($dbname, (int)$shard['id']); + $count = $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index); + $meta['shards'][$i]['count'] = $count; + $total += $count; + if($index !== null){ + // n of a shard counts every key it ever had (deleted ones and the kept top tombstone too) + $nextKey = max($nextKey, \noneDB\Internal\Storage\KeyIndex::maxKey($index) + 1); + if((int)$index['n'] > 0){ + $nextKey = max($nextKey, (int)$shard['id'] * $size + (int)$index['n']); + } + } + unset($index); + $this->cache->evictBetweenSegments(); + } + $meta['totalRecords'] = $total; + $meta['nextKey'] = max((int)($meta['nextKey'] ?? 0), $nextKey); + $this->writeMeta($dbname, $meta); + } +} +} +namespace noneDB\Internal\Sharding { + +use function array_column, array_diff_key, array_flip, array_keys, array_map, basename, count, date, intdiv, + max, min, preg_match, strlen, substr; + +/** + * @internal + */ +final class ShardRouter { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $fieldIndexCache; + private $indexCache; + private $shardFilesUpTo; + private $spatialIndexCache; + private $cache; + private $checkpoints; + private $io; + private $layout; + private $locks; + private $records; + private $recovery; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->records, $this->recovery, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->shardFilesUpTo = &$this->k->shardFilesUpTo; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recovery = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Insert into a sharded database. Keys come from meta.nextKey; key K lives in shard intdiv(K, shardSize). + */ + public function insertShardedRecords($dbname, array $items, $keyed = null, $endKey = null){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + $size = (int)$meta['shardSize']; + $nextKey = (int)$meta['nextKey']; + // The meta may lag behind the data (a meta update lost in a power loss): never hand out a key the + // last shard has already used + if(!empty($meta['shards'])){ + $last = max(array_map('intval', array_column($meta['shards'], 'id'))); + $lastIndex = $this->checkpoints->readJsonlIndex($dbname, $last); + if($lastIndex !== null){ + $nextKey = max($nextKey, $last * $size + (int)$lastIndex['n']); + } + } + + $byShard = []; + if($keyed !== null){ + foreach($keyed as $key => $item){ + $byShard[intdiv($key, $size)][$key] = $item; + } + $nextKey = max($nextKey, (int)$endKey); + $items = $keyed; + } else { + foreach($items as $item){ + $byShard[intdiv($nextKey, $size)][$nextKey] = $item; + $nextKey++; + } + } + + // Reserve the keys first: a crash later leaves a gap, never a reused key + $shardPos = []; + foreach($meta['shards'] as $i => $shard){ + $shardPos[(int)$shard['id']] = $i; + } + $this->setOrphanShardsAside($dbname, array_keys(array_diff_key($byShard, $shardPos))); + foreach($byShard as $shardId => $records){ + if(!isset($shardPos[$shardId])){ + $meta['shards'][] = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shardId); + $shardPos[$shardId] = count($meta['shards']) - 1; + } + $meta['shards'][$shardPos[$shardId]]['count'] += count($records); + } + // Journal first: if we die after reserving, recovery recounts the meta from the shards + foreach(array_keys($byShard) as $shardId){ + $this->locks->markIndexesPending($dbname, $shardId); + } + $meta['nextKey'] = $nextKey; + $meta['totalRecords'] = (int)$meta['totalRecords'] + count($items); + $this->shardMeta->writeMeta($dbname, $meta); + + // A failure below is undone by the rollback scope (appends of every shard cut back, counters + // recounted); the reserved keys stay unused and the new shards stay listed (empty) + foreach($byShard as $shardId => $records){ + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + if($index === null){ + $this->records->createJsonlDatabase($dbname, $shardId); + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + } + $this->records->appendSegmentRecords($dbname, $shardId, $index, $records); + $index['n'] = max((int)$index['n'], max(array_keys($records)) - $shardId * $size + 1); + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + } + } + + /** + * Files of a shard that shards.json doesn't list (left by a shards.json update lost in a power loss, or + * put there by hand) must not be taken over by a new shard with the same number: their records were + * deleted, or belong to keys the meta doesn't know any more. They are renamed to *.orphan.{time} + * (reported by getDB()), never deleted. + */ + private function setOrphanShardsAside($dbname, array $shardIds){ + if(empty($shardIds)){ + return; + } + // The directory is listed only for shards up to the highest one that had files at the last listing: + // above it only this process created shard files since (another process's write drops the number, + // invalidateDbCaches). A listing per new shard made inserts into many shards O(shards²) + // Only data and checkpoint files: field and spatial indexes check themselves against the data + // (segment epoch, covered bytes), so one left by an earlier shard of that number is never used + $base = $this->layout->genDir($dbname) . 'segments/s'; + $upTo = max(array_map('intval', $shardIds)); + if(isset($this->shardFilesUpTo[$base]) && min(array_map('intval', $shardIds)) > $this->shardFilesUpTo[$base]){ + $this->shardFilesUpTo[$base] = $upTo; + return; + } + $wanted = array_flip(array_map('intval', $shardIds)); + $files = []; + foreach($this->segments->listFilesWithPrefix($base, '/^\d+\./') as $file){ + $shardId = (int)substr($file, strlen($base)); + $upTo = max($upTo, $shardId); + if(isset($wanted[$shardId])){ + $files[] = $file; + } + } + $stamp = '.orphan.' . date('YmdHis'); + $moved = false; + foreach($files as $file){ + if(preg_match('/\.orphan\.\d+$/', $file)){ + continue; + } + if(!$moved){ + $this->locks->bumpEpoch($dbname); // a shard file of this name goes away: no cache of it is kept + $moved = true; + } + if(!$this->io->fsRename($file, $file . $stamp)){ + throw new \RuntimeException("noneDB: could not move aside the leftover file " . basename($file) . " of database '{$dbname}'"); + } + unset($this->indexCache[$file], $this->fieldIndexCache[$file], $this->spatialIndexCache[$file]); + $this->cache->markFileNotExists($file); + } + $this->shardFilesUpTo[$base] = $upTo; // after bumpEpoch(), which drops it + } +} +} +namespace noneDB\Internal\Storage { + +use function basename, clearstatcache, count, dirname, fclose, fgets, file_exists, filesize, fopen, is_array, + json_decode, max, preg_match, register_shutdown_function, strlen, substr, time, trim; + +/** + * @internal + */ +final class Buffer { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $bufferAutoFlushOnShutdown; + private $bufferCountLimit; + private $bufferFlushInterval; + private $bufferLastFlush; + private $bufferSizeLimit; + private $shutdownHandlerRegistered; + private $bufferCounts; + private $bufferEnds; + private $flushingNow; + private $lockRegistry; + private $checkpoints; + private $db; + private $io; + private $layout; + private $locks; + private $recovery; + private $recoveryLog; + private $scope; + private $segments; + private $shardMeta; + private $sync; + private $writer; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->db, $this->io, $this->layout, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->segments, $this->shardMeta, $this->sync, $this->writer); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferAutoFlushOnShutdown = &$this->k->bufferAutoFlushOnShutdown; + $this->bufferCountLimit = &$this->k->bufferCountLimit; + $this->bufferFlushInterval = &$this->k->bufferFlushInterval; + $this->bufferLastFlush = &$this->k->bufferLastFlush; + $this->bufferSizeLimit = &$this->k->bufferSizeLimit; + $this->shutdownHandlerRegistered = &$this->k->shutdownHandlerRegistered; + $this->bufferCounts = &$this->rt->bufferCounts; + $this->bufferEnds = &$this->rt->bufferEnds; + $this->flushingNow = &$this->rt->flushingNow; + $this->lockRegistry = &$this->rt->lockRegistry; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->db = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->writer = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE BUFFER + // ========================================== + // Optional (off by default). Buffered records have no key yet; flushing renames the buffer + // (buffer/pending.jsonl of the active generation) to buffer/flushing.{firstKey}.jsonl first and then + // inserts its records through insertRecords(), so the normal indexes and sharding logic apply. + // Pending buffers are flushed before every read and write, whether or not buffering is enabled. + + /** + * Get buffer file path of a database + * @param string $dbname + * @return string + */ + public function getBufferPath($dbname){ + return $this->layout->genPath($dbname, 'buffer/pending.jsonl'); + } + + /** + * Check if buffer exists and has content + * @param string $bufferPath + * @return bool + */ + public function hasBuffer($bufferPath){ + clearstatcache(true, $bufferPath); + return file_exists($bufferPath) && filesize($bufferPath) > 0; + } + + /** + * Number of records in a buffer file; counted once, then tracked by size + */ + public function getBufferRecordCount($bufferPath){ + clearstatcache(true, $bufferPath); + if(!file_exists($bufferPath)){ + return 0; + } + $size = filesize($bufferPath); + if(isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size){ + return $this->bufferCounts[$bufferPath][1]; + } + $count = count($this->readBufferRecords($bufferPath)); + $this->bufferCounts[$bufferPath] = [$size, $count]; + return $count; + } + + /** + * Read all records from a buffer file (JSONL). An unterminated last line is not a record (its + * append never completed). + * @param string $bufferPath + * @param bool $clean Set to false when the file holds anything that isn't a complete record line + * @return array + */ + public function readBufferRecords($bufferPath, &$clean = true){ + $clean = true; + $records = []; + $fp = @fopen($bufferPath, 'rb'); + if($fp === false){ + return $records; + } + while(($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $clean = false; + break; + } + $line = trim($line); + if($line === ''){ + continue; + } + $record = json_decode($line, true); + if(is_array($record)){ + $records[] = $record; + } else { + $clean = false; + } + } + fclose($fp); + return $records; + } + + /** + * Before appending to a buffer: a torn last line (crashed append) goes to quarantine and is cut off, + * so the next record isn't glued to it. Skipped when the file has the size this process wrote last. + */ + private function repairBufferTail($dbname, $bufferPath){ + clearstatcache(true, $bufferPath); + $size = @filesize($bufferPath); + if(!$size || ($this->bufferEnds[$bufferPath] ?? null) === $size){ + return; + } + $last = $this->checkpoints->lastNewlineIn($bufferPath, max(0, $size - 65536), $size); + if($last !== $size - 1){ + if($last < 0){ + $last = $this->checkpoints->lastNewlineIn($bufferPath, 0, $size); + } + $this->recovery->quarantineAndCut($dbname, 'b', $bufferPath, $last + 1, $size); + unset($this->bufferCounts[$bufferPath]); + } + } + + /** + * Before a flushed buffer file is deleted: anything in it that wasn't a record is kept in quarantine + */ + private function quarantineBufferRest($dbname, $file){ + clearstatcache(true, $file); + $size = (int)@filesize($file); + if($size > 0){ + $quarantine = $this->recovery->quarantineBytes($dbname, 'b', $file, 0, $size); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'drop-buffer', 'id' => $quarantine['id'], 'file' => basename($file), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]); + } + } + + /** + * True if the database has buffered records waiting + */ + private function hasPendingBuffers($dbname){ + // Under the lock the state record says it (flag 'b', set before a buffer gets records and + // cleared after they are flushed), so normal operations don't touch the file system for this + $path = $this->locks->getLockPath($dbname); + if(isset($this->lockRegistry[$path]) && $this->lockRegistry[$path]['epoch'] !== null){ + return !empty($this->lockRegistry[$path]['flags']['b']); + } + return $this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)); + } + + /** + * Buffer files whose flush was interrupted: buffer/flushing.{firstKey}.jsonl + */ + public function flushingFiles($dbname){ + return $this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'buffer/flushing.', '/^\d+\.jsonl$/'); + } + + /** + * Flush buffered records into the database (exclusive lock) + * @return int Number of records flushed + */ + public function flushPendingBuffers($dbname){ + $base = $this->layout->dbHome($dbname); + // Flushing inserts records, which can switch the database to shards, which flushes again: don't re-enter + if(isset($this->flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + return (int)$this->locks->withLock($dbname, true, function() use ($dbname, $base){ + // In place before the caller appended anything, else after the caller's write committed + return $this->scope->runIndependent($dbname, 'flush', function() use ($dbname, $base){ + if(isset($this->flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + $this->flushingNow[$base] = true; + try { + return $this->flushPendingBuffersLocked($dbname); + } finally { + unset($this->flushingNow[$base]); + } + }); + }, 'flush'); + } + + private function flushPendingBuffersLocked($dbname){ + { + $flushed = 0; + $bufferPath = $this->getBufferPath($dbname); + // Interrupted flushes first. A flush renames the buffer to flushing.{firstKey}.jsonl: the + // rename records atomically which keys its records get, so replaying it is idempotent. + foreach($this->flushingFiles($dbname) as $file){ + preg_match('/flushing\.(\d+)\.jsonl$/', $file, $m); + $flushed += $this->flushBufferFile($dbname, $file, (int)$m[1]); + } + if($this->hasBuffer($bufferPath)){ + if(!$this->segments->dbExists($dbname)){ + $this->db->createDBUnlocked($dbname); + } + $startKey = $this->nextKeyOf($dbname); + $flushingPath = dirname($bufferPath) . '/flushing.' . $startKey . '.jsonl'; + if(!$this->sync->renameWithRetry($bufferPath, $flushingPath)){ + throw new \RuntimeException("noneDB: could not flush the write buffer of database '{$dbname}'"); + } + $this->bufferEnds[$bufferPath] = 0; // the buffer is gone: the next append creates it + // The rename must survive a power loss before records get keys (else replayed twice) + $this->sync->requireSync(null, dirname($bufferPath), 'dir'); + $flushed += $this->flushBufferFile($dbname, $flushingPath, $startKey); + } + unset($this->bufferCounts[$bufferPath]); + $this->locks->setBufferFlag($dbname, false); + $this->bufferLastFlush[$dbname] = time(); + return $flushed; + } + } + + /** + * Insert the records of one buffer file, starting at $startKey + * @return int + */ + private function flushBufferFile($dbname, $file, $startKey){ + $records = $this->readBufferRecords($file, $clean); + if(!empty($records)){ + $result = $this->writer->insertRecords($dbname, $records, $startKey); + if(isset($result['error'])){ + throw new \RuntimeException("noneDB: flushing the write buffer of '{$dbname}' failed: " . $result['error']); + } + } + $this->scope->commitIndependentStep($dbname); // the records are stored for good before their buffer goes + if(!$clean){ + $this->quarantineBufferRest($dbname, $file); + } + $this->io->fsUnlink($file); + return count($records); + } + + /** + * The key the next inserted record gets + */ + private function nextKeyOf($dbname){ + if($this->segments->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + return $meta === null ? 0 : (int)$meta['nextKey']; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + return $index === null ? 0 : (int)$index['n']; + } + /** + * Append validated records to the database's write buffer (exclusive lock) + * @return array + */ + public function insertBuffered($dbname, array $validItems){ + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->db->createDBUnlocked($dbname); + } + $this->registerShutdownHandler(); + + $lines = $this->io->withExactFloats(function() use ($validItems){ + $lines = ''; + foreach($validItems as $item){ + $lines .= $this->io->encodeRecord($item) . "\n"; + } + return $lines; + }); + + $bufferPath = $this->getBufferPath($dbname); + // One size for the flush limits, the torn tail and the count. The size this process's last append + // left holds while no other process wrote the database: a write of another process moves the lock + // generation, which drops this cache (CacheManager::invalidateDbCaches), so no stat per insert. + $size = $this->bufferEnds[$bufferPath] ?? null; + if($size === null){ + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + $known = isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size ? $this->bufferCounts[$bufferPath][1] : null; + if($size > 0 && $this->shouldFlushBuffer($bufferPath, $dbname, $size, $known)){ + $this->flushPendingBuffers($dbname); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + if($size > 0 && ($this->bufferEnds[$bufferPath] ?? null) !== $size){ + $this->repairBufferTail($dbname, $bufferPath); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + $count = $size === 0 ? 0 : (isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size + ? $this->bufferCounts[$bufferPath][1] : $this->getBufferRecordCount($bufferPath)); + $this->locks->setBufferFlag($dbname, true); + if($this->io->appendToFile($bufferPath, $lines, $dbname) === false){ + throw new \RuntimeException("noneDB: failed to write the write buffer of database '{$dbname}'; nothing was written"); + } + $size += strlen($lines); // the append ended there (checked against the file next time) + $this->bufferCounts[$bufferPath] = [$size, $count + count($validItems)]; + $this->bufferEnds[$bufferPath] = $size; + if(!isset($this->bufferLastFlush[$dbname])){ + $this->bufferLastFlush[$dbname] = time(); + } + + if($this->shouldFlushBuffer($bufferPath, $dbname, $size, $count + count($validItems))){ + $this->flushPendingBuffers($dbname); + } + return ["n" => count($validItems)]; + } + + /** + * Check if buffer needs flushing (size, count, or time based) + * @param string $bufferPath + * @param string $flushKey Key for tracking last flush time + * @return bool + */ + private function shouldFlushBuffer($bufferPath, $flushKey, $size = null, $count = null){ + if($size === null){ + clearstatcache(true, $bufferPath); + $size = file_exists($bufferPath) ? (int)filesize($bufferPath) : 0; + } + if($size <= 0){ + return false; + } + if($size >= $this->bufferSizeLimit){ + return true; + } + if(($count ?? $this->getBufferRecordCount($bufferPath)) >= $this->bufferCountLimit){ + return true; + } + if($this->bufferFlushInterval > 0){ + $lastFlush = $this->bufferLastFlush[$flushKey] ?? time(); + if((time() - $lastFlush) >= $this->bufferFlushInterval){ + return true; + } + } + return false; + } + + /** + * Register shutdown handler for flushing all buffers + */ + private function registerShutdownHandler(){ + if($this->shutdownHandlerRegistered){ + return; + } + if($this->bufferAutoFlushOnShutdown){ + // Weak reference: the handler must not keep the instance alive until shutdown + $ref = \WeakReference::create($this->k); + register_shutdown_function(static function() use ($ref){ + $db = $ref->get(); + if($db !== null){ + try { + $db->maintenance->flushAllBuffers(); + } catch(\Throwable $e){ + // The records stay in the buffer (committed): the next access flushes them. An + // exception here would end the script with a fatal error after its work was done. + } + } + }); + $this->shutdownHandlerRegistered = true; + } + } +} +} +namespace noneDB\Internal\Storage { + +use function array_keys, basename, bin2hex, clearstatcache, count, crc32, end, fclose, file_exists, filesize, fopen, + fread, fseek, hash, in_array, intdiv, is_array, is_file, is_int, is_string, json_decode, max, min, random_bytes, + range, sprintf, strlen, strpos, strrpos, substr, time; +use const LOCK_EX, LOCK_SH; + +/** + * @internal + */ +final class Checkpoints { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + /** @var array data file => its checkpoint file (a pure memo of ckptPath()) */ + private static $ckptPaths = []; + + private $dataEpochs; + private $fieldIndexCache; + private $indexCache; + private $spatialIndexCache; + private $verifiedIndexes; + private $writableIndexes; + private $cacheGrew; + private $sessionCounter; + private $cacheTick; + private $cacheTouch; + private $lockRegistry; + private $metrics; + private $noWrites; + private $uncheckpointed; + private $decodeDue; + private $cache; + private $fieldIndex; + private $io; + private $locks; + private $recovery; + private $recoveryLog; + private $rtree; + private $secondary; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->uncheckpointed = &$this->rt->uncheckpointed; + $this->decodeDue = &$this->rt->decodeDue; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // JSONL SEGMENT INDEX + // ========================================== + + /** + * Get JSONL index path + * @param string $dbname + * @param int|null $shardId Null for non-sharded + * @return string + */ + public function getJsonlIndexPath($dbname, $shardId = null){ + return self::ckptPath($this->segments->segDataPath($dbname, $shardId)); + } + + /** + * Checkpoint of a data file: segments/{seg}.jsonl -> segments/{seg}.kix + */ + public static function ckptPath($dataPath){ + if(isset(self::$ckptPaths[$dataPath])){ + return self::$ckptPaths[$dataPath]; + } + if(count(self::$ckptPaths) >= 65536){ + self::$ckptPaths = []; + } + return self::$ckptPaths[$dataPath] = substr($dataPath, 0, -6) . '.kix'; + } + + public function newJsonlIndex(){ + return KeyIndex::emptyIndex(); + } + + /** + * A segment's key index (offset and length of every live record), from the cache, or loaded from its + * checkpoint and the data after it + * @param string $dbname + * @param int|null $shardId + * @return array|null Null when the segment doesn't exist + */ + public function readJsonlIndex($dbname, $shardId = null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + // A cached index was checked against its data file (no file check on this hot path); else the file + // must exist + if(!isset($this->indexCache[$path]) && !$this->cache->cachedFileExists($dataPath)){ + return null; + } + if(isset($this->indexCache[$path])){ + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->verifiedIndexes[$path])){ + if(($this->indexCache[$path]['kb'] !== '' || empty($this->indexCache[$path]['few'])) && $this->indexCache[$path]['ses'] !== $this->sessionCounter){ + // One more operation: a body is decoded once they have paid for it (a database of few segments + // decodes earlier: loadSegmentIndex(), decodeDueIndexes()); they count on after that for the + // data bytes of a segment (RecordIO::heldData()), which a database of few segments doesn't hold + $this->indexCache[$path]['ses'] = $this->sessionCounter; + if((++$this->indexCache[$path]['uses'] + $this->indexCache[$path]['looked']) * self::KEYS_PER_LOOKUP >= $this->indexCache[$path]['live'] + && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeKeyIndex($path); + } + } + $this->metrics['hits']++; + return $this->indexCache[$path]; + } + // Another process wrote the database since this one cached the index, without replacing a data + // file (same epoch): the data only grew, the new lines are replayed + $index = $this->indexCache[$path]; + unset($this->indexCache[$path]); + clearstatcache(true, $dataPath); + $size = @filesize($dataPath); + if($size !== false && $size >= (int)$index['s']){ + if($size > (int)$index['s']){ + $this->replaySegment($dbname, $shardId, $index, $size); + } + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->maybeReaderCheckpoint($dbname, $shardId); + $this->metrics['hits']++; + return $this->indexCache[$path]; + } + } + return $this->loadSegmentIndex($dbname, $shardId); + } + + /** + * Load a segment index: its checkpoint when it fits the data file, then the lines after it. A + * checkpoint that doesn't fit (the data changed behind it) rebuilds the index and the segment's + * secondary indexes under the exclusive lock; a missing one (or one of an unknown version) only + * replays the whole file. + * @return array|null + */ + private function loadSegmentIndex($dbname, $shardId){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + clearstatcache(true, $dataPath); + if(!is_file($dataPath)){ + if(file_exists($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + $this->cache->markFileNotExists($dataPath); + return null; + } + $size = (int)filesize($dataPath); + $this->metrics['misses']++; + // A checkpoint whose header, body length or a block crc is not valid is no checkpoint (full replay) + $checkpoint = KeyIndex::readFile($path); + if($checkpoint !== null){ + $this->cache->noteIndexRead(KeyIndex::HEADER + strlen($checkpoint[1])); + if(!$this->checkpointFits($dataPath, $checkpoint[0]['s'], $size, $checkpoint[0]['h'] ?? null)){ + return $this->repairSegmentIndex($dbname, $shardId); + } + $index = KeyIndex::fromFile($checkpoint[0], $checkpoint[1]); + $index['ckh'] = $checkpoint[0]['h']; // checked: the crc of the bytes before ck + } else { + $index = $this->newJsonlIndex(); + } + $this->replaySegment($dbname, $shardId, $index, $size); + $index['ses'] = $this->sessionCounter; // the operation that loads it is its first + $index['uses'] = 1; + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->indexCache[$path]['few'] = $this->segments->segmentCount($dbname) <= Segments::FEW_SEGMENTS; + $this->indexCache[$path]['dp'] = $dataPath; + $this->cacheTouch[$path] = ++$this->cacheTick; + $this->cacheGrew = true; + if($index['ck'] < 0){ + // No checkpoint to continue (missing, empty): the next write session of this process writes one + $this->uncheckpointed[$this->locks->getLockPath($dbname)][$shardId === null ? 'main' : $shardId] = $shardId; + } + $this->maybeReaderCheckpoint($dbname, $shardId); + // A database of few segments (§6.4): a small body decoded at its first use, except in the process's + // first operation (a single cold operation keeps the body); any other at the start of the database's + // next operation (LockManager::lockSession() calls decodeDueIndexes()) + if(!empty($this->indexCache[$path]['few']) && $this->indexCache[$path]['kb'] !== ''){ + if(strlen($this->indexCache[$path]['kb']) <= self::DECODE_AT_FIRST_USE && $this->sessionCounter >= 2){ + $this->decodeKeyIndex($path); + } + if(isset($this->indexCache[$path]) && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeDue[$this->locks->getLockPath($dbname)][$path] = true; + } + } + return $this->indexCache[$path]; + } + + /** + * True when a checkpoint covering $covered bytes belongs to this data file: within it, at a line end, + * and the bytes before $covered have the checksum it recorded + */ + public function checkpointFits($dataPath, $covered, $size, $crc){ + if($covered < 0 || $covered > $size || !is_string($crc)){ + return false; + } + return $this->segmentCrc($dataPath, $covered) === $crc; + } + + /** + * crc32b of the up to 256 bytes before $offset of a data file; null when $offset is not a line end + */ + public function segmentCrc($dataPath, $offset){ + if($offset === 0){ + return sprintf('%08x', crc32('')); + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return null; + } + $start = max(0, $offset - 256); + fseek($fp, $start); + $bytes = (string)fread($fp, $offset - $start); + fclose($fp); + return strlen($bytes) === $offset - $start && substr($bytes, -1) === "\n" ? sprintf('%08x', crc32($bytes)) : null; + } + + /** + * Apply the complete lines of bytes [s, $size) of a segment's data file to its index: the last line of + * a key wins, a tombstone deletes. An unterminated tail stays outside ("s" ends before it; the next + * writer quarantines and cuts it). A complete line that doesn't decode (a damaged region before it, + * STORAGE_FORMAT.md §6.1) is salvaged when a record or tombstone can be found at its end, and reported + * either way. + * @param array|null $hint an earlier index of the segment (p/l), used to find records in damaged lines + * @param bool $report Report salvaged and skipped lines (not for checkIntegrity()) + */ + public function replaySegment($dbname, $shardId, array &$index, $size, $hint = null, $report = true){ + $from = (int)$index['s']; + if($shardId !== null){ + $index['z'] = $this->segments->shardSizeOf($dbname); + $index['sid'] = $shardId; + } + if($size <= $from){ + return; + } + $fp = @fopen($this->segments->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return; + } + $base = $shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->segments->shardSizeOf($dbname); + $p = $index['p']; + $l = $index['l']; + $x = $index['x']; + unset($index['p'], $index['l'], $index['x']); // sole owner: changed in place, not copied + $hasBody = $index['kb'] !== ''; + $live = $index['live']; + $n = (int)$index['n']; + $d = (int)$index['d']; + $t = (int)$index['t']; + $lines = 0; + $oldEnds = null; + $events = []; + fseek($fp, $from); + $read = $from; + $offset = $from; // file offset of $buffer[0] + $buffer = ''; + while($read < $size){ + $chunk = fread($fp, min(1048576, $size - $read)); + if($chunk === false || $chunk === ''){ + break; + } + $read += strlen($chunk); + $buffer .= $chunk; + $pos = 0; + while(($newline = strpos($buffer, "\n", $pos)) !== false){ + $json = substr($buffer, $pos, $newline - $pos); + $lineOffset = $offset + $pos; + $entry = $this->classifyLine($json, $base, $limit); // a key of another shard is no line of this one + if($entry === null){ + if($oldEnds === null){ + $oldEnds = []; + [$hintOffsets, $hintLengths] = $hint === null ? [[], []] : KeyIndex::locate($hint, null); + foreach($hintOffsets as $k => $at){ + $oldEnds[(int)$at + (int)$hintLengths[$k]] = [(int)$k, (int)$at]; + } + $hintOffsets = $hintLengths = null; + } + $entry = $this->salvageLine($json, $lineOffset, $oldEnds, $base, $limit); + $events[] = [ + 'action' => $entry === null ? 'skip-line' : 'salvaged', + 'seg' => \noneDB\Internal\Storage\Layout::segmentName($shardId), + 'off' => $lineOffset, 'len' => strlen($json) + 1, + 'sha256' => hash('sha256', $json . "\n"), + ] + ($entry === null ? [] : ['key' => $entry[1]]); + } + if($entry === null){ + $d++; + } else { + $key = $entry[1]; + // The key's record so far: written since the checkpoint (p), deleted since (x), or in the body + $inBody = $hasBody && !isset($x[$key]) && KeyIndex::baseGet($index, $key) !== null; + $existed = isset($p[$key]) || $inBody; + if($entry[0] === 't'){ + if($existed){ + unset($p[$key], $l[$key]); + if($inBody){ + $x[$key] = true; + } + $live--; + $d++; + } + $t++; + } else { + if($existed){ + $d++; + } else { + $live++; + } + $p[$key] = $lineOffset + $entry[2]; + $l[$key] = $entry[3]; + unset($x[$key]); + } + if($key >= $base && $key - $base + 1 > $n){ + $n = $key - $base + 1; + } + } + $lines++; + $pos = $newline + 1; + } + $offset += $pos; + $buffer = (string)substr($buffer, $pos); + } + fclose($fp); + $index['p'] = $p; + $index['l'] = $l; + $index['x'] = $x; + $index['live'] = $live; + $index['n'] = $n; + $index['d'] = $d; + $index['t'] = $t; + $index['s'] = $offset; + $index['tl'] = (int)$index['tl'] + $lines; + foreach($report ? $events : [] as $event){ + $event['id'] = $event['seg'] . '.' . $event['off'] . '.' . $event['len'] . '.' . $event['sha256']; + unset($event['sha256']); + $this->recoveryLog->reportRecoveryEvent($dbname, $event); + } + } + + /** + * A segment index changed by a write (exclusive lock): cached, and checkpointed when the session ends + */ + public function storeSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->lockRegistry[$this->locks->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + } + + /** + * True when the lines after a segment's checkpoint are many enough to write a new one + */ + private function needsCheckpoint(array $index){ + return (int)$index['ck'] < 0 + || (int)$index['tl'] >= self::CHECKPOINT_LINES + intdiv($index['live'], 16) + || (int)$index['s'] - (int)$index['ck'] >= self::CHECKPOINT_BYTES; + } + + /** + * Write the checkpoint file of a data file's key index (an index with its epoch; exclusive lock, or a reader: + * the content is the data's): [layout, slots, body, max, live, header, base], or null when its bytes don't end + * at a line or the file can't be written + * @return array|null + */ + public function writeCheckpointFile($dataPath, $shardId, $shardSize, array $index){ + $crc = $this->segmentCrc($dataPath, (int)$index['s']); + if($crc === null){ + return null; + } + $base = $shardId === null ? 0 : $shardId * $shardSize; + [$layout, $slots, $body, $max, $live] = KeyIndex::encodeBody($index, $base); + $header = ['kind' => 'kix', 'v' => 1, 'layout' => $layout, 'base' => $base, 'slots' => $slots, 'live' => $live, + 'n' => (int)$index['n'], 'd' => (int)$index['d'], 't' => (int)$index['t'], 's' => (int)$index['s'], 'h' => $crc, + 'e' => $index['e'], 'created' => (int)$index['created'], 'updated' => time()]; + if($shardId !== null){ + $header['z'] = $shardSize; + $header['sid'] = $shardId; + } + if(!$this->io->writeDerivedFile(self::ckptPath($dataPath), KeyIndex::encodeFile($header, $body))){ + return null; + } + return [$layout, $slots, $body, $max, $live, $header, $base]; + } + + /** + * Write a segment's checkpoint (best effort: a failure only means a longer replay next time). An + * index without epoch gets one, which only the exclusive lock holder may assign. + * @return bool + */ + public function writeCheckpoint($dbname, $shardId, array &$index){ + if($this->noWrites > 0){ + return false; + } + $lock = $this->locks->getLockPath($dbname); + $exclusive = isset($this->lockRegistry[$lock]) && $this->lockRegistry[$lock]['mode'] === LOCK_EX; + if($index['e'] === null){ + if(!$exclusive){ + return false; + } + $index['e'] = bin2hex(random_bytes(4)); + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $written = $this->writeCheckpointFile($dataPath, $shardId, $shardId === null ? null : $this->segments->shardSizeOf($dbname), $index); + if($written === null){ + return false; + } + [$layout, $slots, $body, $max, $live, $header, $base] = $written; + if($index['kb'] !== ''){ + $index = KeyIndex::rebased($index, $layout, $slots, $body, $max, $base); // compact again + } + // (an index held as arrays stays so: it is in use, or was just replayed whole; eviction compacts it) + $index['live'] = $live; + $index['updated'] = $header['updated']; + $index['ck'] = (int)$index['s']; + $index['ckh'] = $header['h']; + $index['tl'] = 0; + $this->indexCache[self::ckptPath($dataPath)] = $index; + $this->dataEpochs[$dataPath] = $index['e']; + return true; + } + + /** + * End of a successful write session: checkpoint the segments it changed that need one + */ + public function checkpointTouchedSegments($dbname, $lock){ + $touched = $this->lockRegistry[$lock]['touched'] ?? []; + $segments = $touched + ($this->uncheckpointed[$lock] ?? []); + unset($this->uncheckpointed[$lock]); + foreach($segments as $shardId){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + if(!isset($this->indexCache[$path], $this->verifiedIndexes[$path]) || !$this->needsCheckpoint($this->indexCache[$path])){ + continue; + } + $index = $this->indexCache[$path]; + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + // a checkpoint is never needed for correctness + } + } + $this->lockRegistry[$lock]['touched'] = []; + foreach($touched as $shardId){ + if(!isset($this->indexCache[$this->getJsonlIndexPath($dbname, $shardId)])){ + continue; // dropped by the cache limit during the session: the next reader that loads it checkpoints it + } + try { + $this->secondary->checkpointSegmentSecondaries($dbname, $shardId); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + } + } + } + + /** + * A reader that replayed a long tail writes the checkpoint itself (shared lock: the data can't change, + * the content is the same as a writer's; no generation bump). Only on top of an existing checkpoint with + * an epoch, and not inside a write session of this process (that one checkpoints at its end). + */ + private function maybeReaderCheckpoint($dbname, $shardId){ + $lock = $this->locks->getLockPath($dbname); + if($this->noWrites > 0 || !isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_SH){ + return; + } + $path = $this->getJsonlIndexPath($dbname, $shardId); + $index = $this->indexCache[$path]; + if($index['e'] === null || (int)$index['ck'] < 0 || !$this->needsCheckpoint($index)){ + return; + } + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + } + } + + /** + * Position of the last "\n" in bytes [$from, $to) of a file, -1 if there is none + */ + public function lastNewlineIn($path, $from, $to){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return -1; + } + try { + $end = $to; + while($end > $from){ + $start = max($from, $end - 65536); + fseek($fp, $start); + $chunk = (string)fread($fp, $end - $start); + $pos = strrpos($chunk, "\n"); + if($pos !== false){ + return $start + $pos; + } + $end = $start; + } + return -1; + } finally { + fclose($fp); + } + } + + /** + * Rebuild a segment index from its data file (exclusive lock, upgrading a shared one): a checkpoint + * that doesn't fit the data, a key the index points at that isn't there. The rebuilt index has a new + * epoch, so the segment's field and spatial indexes are rebuilt as well (STORAGE_FORMAT.md §6.5). + * @param bool $secondaries Also rebuild field/spatial indexes when needed (recovery does it itself) + * @return array|null + */ + public function repairSegmentIndex($dbname, $shardId, $secondaries = true){ + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId, $secondaries){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + clearstatcache(true, $dataPath); + if(!file_exists($dataPath)){ + return null; + } + if(!is_file($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + $old = $this->checkpointOnDisk($path); + $index = $this->rebuildSegmentIndexFromData($dbname, $shardId, $old); + if($shardId !== null){ + $this->shardMeta->recountMeta($dbname); + } + // The rebuilt index has a new epoch: every field and spatial index of the segment is invalid now (§6.5), + // also when the entries are the old ones (the data changed in place). Built again while this exclusive + // lock is held, not by the next reader + if($secondaries){ + $this->rebuildSecondaryIndexes($dbname, [$shardId]); + } + return $index; + }, 'rebuildIndex'); + } + + + /** + * The key index a checkpoint file on disk holds (valid by the format, not checked against the data), or null + */ + private function checkpointOnDisk($path){ + $checkpoint = KeyIndex::readFile($path); + if($checkpoint === null){ + return null; + } + $this->cache->noteIndexRead(KeyIndex::HEADER + strlen($checkpoint[1])); + return KeyIndex::fromFile($checkpoint[0], $checkpoint[1]); + } + + /** + * Rebuild field and spatial indexes of some segments (and the global ones of a sharded database) + * from the data. Caller holds the exclusive lock. + */ + public function rebuildSecondaryIndexes($dbname, array $segments){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; + } + $known = $this->segments->listSegments($dbname); + // One index that can't be built must never lock the whole database: it is dropped (queries scan, + // the next write or query builds it again) and logged + $guard = function(callable $build, $what, array $files) use ($dbname){ + try { + $build(); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + foreach($files as $file){ + $this->io->fsUnlink($file); + unset($this->fieldIndexCache[$file], $this->spatialIndexCache[$file]); + } + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'index-dropped', 'index' => $what, 'error' => $e->getMessage()]); + } + }; + foreach($segments as $shardId){ + if(!in_array($shardId, $known, true)){ + continue; + } + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field, $shardId){ + $index = $this->fieldIndex->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + }, 'field ' . $field, [$this->fieldIndex->getFieldIndexPath($dbname, $field, $shardId)]); + } + foreach($manifest['spatial'] as $field){ + $guard(function() use ($dbname, $field, $shardId, $manifest){ + $tree = $this->rtree->buildSegmentSpatialIndex($dbname, $field, $shardId, + isset($manifest['spatialOptions'][$field]['nodeSize']) ? ['nodeSize' => $manifest['spatialOptions'][$field]['nodeSize']] : []); + $this->rtree->writeSpatialIndex($dbname, $field, $tree, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); + } + }, 'spatial ' . $field, [$this->rtree->getSpatialIndexPath($dbname, $field, $shardId)]); + } + $this->cache->evictBetweenSegments(); + } + if($this->segments->isSharded($dbname)){ + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field){ + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + }, 'global field ' . $field, [$this->fieldIndex->getGlobalFieldIndexPath($dbname, $field)]); + } + } + } + + /** + * Index of a segment a writer is about to change (exclusive lock), taken out of the cache: the + * writer changes it in place and stores it back (storeSegmentIndex). Checked against the data file + * when it was not yet known to match it: an unterminated tail (crashed append) is quarantined and cut + * off, so the next line can't be glued to it. A checked index stays checked until its cache is + * dropped; appendToFile() compares the size it finds with the index anyway (no extra system call), + * which catches anything else. + * @return array|null + */ + public function writableSegmentIndex($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null){ + return null; + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + if(!isset($this->writableIndexes[$path])){ + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + if($size < (int)$index['s']){ + $index = $this->repairSegmentIndex($dbname, $shardId); + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + } + if($size > (int)$index['s']){ + $this->recovery->quarantineAndCut($dbname, $shardId, $dataPath, (int)$index['s'], $size); + } + $this->writableIndexes[$path] = true; + } + unset($this->indexCache[$path]); // the caller is its only holder until it stores it back + return $index; + } + + /** + * A writer's index goes back to the cache unchanged (nothing to write after all) + */ + public function releaseSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + } + + /** + * Rebuild a segment index from the whole data file and write its checkpoint (exclusive lock): new + * segment epoch, as there is no checkpoint it continues + * @param array|null $old The checkpoint found (created time, key counter, records in damaged lines) + * @return array + */ + public function rebuildSegmentIndexFromData($dbname, $shardId, $old = null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + if($old === null){ + $old = $this->checkpointOnDisk(self::ckptPath($dataPath)); + } + $index = $this->newJsonlIndex(); + if($old !== null){ + $index['created'] = $old['created'] ?? $index['created']; + if($shardId === null && isset($old['n'])){ + $index['n'] = (int)$old['n']; + } + } + clearstatcache(true, $dataPath); + $this->replaySegment($dbname, $shardId, $index, (int)@filesize($dataPath), $old); // its entries only if a line is damaged + $index['e'] = bin2hex(random_bytes(4)); + $path = self::ckptPath($dataPath); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->writeCheckpoint($dbname, $shardId, $index); + return $this->indexCache[$path]; + } + + /** + * Classify one complete data line: ['r', key, 0, length] (record), ['t', key] (tombstone), or null + * when it isn't a valid line of this segment + */ + public function classifyLine($json, $base, $limit){ + $record = json_decode($json, true); + if(!is_array($record)){ + return null; + } + if(count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $key = $record['__nonedb_deleted']; + return $key >= $base && ($limit === null || $key < $limit) ? ['t', $key] : null; + } + if(isset($record['key']) && is_int($record['key'])){ + $key = $record['key']; + return $key >= $base && ($limit === null || $key < $limit) ? ['r', $key, 0, strlen($json)] : null; + } + return null; + } + + /** + * Find a record or tombstone at the end of a line that doesn't decode (a damaged region before it). + * First where the earlier index says a record ended; then candidates from the + * rightmost "{" leftwards: inside a record no "{" starts a JSON text that also ends the line, so the + * first one that decodes is where the record starts. + * @return array|null Like classifyLine(), with the record's offset inside the line + */ + public function salvageLine($json, $lineOffset, array $oldEnds, $base, $limit){ + $end = $lineOffset + strlen($json); + if(isset($oldEnds[$end]) && $oldEnds[$end][1] > $lineOffset){ + $start = $oldEnds[$end][1] - $lineOffset; + $entry = $this->classifySalvaged(substr($json, $start), $base, $limit); + if($entry !== null && $entry[0] === 'r' && $entry[1] === $oldEnds[$end][0]){ + return ['r', $entry[1], $start, strlen($json) - $start]; + } + } + $pos = strlen($json); + for($tries = 0; $tries < 4096; $tries++){ + $pos = $pos > 0 ? strrpos($json, '{', $pos - strlen($json) - 1) : false; + if($pos === false || $pos === 0){ + return null; + } + $entry = $this->classifySalvaged(substr($json, $pos), $base, $limit); + if($entry !== null){ + return $entry[0] === 't' ? $entry : ['r', $entry[1], $pos, strlen($json) - $pos]; + } + } + return null; + } + + /** + * classifyLine() for a salvage candidate: also requires an object whose last member is "key" + */ + private function classifySalvaged($candidate, $base, $limit){ + $entry = $this->classifyLine($candidate, $base, $limit); + if($entry === null || $entry[0] === 't'){ + return $entry; + } + $record = json_decode($candidate, true); + $keys = array_keys($record); + return end($keys) === 'key' && $keys !== range(0, count($keys) - 1) ? $entry : null; + } + + // Segment index: checkpoint + replay (docs/STORAGE_FORMAT.md §6.4). The data file is a log; the .kix + // file is a checkpoint of the index at byte offset "s" of it: a 512-byte JSON header ("kind":"kix", "v":1, + // layout, base, slots, live, n next key, d garbage lines, t tombstone lines, s, h = crc32b of the up to 256 + // bytes before s, e segment epoch, z shard size, sid shard id), then the slots in 4 KiB blocks with their crc + // (KeyIndex). Loading reads the checkpoint and replays the complete lines after S (an unterminated tail is never + // part of it, readers never cut it). Writes append to the data and update the cached index only; a + // write session writes a new checkpoint of a segment it changed once its tail reached 64 lines plus + // 1/16 of the live records, or 1 MiB (checkpointTouchedSegments). A checkpoint counts only for its data + // file: S within it, at a line end, h matching; a data file is replaced only after its checkpoint was + // deleted. "e" is the segment's data epoch for the secondary indexes: new when the data file is + // replaced or the index rebuilt without a checkpoint, assigned only under the exclusive lock. + // Runtime keys of a cached index (never written): "ck" = S of the checkpoint on disk (-1: none), + // "tl" = lines after it. + + /** A key index body of at most DECODE_MAX_KEYS keys may be decoded into arrays */ + private const DECODE_MAX_KEYS = 250000; + /** A body of at most this many bytes, in a database of few segments, is decoded at its first use (§6.4) */ + private const DECODE_AT_FIRST_USE = 65536; + /** Estimated memory of a decoded key index per key */ + private const DECODED_KEY_BYTES = 80; + /** + * Keys whose decoding costs about what one key lookup costs more on a body than on arrays (measured: + * 0.17 µs per key to decode, 1.3 µs more per lookup on a body) + */ + private const KEYS_PER_LOOKUP = 8; + + /** + * Decode a cached key index body into arrays (lookups at array speed) when it has at most + * DECODE_MAX_KEYS keys and memory allows (CacheManager::decodeFits()). In a database of few segments + * (Segments::FEW_SEGMENTS: each is in constant use): at its first use when the body is at most + * DECODE_AT_FIRST_USE bytes (loadSegmentIndex()), else at its second operation. Otherwise once the + * lookups through the body have paid for decoding: operations (one key each, counted in readJsonlIndex()) + * and keys looked up in bulk (noteLookups()), KEYS_PER_LOOKUP live keys each. Light use keeps the body: + * a cold query over many segments, a few reads of each segment. + */ + private function decodeKeyIndex($path){ + $live = $this->indexCache[$path]['live']; + if($live <= self::DECODE_MAX_KEYS && \noneDB\Internal\Cache\CacheManager::decodeFits(self::DECODED_KEY_BYTES * $live)){ + $this->indexCache[$path] = KeyIndex::materialize($this->indexCache[$path]); + $this->cacheGrew = true; + } + } + + /** + * Start of an operation (lock session) on a database of few segments: the key index bodies an earlier + * operation loaded are decoded now (§6.4), whether this operation reads them or not. Called once the + * lock state was checked: an index another process made stale is gone from the cache by then. + */ + public function decodeDueIndexes($lockPath){ + foreach($this->decodeDue[$lockPath] as $path => $unused){ + if(isset($this->indexCache[$path], $this->verifiedIndexes[$path]) && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeKeyIndex($path); + } + } + unset($this->decodeDue[$lockPath]); + } + + /** + * A segment's key index for a read of every record: a whole read decodes the body anyway, so the decoded + * index is kept (when memory allows) and the writes that often follow such a read work on arrays + */ + public function decodedIndex($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null || $index['kb'] === ''){ + return $index; + } + $path = self::ckptPath($this->segments->segDataPath($dbname, $shardId)); + $this->decodeKeyIndex($path); + return $this->indexCache[$path] ?? $index; + } + + /** Keys looked up in bulk through the key index body of a data file (see decodeKeyIndex()) */ + public function noteLookups($dataPath, $n){ + $path = self::ckptPath($dataPath); + if(!isset($this->indexCache[$path]) || $this->indexCache[$path]['kb'] === ''){ + return; + } + $this->indexCache[$path]['looked'] += $n; + if(($this->indexCache[$path]['uses'] + $this->indexCache[$path]['looked']) * self::KEYS_PER_LOOKUP >= $this->indexCache[$path]['live']){ + $this->decodeKeyIndex($path); + } + } + public const CHECKPOINT_LINES = 64; + public const CHECKPOINT_BYTES = 1048576; +} +} +namespace noneDB\Internal\Storage { + +use function array_combine, array_fill, array_filter, array_intersect_key, array_key_last, array_keys, + array_replace, array_slice, array_values, count, crc32, in_array, intdiv, is_array, is_int, is_string, + fclose, fopen, fread, fstat, json_decode, json_encode, ksort, max, min, pack, range, rtrim, str_pad, strlen, strrpos, substr, time, unpack; +use const JSON_UNESCAPED_SLASHES, PHP_INT_MAX, PHP_INT_MIN; + +/** + * A key index is an array: the checkpoint's members (n, d, t, s, e, created, updated, z, sid), the runtime + * members ck (bytes the checkpoint on disk covers, -1 none) and tl (lines replayed after it), and the keys: + * - kb: the verified body of the checkpoint (blocks of 4096 bytes with their crc), kl its layout ('dense' or + * 'sparse'), kbase its first key, kslots its slots (dense) or entries (sparse), kmax its highest live key; + * - p / l: offset and length of every key written since the checkpoint; x: keys of the body deleted since; + * - live: the number of live keys. + * No per-key array of the whole segment is built: a body costs 12 bytes per slot. + * @internal + */ +final class KeyIndex { + public const HEADER = 512; + public const BLOCK = 4096; + public const DENSE_SLOTS = 341; // 341 offsets (8 bytes) then 341 lengths (4 bytes) = 4092 bytes + the crc + public const SPARSE_ENTRIES = 255; // 255 keys (4), offsets (8), lengths (4) = 4080 bytes, zero bytes up to 4092, the crc + /** crc32 of bytes followed by their own crc32 (u32 little-endian): a block checks with one call */ + public const CRC_RESIDUE = 0x2144DF1C; + + /** An empty key index */ + public static function emptyIndex(): array { + $now = time(); + return ['n' => 0, 'd' => 0, 't' => 0, 's' => 0, 'e' => null, 'created' => $now, 'updated' => $now, + 'ck' => -1, 'ckh' => null, 'tl' => 0, 'kb' => '', 'kl' => 'dense', 'kbase' => 0, 'kslots' => 0, 'kmax' => -1, 'live' => 0, + 'p' => [], 'l' => [], 'x' => [], 'looked' => 0, 'uses' => 0, 'ses' => -1]; + } + + // ------------------------------------------------------------------ lookups + + /** [offset, length] of a key in the body, or null */ + public static function baseGet(array $index, int $key): ?array { + $kb = $index['kb']; + if($kb === ''){ + return null; + } + $local = $key - $index['kbase']; + if($local < 0){ + return null; + } + $slots = $index['kslots']; + if($index['kl'] === 'dense'){ + if($local >= $slots){ + return null; + } + $block = intdiv($local, self::DENSE_SLOTS); + $first = $block * self::DENSE_SLOTS; + $m = $slots - $first < self::DENSE_SLOTS ? $slots - $first : self::DENSE_SLOTS; + $at = $block * self::BLOCK; + $length = unpack('V', $kb, $at + 8 * $m + 4 * ($local - $first))[1]; + return $length === 0 ? null : [unpack('P', $kb, $at + 8 * ($local - $first))[1], $length]; + } + $lo = 0; + $hi = $slots - 1; + while($lo <= $hi){ + $mid = ($lo + $hi) >> 1; + $block = intdiv($mid, self::SPARSE_ENTRIES); + $first = $block * self::SPARSE_ENTRIES; + $i = $mid - $first; + $at = $block * self::BLOCK; + $found = unpack('V', $kb, $at + 4 * $i)[1]; + if($found === $local){ + $m = $slots - $first < self::SPARSE_ENTRIES ? $slots - $first : self::SPARSE_ENTRIES; + return [unpack('P', $kb, $at + 4 * $m + 8 * $i)[1], unpack('V', $kb, $at + 12 * $m + 4 * $i)[1]]; + } + if($found < $local){ + $lo = $mid + 1; + } else { + $hi = $mid - 1; + } + } + return null; + } + + public static function has(array $index, int $key): bool { + return isset($index['p'][$key]) || ($index['kb'] !== '' && !isset($index['x'][$key]) && self::baseGet($index, $key) !== null); + } + + public static function count(array $index): int { + return $index['live']; + } + + /** + * Point a key at a line; true when the key had a record before (its old line becomes garbage) + */ + public static function set(array &$index, int $key, int $offset, int $length): bool { + $existed = isset($index['p'][$key]) || (!isset($index['x'][$key]) && self::baseGet($index, $key) !== null); + $index['p'][$key] = $offset; + $index['l'][$key] = $length; + unset($index['x'][$key]); + if(!$existed){ + $index['live']++; + } + return $existed; + } + + /** + * Point the keys of appended lines at them: $lines [key => line without "\n"] written in this order from + * byte $offset. Returns how many keys had a record before (their old lines are garbage now). + */ + public static function setLines(array &$index, array $lines, int $offset): int { + $replaced = 0; + $added = 0; + $p = &$index['p']; + $l = &$index['l']; + if($index['kb'] === ''){ + foreach($lines as $key => $json){ + if(isset($p[$key])){ + $replaced++; + } else { + $added++; + } + $p[$key] = $offset; + $l[$key] = $length = strlen($json); + $offset += $length + 1; + } + } else { + $x = &$index['x']; + foreach($lines as $key => $json){ + if(isset($p[$key]) || (!isset($x[$key]) && self::baseGet($index, $key) !== null)){ + $replaced++; + } else { + $added++; + } + $p[$key] = $offset; + $l[$key] = $length = strlen($json); + unset($x[$key]); + $offset += $length + 1; + } + unset($x); + } + unset($p, $l); + $index['live'] += $added; + return $replaced; + } + + /** Delete keys; returns how many had a record */ + public static function removeMany(array &$index, array $keys): int { + $removed = 0; + if($index['kb'] === ''){ + // A decoded index: its arrays only, no call per key + $p = &$index['p']; + $l = &$index['l']; + foreach($keys as $key){ + if(isset($p[$key])){ + unset($p[$key], $l[$key]); + $removed++; + } + } + unset($p, $l); + $index['live'] -= $removed; + return $removed; + } + foreach($keys as $key){ + if(self::remove($index, (int)$key)){ + $removed++; + } + } + return $removed; + } + + /** Delete a key; true when it had a record */ + public static function remove(array &$index, int $key): bool { + if(isset($index['p'][$key])){ + unset($index['p'][$key], $index['l'][$key]); + if(self::baseGet($index, $key) !== null){ + $index['x'][$key] = true; + } + $index['live']--; + return true; + } + if(isset($index['x'][$key]) || self::baseGet($index, $key) === null){ + return false; + } + $index['x'][$key] = true; + $index['live']--; + return true; + } + + /** The highest live key (-1: none) */ + public static function maxKey(array $index): int { + $max = $index['kmax'] >= 0 && !isset($index['x'][$index['kmax']]) ? $index['kmax'] : self::bodyMax($index); + foreach($index['p'] as $key => $unused){ + if($key > $max){ + $max = $key; + } + } + return $max; + } + + + /** Every live key: ascending, but in the order of its arrays for a decoded index (the caller sorts) */ + public static function keys(array $index): array { + if($index['kb'] === ''){ + return array_keys($index['p']); + } + if(empty($index['p']) && empty($index['x'])){ + return self::baseKeys($index); + } + // The body's keys, the removed ones out, the delta's in (no array per key) + $live = self::bodyArrays($index)[1]; + foreach($index['x'] as $key => $unused){ + unset($live[$key]); + } + if(!empty($index['l'])){ + foreach($index['l'] as $key => $length){ + $live[$key] = $length; + } + ksort($live); + } + return array_keys($live); + } + + /** + * Offsets and lengths of $keys (all live keys when null) for a batch read: [key => offset], [key => length] + */ + public static function locate(array $index, ?array $keys): array { + $positions = []; + $lengths = []; + if($keys === null){ + // Every live key: the arrays of a decoded index (the body unpacked in a few C-level calls, the + // delta applied); in no particular order, the batch read orders by position + $all = self::materialize($index); + return [$all['p'], $all['l']]; + } + $p = $index['p']; + $l = $index['l']; + if($index['kb'] === ''){ + // Every key in the arrays (a decoded index): the hot path, no call per key + foreach($keys as $key){ + if(isset($p[$key])){ + $positions[$key] = $p[$key]; + $lengths[$key] = $l[$key]; + } + } + return [$positions, $lengths]; + } + $x = $index['x']; + foreach($keys as $key){ + if(isset($p[$key])){ + $positions[$key] = $p[$key]; + $lengths[$key] = $l[$key]; + } elseif(!isset($x[$key]) && ($e = self::baseGet($index, (int)$key)) !== null){ + $positions[$key] = $e[0]; + $lengths[$key] = $e[1]; + } + } + return [$positions, $lengths]; + } + + /** + * The same index with every key in p/l (no body): lookups at array speed, about 5× the memory of a body. + * For a segment in repeated use (plan D6: hot segments are kept decoded within the cache budget). + */ + public static function materialize(array $index): array { + if($index['kb'] === ''){ + return $index; + } + [$p, $l] = self::bodyArrays($index); + foreach($index['x'] as $key => $unused){ + unset($p[$key], $l[$key]); + } + foreach($index['p'] as $key => $offset){ + $p[$key] = $offset; + $l[$key] = $index['l'][$key]; + } + $index['p'] = $p; + $index['l'] = $l; + $index['x'] = []; + $index['kb'] = ''; + $index['kslots'] = 0; + $index['kmax'] = -1; + return $index; + } + + /** The body's live keys as [key => offset], [key => length], ascending (a few C-level calls per block) */ + private static function bodyArrays(array $index): array { + $kb = $index['kb']; + $base = $index['kbase']; + $slots = $index['kslots']; + $p = []; + $l = []; + if($index['kl'] === 'dense'){ + if($slots === 0){ + return [[], []]; + } + // The offset and length runs of every block joined, then decoded at once + $offsets = ''; + $lengths = ''; + for($block = 0, $first = 0; $first < $slots; $block++, $first += self::DENSE_SLOTS){ + $m = min(self::DENSE_SLOTS, $slots - $first); + $at = $block * self::BLOCK; + $offsets .= substr($kb, $at, 8 * $m); + $lengths .= substr($kb, $at + 8 * $m, 4 * $m); + } + // One array at a time, the runs released once used (the peak of a decode stays near its result) + $keys = range($base, $base + $slots - 1); + $l = array_combine($keys, unpack('V*', $lengths)); + $lengths = null; + if(in_array(0, $l, true)){ + $l = array_filter($l); // slots without a record (length 0) + } + $p = array_combine($keys, unpack('P*', $offsets)); + $offsets = $keys = null; + if(count($l) < $slots){ + $p = array_intersect_key($p, $l); + } + return [$p, $l]; + } + for($block = 0, $first = 0; $first < $slots; $block++, $first += self::SPARSE_ENTRIES){ + $m = min(self::SPARSE_ENTRIES, $slots - $first); + $at = $block * self::BLOCK; + $keys = []; + foreach(unpack('V' . $m, $kb, $at) as $local){ + $keys[] = $base + $local; + } + $p += array_combine($keys, unpack('P' . $m, $kb, $at + 4 * $m)); + $l += array_combine($keys, unpack('V' . $m, $kb, $at + 12 * $m)); + } + return [$p, $l]; + } + + + // ------------------------------------------------------------------ the body + + /** Live entries of the body: key => [offset, length], ascending */ + + /** Live keys of the body, ascending */ + private static function baseKeys(array $index): array { + return array_keys(self::bodyArrays($index)[1]); + } + + /** + * Encode the live keys of an index as a checkpoint body for a segment whose first key is $base: + * [layout, slots, body, highest key, live keys] + */ + public static function encodeBody(array $index, int $base): array { + [$p, $l] = self::allArrays($index); + $live = count($p); + $max = $live === 0 ? -1 : (int)array_key_last($p); + $slots = $max < $base ? 0 : $max - $base + 1; + $body = ''; + if($slots <= max(1024, 4 * $live)){ + // Every slot in key order (0 where no record), built without a loop per key + $zeros = $slots === 0 ? [] : array_fill($base, $slots, 0); + $offsets = array_values($live === $slots ? $p : array_replace($zeros, $p)); + $lengths = array_values($live === $slots ? $l : array_replace($zeros, $l)); + unset($zeros); + for($first = 0; $first < $slots; $first += self::DENSE_SLOTS){ + $m = min(self::DENSE_SLOTS, $slots - $first); + $data = pack('P*', ...array_slice($offsets, $first, $m)) . pack('V*', ...array_slice($lengths, $first, $m)); + $body .= $data . pack('V', crc32($data)); + } + return ['dense', $slots, $body, $max, $live]; + } + $keys = []; + $offsets = []; + $lengths = []; + foreach($p as $key => $offset){ + $keys[] = $key - $base; + $offsets[] = $offset; + $lengths[] = $l[$key]; + if(count($keys) === self::SPARSE_ENTRIES){ + $data = str_pad(pack('V*', ...$keys) . pack('P*', ...$offsets) . pack('V*', ...$lengths), self::BLOCK - 4, "\0"); + $body .= $data . pack('V', crc32($data)); + $keys = []; + $offsets = []; + $lengths = []; + } + } + if(!empty($keys)){ + $data = str_pad(pack('V*', ...$keys) . pack('P*', ...$offsets) . pack('V*', ...$lengths), self::BLOCK - 4, "\0"); + $body .= $data . pack('V', crc32($data)); + } + return ['sparse', $live, $body, $max, $live]; + } + + /** Every live key as [key => offset], [key => length], ascending by key (no pair per key) */ + private static function allArrays(array $index): array { + if($index['kb'] === ''){ + $p = $index['p']; + $l = $index['l']; + } else { + [$p, $l] = self::bodyArrays($index); + foreach($index['x'] as $key => $unused){ + unset($p[$key], $l[$key]); + } + foreach($index['p'] as $key => $offset){ + $p[$key] = $offset; + $l[$key] = $index['l'][$key]; + } + } + // Sorted only when not ascending already (sorting copies arrays the index shares), both by key + $previous = PHP_INT_MIN; + foreach($p as $key => $unused){ + if($key < $previous){ + ksort($p); + ksort($l); + break; + } + $previous = $key; + } + return [$p, $l]; + } + + /** A derived file's header: its JSON with its own crc as the last member (§6.5) */ + public static function sealHeader(string $json): string { + return substr($json, 0, -1) . ',"crc":' . crc32($json) . '}'; + } + + /** The members of a derived file's header, or null when its crc doesn't fit */ + public static function openHeader(string $line): ?array { + $line = rtrim($line, " \n"); + $at = strrpos($line, ',"crc":'); + if($at === false || substr($line, $at + 7, -1) !== (string)crc32(substr($line, 0, $at) . '}')){ + return null; + } + $header = json_decode($line, true); + if(!is_array($header)){ + return null; + } + unset($header['crc']); + return $header; + } + + /** The bytes of a checkpoint file: header line and body */ + public static function encodeFile(array $header, string $body): string { + return str_pad(self::sealHeader((string)json_encode($header, JSON_UNESCAPED_SLASHES)), self::HEADER - 1, ' ') . "\n" . $body; + } + + /** + * Read a checkpoint file's bytes: [header, body] when its header, body length and every block crc are + * valid (§6.4), else null + */ + public static function decodeFile($raw): ?array { + if(!is_string($raw) || strlen($raw) < self::HEADER){ + return null; + } + $header = self::headerOf(substr($raw, 0, self::HEADER)); + if($header === null || strlen($raw) !== self::HEADER + self::bodyLength($header)){ + return null; + } + $body = substr($raw, self::HEADER); + return self::bodyFits($body) ? [$header, $body] : null; + } + + /** + * The checkpoint at $path, as decodeFile() returns it, read through one handle: the header, then the + * body it announces (nothing may follow). Only the body is held, never the file and its body at once. + */ + public static function readFile(string $path): ?array { + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + try { + $header = self::headerOf((string)fread($fp, self::HEADER)); + if($header === null){ + return null; + } + $length = self::bodyLength($header); + if((fstat($fp)['size'] ?? null) !== self::HEADER + $length){ + return null; // a body the file doesn't hold is never asked for + } + $body = $length === 0 ? '' : (string)fread($fp, $length); + if(strlen($body) !== $length){ + return null; + } + return self::bodyFits($body) ? [$header, $body] : null; + } finally { + fclose($fp); + } + } + + /** The header of a checkpoint (its first HEADER bytes, ending with a newline), or null when it is not valid */ + private static function headerOf(string $head): ?array { + if(strlen($head) !== self::HEADER || $head[self::HEADER - 1] !== "\n"){ + return null; + } + $header = self::openHeader(substr($head, 0, self::HEADER - 1)); + if($header === null || ($header['kind'] ?? null) !== 'kix' || ($header['v'] ?? null) !== 1 + || !in_array($header['layout'] ?? null, ['dense', 'sparse'], true)){ + return null; + } + foreach(['base', 'slots', 'live', 'n', 'd', 't', 's'] as $member){ + if(!is_int($header[$member] ?? null) || $header[$member] < 0){ + return null; + } + } + // Its body's length is an integer, and so is every key: base + slot, or base + a local key of 32 bits + return $header['slots'] > intdiv(PHP_INT_MAX, self::BLOCK) || $header['base'] > PHP_INT_MAX - $header['slots'] - 0xFFFFFFFF ? null : $header; + } + + /** The body length a valid header announces */ + private static function bodyLength(array $header): int { + $dense = $header['layout'] === 'dense'; + $per = $dense ? self::DENSE_SLOTS : self::SPARSE_ENTRIES; + $slots = $header['slots']; + $blocks = $slots === 0 ? 0 : intdiv($slots - 1, $per) + 1; + return $blocks === 0 ? 0 : ($blocks - 1) * self::BLOCK + ($dense ? ($slots - ($blocks - 1) * $per) * 12 + 4 : self::BLOCK); + } + + /** True when every block of a body is sealed by its crc (one block at a time, no copy of the body) */ + private static function bodyFits(string $body): bool { + for($at = 0, $length = strlen($body); $at < $length; $at += self::BLOCK){ + if(crc32(substr($body, $at, self::BLOCK)) !== self::CRC_RESIDUE){ + return false; + } + } + return true; + } + + /** + * A key index from a decoded checkpoint (fits checked by the caller) + */ + public static function fromFile(array $header, string $body): array { + $index = self::emptyIndex(); + foreach(['n', 'd', 't', 's', 'created', 'updated', 'z', 'sid'] as $member){ + if(isset($header[$member])){ + $index[$member] = $header[$member]; + } + } + $index['e'] = isset($header['e']) && is_string($header['e']) ? $header['e'] : null; + $index['kb'] = $body; + $index['kl'] = $header['layout']; + $index['kbase'] = $header['base']; + $index['kslots'] = $header['slots']; + $index['live'] = $header['live']; + $index['kmax'] = self::bodyMax($index); + $index['ck'] = $header['s']; + return $index; + } + + /** + * The highest key of the body that the delta didn't remove (-1: none), from the top down: no array of the + * body is made (a deleted highest key of a large segment costs a few lookups, not a decode) + */ + private static function bodyMax(array $index): int { + if($index['kl'] === 'sparse'){ + for($i = $index['kslots'] - 1; $i >= 0; $i--){ + $block = intdiv($i, self::SPARSE_ENTRIES); + $key = $index['kbase'] + unpack('V', $index['kb'], $block * self::BLOCK + 4 * ($i - $block * self::SPARSE_ENTRIES))[1]; + if(!isset($index['x'][$key])){ + return $key; + } + } + return -1; + } + for($local = $index['kslots'] - 1; $local >= 0; $local--){ + $key = $index['kbase'] + $local; + if(!isset($index['x'][$key]) && self::baseGet($index, $key) !== null){ + return $key; + } + } + return -1; + } + + /** + * The index after a checkpoint of it was written: the new body, no delta + */ + public static function rebased(array $index, string $layout, int $slots, string $body, int $max, int $base): array { + $index['kb'] = $body; + $index['kl'] = $layout; + $index['kbase'] = $base; + $index['kslots'] = $slots; + $index['kmax'] = $max; + $index['p'] = []; + $index['l'] = []; + $index['x'] = []; + return $index; + } +} +} +namespace noneDB\Internal\Storage { + +use function bin2hex, clearstatcache, count, ctype_digit, dirname, file_get_contents, in_array, is_array, is_dir, + is_file, is_int, is_link, is_string, max, preg_match, random_bytes, rtrim, scandir, sort, strlen, substr, time; +use const LOCK_EX; + +/** + * @internal + */ +final class Layout { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $currentGenerations; + private $dbDir; + private $dbHomes; + private $durable; + private $formatRecords; + private $genPaths; + private $lockPaths; + private $shardSize; + private $lockRegistry; + private $noWrites; + private $staticCacheEnabled; + private $staticFileExistsCache; + private $staticSanitizeCache; + private $cache; + private $io; + private $locks; + private $recovery; + private $recoveryLog; + private $store; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->dbDir = &$this->k->dbDir; + $this->dbHomes = &$this->k->dbHomes; + $this->durable = &$this->k->durable; + $this->formatRecords = &$this->k->formatRecords; + $this->genPaths = &$this->k->genPaths; + $this->lockPaths = &$this->k->lockPaths; + $this->shardSize = &$this->k->shardSize; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->staticFileExistsCache = &$this->rt->staticFileExistsCache; + $this->staticSanitizeCache = &$this->rt->staticSanitizeCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Validate a database name. + * Allowed characters: A-Z a-z 0-9, space, apostrophe and hyphen. Anything else is rejected, never + * stripped: stripping would let different names collide. + * + * @param mixed $dbname Database name + * @return string The (unchanged) database name + * @throws \InvalidArgumentException + */ + public function normalizeDbName($dbname){ + if(is_int($dbname)){ + $dbname = (string)$dbname; + } + if(!is_string($dbname)){ + throw new \InvalidArgumentException('noneDB: database name must be a string'); + } + if(isset($this->staticSanitizeCache[$dbname])){ + return $dbname; + } + if($dbname === '' || preg_match("/^[A-Za-z0-9' -]+$/D", $dbname) !== 1){ + throw new \InvalidArgumentException( + "noneDB: invalid database name '{$dbname}'. Only letters A-Z/a-z, digits, space, apostrophe (') and hyphen (-) are allowed." + ); + } + if($this->staticCacheEnabled){ + $this->staticSanitizeCache[$dbname] = true; + } + return $dbname; + } + + public function dbHome($dbname){ + if(isset($this->dbHomes[$dbname])){ + return $this->dbHomes[$dbname]; + } + if(count($this->dbHomes) >= 4096){ + $this->dbHomes = $this->lockPaths = []; + } + $home = $this->dbHomes[$dbname] = $this->dbDir . $this->store->locator($dbname) . '.ndb4/'; + $this->lockPaths[$dbname] = $home . 'LOCK'; + return $home; + } + + /** + * Active generation of a database. A database without a valid CURRENT (absent, being created) uses + * generation 1: the state check in prepareDb() runs before any of its files is used. + */ + public function generationNumber($dbname){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->currentGenerations[$lock])){ + return $this->currentGenerations[$lock]; + } + $format = $this->readFormatRecord($dbname); + $current = $format === null ? null : $this->readCurrentRecord($dbname, $format); + $generation = $current === null ? 1 : $current['generation']; + if($current !== null){ + $this->currentGenerations[$lock] = $generation; + } + return $generation; + } + + public function genDir($dbname){ + $home = $this->dbHomes[$dbname] ?? $this->dbHome($dbname); + $generation = $this->currentGenerations[$this->lockPaths[$dbname]] ?? $this->generationNumber($dbname); + if(($this->genPaths[$dbname]['gen'] ?? null) !== $generation){ + // Paths of the active generation are built once: the same strings (their hashes computed once) + // serve every lookup of the caches keyed by them + if(count($this->genPaths) >= 4096){ + $this->genPaths = []; + } + $this->genPaths[$dbname] = ['gen' => $generation, 'dir' => $home . 'generations/' . $generation . '/']; + } + return $this->genPaths[$dbname]['dir']; + } + + /** + * A file of the active generation by its path relative to the generation directory (cached) + */ + public function genPath($dbname, $relative){ + // Hot path: the cached path while its generation is still the active one (no call) + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['f'][$relative]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['f'][$relative]; + } + $dir = $this->genDir($dbname); + return $this->genPaths[$dbname]['f'][$relative] ?? ($this->genPaths[$dbname]['f'][$relative] = $dir . $relative); + } + + public static function segmentName($shardId){ + return $shardId === null ? 'main' : 's' . $shardId; + } + + /** + * FORMAT of a database, validated; null when missing or invalid + */ + public function readFormatRecord($dbname){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->formatRecords[$lock])){ + return $this->formatRecords[$lock]; + } + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'FORMAT'), 'database'); + if($record !== null && !$this->validIdentity($record, $dbname)){ + $record = null; + } + if($record !== null){ + $this->formatRecords[$lock] = $record; + } + return $record; + } + + /** + * The identity members of a FORMAT or metadata record are valid for a database named $dbname (§4.1) + */ + public function validIdentity(array $record, $dbname){ + return ($record['storageFormat'] ?? null) === \noneDB::STORAGE_FORMAT && is_string($record['databaseId'] ?? null) + && ($record['name'] ?? null) === (string)$dbname && is_int($record['shardSize'] ?? null) && $record['shardSize'] > 0 + && is_int($record['createdAt'] ?? null); + } + + /** + * CURRENT, valid by §4.3 (checksum, databaseId of FORMAT, valid metadata of its generation), or null + */ + private function readCurrentRecord($dbname, array $format){ + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'CURRENT'), 'current'); + if($record === null || ($record['databaseId'] ?? null) !== $format['databaseId'] || !is_int($record['generation'] ?? null) + || $record['generation'] < 1){ + return null; + } + return $this->readGenerationMetadata($dbname, $record['generation'], $format['databaseId']) === null ? null : $record; + } + + private function readGenerationMetadata($dbname, $generation, $databaseId){ + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'generations/' . $generation . '/metadata'), 'generation'); + if($record === null || ($record['databaseId'] ?? null) !== $databaseId || ($record['generation'] ?? null) !== $generation + || !$this->validIdentity($record, $dbname) || !in_array($record['layout'] ?? null, ['single', 'sharded'], true)){ + return null; + } + return $record; + } + + public function identityOf(array $record){ + return ['storageFormat' => $record['storageFormat'], 'databaseId' => $record['databaseId'], 'name' => $record['name'], + 'shardSize' => $record['shardSize'], 'createdAt' => $record['createdAt'], + 'formatOptions' => (object)(is_array($record['formatOptions'] ?? null) ? $record['formatOptions'] : [])]; // always a JSON object + } + + public function writeGenerationMetadata($dbname, $generation, array $identity, $layout){ + $record = ['magic' => 'NONEDB4', 'kind' => 'generation'] + $identity + ['generation' => $generation, 'layout' => $layout]; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'generations/' . $generation . '/metadata', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write the metadata of generation {$generation} of database '{$dbname}'"); + } + } + + private function writeCurrentRecord($dbname, $databaseId, $generation){ + $record = ['magic' => 'NONEDB4', 'kind' => 'current', 'databaseId' => $databaseId, 'generation' => $generation]; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'CURRENT', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write CURRENT of database '{$dbname}'"); + } + $this->currentGenerations[$this->locks->getLockPath($dbname)] = $generation; + } + + private function writeFormatRecord($dbname, array $identity){ + $record = ['magic' => 'NONEDB4', 'kind' => 'database'] + $identity; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'FORMAT', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write FORMAT of database '{$dbname}'"); + } + $this->formatRecords[$this->locks->getLockPath($dbname)] = \noneDB\Internal\Storage\StoreRecord::openRecord(\noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'database'); + $this->cache->markFileExists($this->dbHome($dbname) . 'FORMAT'); + } + + /** + * True when the database exists (FORMAT valid). Cached until the lock epoch changes. + */ + public function dbPresent($dbname){ + return $this->readFormatRecord($dbname) !== null; + } + + /** + * Entries of the database directory other than LOCK (and temp files) + * @return string[] names + */ + private function dbHomeEntries($dbname){ + $entries = []; + foreach(@scandir($this->dbHome($dbname)) ?: [] as $entry){ + if($entry !== '.' && $entry !== '..' && $entry !== 'LOCK' && preg_match('/\.tmp[0-9a-f]{8}$/', $entry) !== 1){ + $entries[] = $entry; + } + } + return $entries; + } + + /** + * Resolve the state of a database directory (§8.3) under its lock; called by prepareDb(). + * Interrupted creation and drop are completed (exclusive); a damaged directory is repaired only with + * proof, otherwise FORMAT_DAMAGED. Under $noWrites nothing is changed: the state is only reported. + */ + public function resolveDatabaseState($dbname, $path){ + $flags = $this->lockRegistry[$path]['flags']; + $format = $this->readFormatRecord($dbname); + if(!empty($flags['d'])){ + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname, $path){ + $this->dropDatabaseFiles($dbname, $path); + }); + return; + } + if(!empty($flags['c'])){ + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname, $path){ + if($this->readFormatRecord($dbname) === null){ + $this->undoCreation($dbname, $path); + return; + } + unset($this->lockRegistry[$path]['flags']['c']); + $this->locks->saveLockState($path); + }); + return; + } + if($format !== null){ + if(isset($this->currentGenerations[$path])){ + return; // CURRENT was validated under this epoch + } + $current = $this->readCurrentRecord($dbname, $format); + if($current === null){ + $this->forgetDatabaseLayout($dbname); // the next operation checks again, never the hot path + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: CURRENT is missing or invalid", \noneDB::FORMAT_DAMAGED); + } + $this->currentGenerations[$path] = $current['generation']; + return; + } + if(empty($this->dbHomeEntries($dbname))){ + return; // absent: only LOCK + } + // Damaged: FORMAT missing or invalid, other entries exist. Repair only with proof (§8.3). + if($this->noWrites > 0){ + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: FORMAT is missing or invalid", \noneDB::FORMAT_DAMAGED); + } + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname){ + if($this->readFormatRecord($dbname) !== null){ + return; + } + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'CURRENT'), 'current'); + $metadata = $current === null || !is_int($current['generation'] ?? null) ? null + : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'generations/' . $current['generation'] . '/metadata'), 'generation'); + if($metadata === null || ($metadata['databaseId'] ?? null) !== ($current['databaseId'] ?? false) + || ($metadata['generation'] ?? null) !== $current['generation'] || !$this->validIdentity($metadata, $dbname)){ + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: FORMAT is missing or invalid and cannot be repaired with proof", \noneDB::FORMAT_DAMAGED); + } + $this->writeFormatRecord($dbname, $this->identityOf($metadata)); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'format-repaired', 'generation' => $current['generation']]); + }); + } + + /** + * Run a recovery step that needs the exclusive lock; under $noWrites report instead of changing + */ + private function requireExclusiveRecovery($dbname, $path, callable $fn){ + if($this->noWrites > 0){ + return; + } + if($this->lockRegistry[$path]['mode'] === LOCK_EX){ + $fn(); + return; + } + $this->locks->withLock($dbname, true, $fn, 'recovery'); + } + + /** + * Drop every cached view of a database's layout (FORMAT, CURRENT) + */ + public function forgetDatabaseLayout($dbname){ + $lock = $this->locks->getLockPath($dbname); + unset($this->currentGenerations[$lock], $this->formatRecords[$lock], $this->genPaths[$dbname]); + unset($this->staticFileExistsCache[$this->dbHome($dbname) . 'FORMAT']); + } + + /** + * Create a database (§8.1); caller holds its lock exclusively and STORE.LOCK shared. Durable mode + * (DURABILITY.md §3): "c" is synced before anything is created, every new entry with its directory, + * and the creation commits when FORMAT is durable. + * restore() (§10.3) passes the identity and $prepare($dir, 1), which builds and verifies generation 1 from + * a backup; when it fails the creation is undone at once (nothing of the database was committed). + */ + public function createDatabaseFiles($dbname, ?array $identity = null, ?callable $prepare = null){ + $path = $this->locks->getLockPath($dbname); + $this->lockRegistry[$path]['flags']['c'] = true; + $this->locks->saveLockState($path, true); + // The database directory's entry in dbDir (LOCK's entry in it goes with the next sync of the directory) + $this->sync->requireSync(null, rtrim($this->dbDir, '/'), 'dir'); + $identity = $identity ?? ['storageFormat' => \noneDB::STORAGE_FORMAT, 'databaseId' => \noneDB\Internal\Storage\StoreRecord::uuid4(), 'name' => (string)$dbname, + 'shardSize' => (int)$this->shardSize, 'createdAt' => time(), 'formatOptions' => new \stdClass()]; + $this->forgetDatabaseLayout($dbname); + $this->removeTree($this->dbHome($dbname) . 'generations'); + if($prepare === null){ + $this->writeGenerationMetadata($dbname, 1, $identity, 'single'); + // Generation 1 is complete with its empty main segment + $dataPath = $this->dbHome($dbname) . 'generations/1/segments/main.jsonl'; + if(!$this->io->fsMkdir(dirname($dataPath)) || (!$this->io->fsTouch($dataPath) && !is_file($dataPath))){ + throw new \RuntimeException("noneDB: cannot create database file for '{$dbname}'"); + } + $this->sync->requireSync(null, dirname($dataPath), 'dir'); + } else { + $dir = $this->dbHome($dbname) . 'generations/1/'; + try { + $prepare($dir, 1); + } catch(\Throwable $e){ + try { + $this->undoCreation($dbname, $path); + } catch(\Throwable $ignored){ + // "c" stays: the next exclusive session undoes the creation + } + throw $e; + } + if($this->bufferHoldsFile($dir)){ + // The copied buffer's records are visible once FORMAT is: "b" is synced before CURRENT + $this->lockRegistry[$path]['flags']['b'] = true; + $this->locks->saveLockState($path); + } + } + $this->writeCurrentRecord($dbname, $identity['databaseId'], 1); + $this->writeFormatRecord($dbname, $identity); + unset($this->lockRegistry[$path]['flags']['c']); + $this->locks->saveLockState($path); + } + + /** + * Undo an interrupted creation (§8.3): nothing of it was committed (creation writes no data, a restore's + * copy is in its backup), so what it left goes; caller holds the lock exclusively + */ + private function undoCreation($dbname, $path){ + foreach(['CURRENT', 'generations'] as $entry){ + $this->removeTree($this->dbHome($dbname) . $entry); + } + $this->forgetDatabaseLayout($dbname); + unset($this->lockRegistry[$path]['flags']['c'], $this->lockRegistry[$path]['flags']['b']); + $this->locks->saveLockState($path); + } + + /** + * Drop a database (§8.2), or finish an interrupted drop; caller holds its lock exclusively. LOCK and + * the directory stay. $request: a dropDB() call, which reports a drop that can't remove FORMAT (its + * commit point) as not committed; finishing an interrupted one throws instead. + * Durable mode: "d" (with a new epoch) is synced before FORMAT goes, the drop commits when FORMAT's + * removal is durable, and "d" is cleared only once the other removals are durable too. + * @return array ['committed' => bool, 'removed' => int files, 'left' => string[] files not removed] + */ + public function dropDatabaseFiles($dbname, $path, $request = false){ + if(empty($this->lockRegistry[$path]['flags']['d'])){ + $before = $this->lockRegistry[$path]['flags']; + $this->lockRegistry[$path]['flags']['d'] = true; + unset($this->lockRegistry[$path]['flags']['b'], $this->lockRegistry[$path]['flags']['j']); + try { + $this->locks->bumpEpoch($dbname); // writes "d" with the new epoch + } catch(\RuntimeException $e){ + if($e->getCode() === \noneDB::DURABILITY_FAILED){ + $this->withdrawDrop($dbname, $path, $before, $e); + } + throw $e; + } + } + $home = $this->dbHome($dbname); + clearstatcache(true, $home . 'FORMAT'); + $removed = 0; + if(is_file($home . 'FORMAT')){ + if(!$this->io->fsUnlink($home . 'FORMAT')){ + // The commit point was not reached: the database stays as it was + if(!$request){ + throw new \RuntimeException("noneDB: cannot finish the interrupted drop of database '{$dbname}': its FORMAT can't be removed"); + } + $this->withdrawDrop($dbname, $path, $before, null); + return ['committed' => false, 'removed' => 0, 'left' => $this->dbHomeFiles($dbname)]; + } + $removed++; + $this->forgetDatabaseLayout($dbname); + // The commit point: FORMAT's removal is durable + $this->sync->requireSync(null, rtrim($home, '/'), 'dir', true); + } + foreach(['CURRENT', 'generations', 'recovery'] as $entry){ + $removed += $this->removeTree($home . $entry); + } + // "d" goes only once these removals are durable: else a power loss could leave the files without it + $durable = !$this->durable || $this->sync->syncDirectory(rtrim($home, '/')) === true; + $this->forgetDatabaseLayout($dbname); + $this->cache->invalidateDbCaches($dbname); + $left = $this->dbHomeFiles($dbname); + if(empty($left) && $durable){ + unset($this->lockRegistry[$path]['flags']['d']); + $this->locks->saveLockState($path); + } + // Otherwise "d" stays: the database is gone, the next exclusive session removes what is left (§8.2) + return ['committed' => true, 'removed' => $removed, 'left' => $left]; + } + + /** + * A drop that did not reach its commit point: "d" must go (with FORMAT present it would finish the drop), + * durably in durable mode. If that can't be done either, the outcome is unknown. + * @throws \RuntimeException OUTCOME_UNKNOWN + */ + private function withdrawDrop($dbname, $path, array $flags, ?\Throwable $cause){ + $this->lockRegistry[$path]['flags'] = $flags; + try { + $this->locks->saveLockState($path, false, true); + } catch(\RuntimeException $e){ + throw new \RuntimeException("noneDB: the drop of database '{$dbname}' failed and its drop flag could not be withdrawn durably: the outcome is unknown, read the state again", \noneDB::OUTCOME_UNKNOWN, $cause ?? $e); + } + } + + /** + * Files of a database directory except LOCK, relative to it (a drop's leftovers) + * @return string[] + */ + private function dbHomeFiles($dbname){ + $home = $this->dbHome($dbname); + $files = []; + $walk = function($dir) use (&$walk, &$files, $home){ + foreach(@scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(is_dir($dir . $entry)){ + $walk($dir . $entry . '/'); + } elseif($dir . $entry !== $home . 'LOCK'){ + $files[] = substr($dir . $entry, strlen($home)); + } + } + }; + $walk($home); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + sort($files); + return $files; + } + + /** + * Remove a file or a directory tree of a database (fsUnlink for files, rmdir for directories) + * @return int files removed + */ + private function removeTree($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(is_file($path) || is_link($path)){ + return $this->io->fsUnlink($path) ? 1 : 0; + } + if(!is_dir($path)){ + return 0; + } + $removed = 0; + foreach(@scandir($path) ?: [] as $entry){ + if($entry !== '.' && $entry !== '..'){ + $removed += $this->removeTree($path . '/' . $entry); + } + } + $this->io->fsRmdir($path); + return $removed; + } + + /** + * Generation switch (§5.2). $prepare($dir, $generation) builds the complete new generation in $dir; + * caller holds the database lock exclusively. Returns the new generation number. + */ + public function switchGeneration($dbname, callable $prepare){ + $path = $this->locks->getLockPath($dbname); + $format = $this->readFormatRecord($dbname); + if($format === null){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist", \noneDB::FORMAT_DAMAGED); + } + $old = $this->generationNumber($dbname); + $new = $old + 1; + $newDir = $this->dbHome($dbname) . 'generations/' . $new . '/'; + $this->removeTree($newDir); // an unreferenced leftover of an earlier, interrupted PREPARE + // PREPARE: the callback builds and verifies generation $new; CURRENT still names $old. A failed + // PREPARE leaves nothing behind (the next switch would remove it anyway). + try { + $prepare($newDir, $new); + } catch(\Throwable $e){ + try { + $this->removeTree($newDir); + } catch(\Throwable $ignored){ + } + throw $e; + } + // INTENT: g=old>new and a new epoch, synced; from now on no process uses a cache of the old epoch. + // "b" when the new generation holds buffered records (a restore's), kept when the old one may + $this->lockRegistry[$path]['flags']['g'] = [$old, $new]; + if($this->bufferHoldsFile($newDir)){ + $this->lockRegistry[$path]['flags']['b'] = true; + } + $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + $this->locks->saveLockState($path, true); + // PUBLISH: the commit point (a failed directory sync after CURRENT's rename: unknown, "g" stays) + $this->writeCurrentRecord($dbname, $format['databaseId'], $new); + // CLEANUP + $oldDir = $this->dbHome($dbname) . 'generations/' . $old; + $this->removeTree($oldDir); + clearstatcache(true, $oldDir); + $cleaned = !is_dir($oldDir) && (!$this->durable || $this->sync->syncDirectory($this->dbHome($dbname) . 'generations') === true); + // DONE ("g" stays while the old generation is there or its removal may not survive a power loss: + // the next session finishes the cleanup) + if($cleaned){ + unset($this->lockRegistry[$path]['flags']['g']); + if(!$this->bufferHoldsFile($newDir)){ + unset($this->lockRegistry[$path]['flags']['b']); + } + $this->locks->saveLockState($path); + } + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $new; + return $new; + } + + /** + * restore() over a damaged database (§10.3): $prepare($dir, X) builds and verifies generation X, one above + * the highest number under generations/; then a new epoch without "g" (and "b" when X holds buffered + * records), CURRENT names X (the commit point), and an invalid FORMAT is rewritten with $identity. The + * earlier generations stay: nothing proves which of them was committed. Caller holds the lock exclusively. + * @return int X + */ + public function replaceDamagedGeneration($dbname, array $identity, callable $prepare){ + $path = $this->locks->getLockPath($dbname); + $highest = 0; + foreach(@scandir($this->dbHome($dbname) . 'generations') ?: [] as $entry){ + if(ctype_digit((string)$entry)){ + $highest = max($highest, (int)$entry); + } + } + $new = $highest + 1; + $dir = $this->dbHome($dbname) . 'generations/' . $new . '/'; + try { + $prepare($dir, $new); + } catch(\Throwable $e){ + try { + $this->removeTree($dir); + } catch(\Throwable $ignored){ + } + throw $e; + } + unset($this->lockRegistry[$path]['flags']['g'], $this->lockRegistry[$path]['flags']['b']); + if($this->bufferHoldsFile($dir)){ + $this->lockRegistry[$path]['flags']['b'] = true; + } + $this->locks->bumpEpoch($dbname); + $this->writeCurrentRecord($dbname, $identity['databaseId'], $new); + if($this->readFormatRecord($dbname) === null){ + $this->writeFormatRecord($dbname, $identity); + } + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $new; + return $new; + } + + /** + * State of a database for restore() (§10.3), once its lock resolved what it could: 'present' (FORMAT and + * CURRENT valid, no switch pending), 'absent' (no FORMAT, nothing but LOCK) or 'damaged' + */ + public function restoreTargetState($dbname){ + $format = $this->readFormatRecord($dbname); + if($format === null){ + return empty($this->dbHomeEntries($dbname)) ? 'absent' : 'damaged'; + } + $flags = $this->lockRegistry[$this->locks->getLockPath($dbname)]['flags'] ?? []; + return empty($flags['g']) && $this->readCurrentRecord($dbname, $format) !== null ? 'present' : 'damaged'; + } + + /** + * Run $fn reading generation $generation of a database as if it were the active one, under $identity, + * writing nothing: restore() verifies a generation before it is published (§10.3). Caller holds the lock + * exclusively. + */ + public function withGenerationView($dbname, $generation, array $identity, callable $fn){ + $lock = $this->locks->getLockPath($dbname); + $saved = [$this->currentGenerations[$lock] ?? null, $this->formatRecords[$lock] ?? null]; + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$lock] = $generation; + $this->formatRecords[$lock] = ['magic' => 'NONEDB4', 'kind' => 'database'] + $identity; + unset($this->genPaths[$dbname]); + $this->noWrites++; + try { + return $fn(); + } finally { + $this->noWrites--; + $this->cache->invalidateDbCaches($dbname); + unset($this->genPaths[$dbname], $this->currentGenerations[$lock], $this->formatRecords[$lock]); + if($saved[0] !== null){ + $this->currentGenerations[$lock] = $saved[0]; + } + if($saved[1] !== null){ + $this->formatRecords[$lock] = $saved[1]; + } + } + } + + /** + * True when a generation directory's buffer/ holds a pending or flushing file (§5.2: "b") + */ + private function bufferHoldsFile($dir){ + if(!is_dir($dir . 'buffer')){ + return false; + } + foreach(@scandir($dir . 'buffer') ?: [] as $entry){ + if($entry === 'pending.jsonl' || preg_match('/^flushing\.\d+\.jsonl$/', $entry) === 1){ + return true; + } + } + return false; + } + + /** + * Recovery of an interrupted generation switch (§5.2): g=N>X found; caller holds the lock exclusively + */ + public function recoverGenerationSwitch($dbname, $path){ + [$old, $new] = $this->lockRegistry[$path]['flags']['g']; + $this->forgetDatabaseLayout($dbname); + $format = $this->readFormatRecord($dbname); + $current = $format === null ? null : $this->readCurrentRecord($dbname, $format); + if($current === null || !in_array($current['generation'], [$old, $new], true)){ + $this->forgetDatabaseLayout($dbname); + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: a generation switch from {$old} to {$new} was interrupted and CURRENT is invalid; neither generation is removed (restore a backup, see docs/OPERATIONS.md)", \noneDB::FORMAT_DAMAGED); + } + $obsolete = $current['generation'] === $new ? $old : $new; + $obsoleteDir = $this->dbHome($dbname) . 'generations/' . $obsolete; + $this->removeTree($obsoleteDir); + clearstatcache(true, $obsoleteDir); + if(is_dir($obsoleteDir)){ + // CURRENT decides, so the data is consistent; what is left is reported by checkIntegrity() + // instead of being retried by every later operation + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'generation-cleanup-failed', 'committed' => $current['generation'], 'left' => $obsolete]); + } else { + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'generation-recovered', 'committed' => $current['generation'], 'removed' => $obsolete]); + } + unset($this->lockRegistry[$path]['flags']['g']); + $this->locks->saveLockState($path); + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $current['generation']; + } +} +} +namespace noneDB\Internal\Storage { + +use function array_chunk, array_key_first, array_key_last, array_keys, array_pop, explode, asort, bin2hex, clearstatcache, count, filesize, end, fclose, file_exists, file_get_contents, fopen, fread, fseek, implode, + fstat, is_array, is_int, is_string, json_decode, ksort, max, random_bytes, reset, sort, strlen, substr; +use const LOCK_SH, SEEK_SET, SORT_NUMERIC; + +/** + * @internal + */ +final class RecordIO { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $appendMismatch; + private $indexCache; + private $fieldIndexCache; + private $cacheTouch; + private $sessionLimit; + private $cacheTick; + private $cacheGrew; + private $verifiedIndexes; + private $writableIndexes; + private $lockRegistry; + private $cache; + private $checkpoints; + private $io; + private $locks; + private $segments; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->locks, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->appendMismatch = &$this->k->appendMismatch; + $this->indexCache = &$this->k->indexCache; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->lockRegistry = &$this->rt->lockRegistry; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->locks = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // RECORD I/O + // ========================================== + + /** A batch whose records lie within this many bytes is read with one call */ + private const SPAN = 262144; + + /** + * Batch read multiple JSONL records (one open, 64KB buffered sequential reads) + * @param string $path File path + * @param array $positions [key => offset, ...] + * @param array $lengths [key => length, ...] + * @param bool $mismatch Set to true when a line doesn't hold the expected record + * @return array [key => record, ...] + */ + public function readJsonlRecordsBatch($path, $positions, $lengths, &$mismatch = false, $held = null){ + $mismatch = false; + if(empty($positions)){ + return []; + } + + if(count($positions) === 1){ + // One record (a key lookup): its bytes only, in one call + $key = array_key_first($positions); + $length = (int)($lengths[$key] ?? 0); + if($length > self::SPAN){ + clearstatcache(true, $path); // a length the file doesn't hold is never asked of it, as large as it says + $length = $positions[$key] + $length > (int)@filesize($path) ? 0 : $length; + } + $json = $length <= 0 ? false : ($held !== null && $positions[$key] + $length <= strlen($held) + ? substr($held, $positions[$key], $length) : @file_get_contents($path, false, null, (int)$positions[$key], $length)); + $record = is_string($json) ? json_decode($json, true) : null; + if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ + $mismatch = true; + return []; + } + return [$key => $record]; + } + + $records = []; + $bufferSize = 65536; + $buffer = ''; + $bufferStart = -1; + $bufferEnd = -1; + + // Read in file order. Appends keep the index in that order; updated records moved to the end. + $inOrder = true; + $previous = -1; + foreach($positions as $position){ + if($position < $previous){ + $inOrder = false; + break; + } + $previous = $position; + } + if(!$inOrder){ + asort($positions, SORT_NUMERIC); + } + + // Records within one span of at most SPAN bytes (the keys of a small segment, a region): the span in + // one call. Otherwise records close together on average (a scan): 64 KB at a time; sparse ones: 64 KB + // when the next record ends within them, else the record alone (look-ahead in read order) + $first = (int)reset($positions); + $lastKey = array_key_last($positions); + $spanEnd = (int)$positions[$lastKey] + (int)($lengths[$lastKey] ?? 0); + $handle = null; + if($held !== null && $spanEnd <= strlen($held)){ + $buffer = $held; + $bufferStart = 0; + $bufferEnd = strlen($held); + } elseif($spanEnd - $first <= self::SPAN){ + $buffer = $spanEnd > $first ? @file_get_contents($path, false, null, $first, $spanEnd - $first) : ''; + if(!is_string($buffer)){ + $mismatch = true; + return []; + } + $bufferStart = $first; + $bufferEnd = $first + strlen($buffer); + } else { + $handle = @fopen($path, 'rb'); + if($handle === false){ + $mismatch = true; + return []; + } + $fileEnd = (int)(fstat($handle)['size'] ?? 0); + } + $dense = (int)end($positions) - $first <= ($bufferSize >> 2) * (count($positions) - 1); + $keys = $dense ? null : array_keys($positions); + $i = 0; + foreach($positions as $key => $offset){ + $offset = (int)$offset; + $length = (int)($lengths[$key] ?? 0); + if($length <= 0){ + $mismatch = true; + $i++; + continue; + } + + if($offset >= $bufferStart && ($offset + $length) <= $bufferEnd){ + $json = substr($buffer, $offset - $bufferStart, $length); + } elseif($handle === null || $offset + $length > $fileEnd){ + $mismatch = true; // the record ends past what was read: the data is shorter than its index + $i++; + continue; + } else { + if($dense){ + $near = true; + } else { + $next = $keys[$i + 1] ?? null; + $near = $next !== null && (int)$positions[$next] + (int)($lengths[$next] ?? 0) <= $offset + $bufferSize; + } + fseek($handle, $offset, SEEK_SET); + $buffer = (string)fread($handle, $near ? max($length, $bufferSize) : $length); + $bufferStart = $offset; + $bufferEnd = $offset + strlen($buffer); + $json = substr($buffer, 0, $length); + } + + if(!$dense){ + $i++; + } + $record = json_decode((string)$json, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ + $mismatch = true; + continue; + } + $records[$key] = $record; + } + + if($handle !== null){ + fclose($handle); + } + return $records; + } + + /** Records per batch of a read over a whole segment: the default shard size (such a segment is read at once) */ + public const BATCH = 10000; + + /** + * The records of a segment ($keys of it; every record when null) in batches of at most BATCH records, each + * [key => record] sorted by key, the batches in the order of $keys (every record: ascending). An operation over a + * segment of any size holds one batch (docs/CONFIGURATION.md "Memory of operations over many segments"). + * A segment that fits one batch: an array of that batch (nothing held, a read as it always was); a larger one: a + * generator. Iterated by reference, a caller that writes between batches sets its variable to null first, so + * the records it doesn't keep are released, not held through the write. + * @return iterable + */ + public function eachSegmentBatch($dbname, $shardId, $keys = null, $index = null){ + $index = $index ?? ($keys === null ? $this->checkpoints->decodedIndex($dbname, $shardId) : $this->checkpoints->readJsonlIndex($dbname, $shardId)); + if($index === null){ + return []; + } + if(($keys === null ? KeyIndex::count($index) : count($keys)) <= self::BATCH){ + $records = $this->readSegmentRecords($dbname, $shardId, $keys, $index); + return empty($records) ? [] : [$records]; + } + // Larger than a batch: the index's arrays once (a body unpacked in a few C-level calls, as a whole read + // does), then the keys a batch at a time + return $this->segmentBatches($dbname, $shardId, $keys, KeyIndex::materialize($index)); + } + + /** + * The batches of a segment larger than one (eachSegmentBatch()), yielded by reference. The writes of a caller + * between two batches touch only keys it was given: $index still holds the keys of the batches to come + * @return \Generator + */ + private function &segmentBatches($dbname, $shardId, $keys, array $index){ + if($keys === null){ + $keys = array_keys($index['p']); + sort($keys, SORT_NUMERIC); + } + foreach(array_chunk($keys, self::BATCH) as $chunk){ + $records = $this->readSegmentRecords($dbname, $shardId, $chunk, $index); + if(!empty($records)){ + yield $records; + } + } + } + + /** + * Read records of one segment, verified against their keys. A stale or corrupt index is + * rebuilt from the data file once. + * @param array|null $keys Keys to read (null = all) + * @param array|null $index The segment's key index the caller just read (null: read here) + * @return array [key => record] sorted by key + */ + public function readSegmentRecords($dbname, $shardId, $keys = null, $index = null){ + for($attempt = 0; $attempt < 2; $attempt++){ + $index = $attempt > 0 ? $this->checkpoints->repairSegmentIndex($dbname, $shardId, true) + : ($index ?? ($keys === null ? $this->checkpoints->decodedIndex($dbname, $shardId) : $this->checkpoints->readJsonlIndex($dbname, $shardId))); + if($index === null){ + return []; + } + if($keys !== null && $index['kb'] === ''){ + // A decoded index (the hot path): its arrays, no call per key; the lengths as they are + $positions = []; + foreach($keys as $key){ + if(isset($index['p'][$key])){ + $positions[$key] = $index['p'][$key]; + } + } + $lengths = $index['l']; + } else { + [$positions, $lengths] = KeyIndex::locate($index, $keys); + } + $dataPath = $index['dp'] ?? $this->segments->segDataPath($dbname, $shardId); + if($keys !== null && $index['kb'] !== '' && count($keys) >= 16){ + $this->checkpoints->noteLookups($dataPath, count($keys)); // bulk lookups (operations count in readJsonlIndex()) + } + if($keys === null && (int)$index['d'] === 0 && (int)$index['t'] === 0 && ($records = $this->liveLines($dataPath, $positions, (int)$index['s'])) !== null){ + return $records; + } + $mismatch = false; + $held = null; + if(empty($index['few']) && ($this->sessionLimit ?? ($this->sessionLimit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes())) > 0){ + $held = $this->heldData($dbname, $dataPath, $index); + } + $records = $this->readJsonlRecordsBatch($dataPath, $positions, $lengths, $mismatch, $held); + if(!$mismatch){ + ksort($records, SORT_NUMERIC); + return $records; + } + unset($this->fieldIndexCache[$dataPath]); + } + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + + /** + * Every record of a segment without garbage, sorted by key: its lines in file order are its records, so they are + * split at once and not cut out by position (about a tenth of such a read), 64 KB at a time. Each line is + * checked against the index as a read by position is. Null when one doesn't fit: the read by position decides. + * @param array $positions [key => offset] of every live key + * @param int $size The bytes the index covers + * @return array|null [key => record] + */ + private function liveLines($path, array $positions, $size){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $records = []; + $carry = ''; + $at = 0; + $last = -1; + $sorted = true; + for($left = $size; $left > 0; $left -= strlen($chunk)){ + $chunk = (string)fread($fp, $left < 65536 ? $left : 65536); + if($chunk === ''){ + break; + } + $lines = explode("\n", $carry . $chunk); + $carry = array_pop($lines); + foreach($lines as $line){ + $record = json_decode($line, true); + $key = $record['key'] ?? null; + if(!is_int($key) || ($positions[$key] ?? -1) !== $at){ + fclose($fp); + return null; + } + $records[$key] = $record; + $sorted = $sorted && $key > $last; + $last = $key; + $at += strlen($line) + 1; + } + } + fclose($fp); + if(count($records) !== count($positions)){ + return null; + } + if(!$sorted){ + ksort($records, SORT_NUMERIC); + } + return $records; + } + + /** A record read from a data file costs an open, a seek and a read: about what reading this many of its bytes at once does */ + private const HELD_BYTES_PER_USE = 131072; + + /** + * The bytes of a data file in repeated use, held whole within the cache's budget: a record is then a substr(), + * not an open (§ "Memory" of docs/ARCHITECTURE.md). The caller asks for a database of many segments, in a process + * with a limit; the bytes are read once the operations have paid for it, under the shared lock: bytes no rollback + * takes back. They stay while the key index keeps its epoch (the bytes it covers can't change within one), newer + * lines come from the file, and they are dropped like any cached index. + * @return string|null + */ + private function heldData($dbname, $dataPath, array $index){ + $entry = $this->fieldIndexCache[$dataPath] ?? null; + if($entry !== null){ + if($entry['e'] === $index['e']){ + $this->cacheTouch[$dataPath] = ++$this->cacheTick; + return $entry['raw']; + } + unset($this->fieldIndexCache[$dataPath]); // the file was replaced + } + $size = (int)$index['s']; + if($index['e'] === null || $index['uses'] < 2 || $index['uses'] * self::HELD_BYTES_PER_USE < $size + || ($this->lockRegistry[$this->locks->getLockPath($dbname)]['mode'] ?? null) !== LOCK_SH + || !\noneDB\Internal\Cache\CacheManager::decodeFits(2 * $size)){ + return null; + } + $raw = @file_get_contents($dataPath, false, null, 0, $size); + if(!is_string($raw) || strlen($raw) !== $size){ + return null; + } + $this->fieldIndexCache[$dataPath] = ['kind' => 'data', 'e' => $index['e'], 'raw' => $raw]; + $this->cacheTouch[$dataPath] = ++$this->cacheTick; + $this->cacheGrew = true; + return $raw; + } + + /** + * Append records ([key => record]) to a segment and point the index at them. + * Everything is encoded before anything is written, so an unencodable record writes nothing. + * @throws \InvalidArgumentException|\RuntimeException + */ + public function appendSegmentRecords($dbname, $shardId, array &$index, array $records){ + if(empty($records)){ + return; + } + $lines = $this->io->withExactFloats(function() use ($records){ + $lines = []; + foreach($records as $key => $record){ + unset($record['key']); + $record['key'] = (int)$key; + $lines[$key] = $this->io->encodeRecord($record); + } + return $lines; + }); + // After encoding: a record that can't be encoded leaves no journal entry behind + $this->locks->markIndexesPending($dbname, $shardId); + $buffer = implode("\n", $lines) . "\n"; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $offset = $this->io->appendToFile($dataPath, $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + $this->cache->markFileExists($dataPath); + $index['d'] += KeyIndex::setLines($index, $lines, $offset); // a replaced key's old line is garbage now + $index['tl'] = (int)$index['tl'] + count($lines); + } + + /** + * Append tombstones for deleted keys, so an index rebuilt from the data won't resurrect them + */ + public function appendTombstones($dbname, $shardId, array &$index, array $keys){ + if(empty($keys)){ + return; + } + $this->locks->markIndexesPending($dbname, $shardId); + $buffer = ''; + foreach($keys as $key){ + $buffer .= '{"__nonedb_deleted":' . (int)$key . "}\n"; + } + $offset = $this->io->appendToFile($this->segments->segDataPath($dbname, $shardId), $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + $index['d'] += KeyIndex::removeMany($index, $keys); // the old record lines + $index['t'] = (int)$index['t'] + count($keys); // tombstone lines + $index['tl'] = (int)$index['tl'] + count($keys); + } + + /** + * An append to a segment wrote nothing: throw. When the data file didn't have the size the index + * covers (it changed behind this process's back), the cached index is dropped as well. + * @throws \RuntimeException + */ + private function failedAppend($dbname, $shardId){ + if($this->appendMismatch){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + throw new \RuntimeException("noneDB: data file of database '{$dbname}' changed unexpectedly; nothing was written, its index is rebuilt on the next access"); + } + throw new \RuntimeException("noneDB: failed to write data of database '{$dbname}'"); + } + + /** + * Create an empty segment (data file + index) + * @param string $dbname + * @param int|null $shardId + * @return bool + */ + public function createJsonlDatabase($dbname, $shardId = null){ + $path = $this->segments->segDataPath($dbname, $shardId); + clearstatcache(true, $path); + if(!file_exists($path) && !$this->io->fsTouch($path)){ + throw new \RuntimeException("noneDB: cannot create data file of database '{$dbname}'"); + } + $this->cache->markFileExists($path); + $index = $this->checkpoints->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = $index; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = true; + // No checkpoint yet (ck -1): the session writes one when it ends, with the first records (until then + // the new segment replays from 0, which is the same index) + $this->lockRegistry[$this->locks->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + return true; + } +} +} +namespace noneDB\Internal\Storage { + +use function array_column, array_map, clearstatcache, count, file_exists, in_array, intdiv, preg_match, scandir, + strlen, strncmp, strrpos, substr; + +/** + * @internal + */ +final class Segments { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $currentGenerations; + private $genPaths; + private $lockPaths; + private $shardSize; + private $shardedCache; + private $layout; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->layout, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->genPaths = &$this->k->genPaths; + $this->lockPaths = &$this->k->lockPaths; + $this->shardSize = &$this->k->shardSize; + $this->shardedCache = &$this->k->shardedCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->layout = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Path of the main data file (single layout) + */ + public function getDataPath($dbname){ + return $this->segDataPath($dbname, null); + } + // ========================================== + // STORAGE CORE + // ========================================== + // + // A database is one "segment" in the single layout (segments/main.jsonl + main.kix) or one segment per + // shard (segments/s{N}.jsonl + s{N}.kix) listed in shards.json, inside its active generation + // ({locator}.ndb4/generations/{N}/, docs/STORAGE_FORMAT.md §5). Segment ids are null (single) or the + // shard id. Every write runs under the database's exclusive lock (withLock), reads under a shared + // lock, so index read-modify-write cycles are never interleaved. + // + // Data files are append-only JSONL. Every record line carries its "key"; deletions append a + // tombstone line {"__nonedb_deleted":key}. The .kix checkpoint maps key => offset/length and stores + // "s", the data size it covers; loading replays the complete lines after it (Checkpoints, KeyIndex). + + /** + * Files of a directory whose path starts with $prefix (a full path: directory + start of the name) and + * whose remaining name matches $regex (if given), sorted. Not glob(): a path containing [ ] * ? is a + * pattern to glob() and matches nothing. + * @return string[] + */ + public function listFilesWithPrefix($prefix, $regex = null){ + $slash = strrpos($prefix, '/'); + $dir = substr($prefix, 0, $slash + 1); + $start = substr($prefix, $slash + 1); + $length = strlen($start); + $files = []; + foreach(@scandir($dir) ?: [] as $file){ + if($file !== '.' && $file !== '..' && strncmp($file, $start, $length) === 0 + && ($regex === null || preg_match($regex, substr($file, $length)) === 1)){ + $files[] = $dir . $file; + } + } + return $files; + } + + /** + * Data file of a shard + */ + public function getShardPath($dbname, $shardId){ + return $this->segDataPath($dbname, (int)$shardId); + } + + /** + * Data file of a segment (null = single layout) + */ + public function segDataPath($dbname, $shardId){ + $k = $shardId ?? -1; + // Hot path: the cached path while its generation is still the active one (no call) + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['seg'][$k]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['seg'][$k]; + } + $dir = $this->layout->genDir($dbname); + return $this->genPaths[$dbname]['seg'][$k] ?? ($this->genPaths[$dbname]['seg'][$k] = $dir . 'segments/' . \noneDB\Internal\Storage\Layout::segmentName($shardId) . '.jsonl'); + } + + /** + * Check if a database is sharded + * @param string $dbname + * @return bool + */ + public function isSharded($dbname){ + $metaPath = $this->layout->genPath($dbname, 'shards.json'); // ShardMeta::getMetaPath(), without the hop + if(isset($this->shardedCache[$metaPath])){ + return $this->shardedCache[$metaPath]; + } + + clearstatcache(true, $metaPath); + $result = file_exists($metaPath); + $this->shardedCache[$metaPath] = $result; + return $result; + } + + /** + * True when the database exists (STORAGE_FORMAT.md §8.3: FORMAT valid) + */ + public function dbExists($dbname){ + return $this->layout->dbPresent($dbname); + } + + /** + * Records per shard of a database: fixed in FORMAT when the database was created + */ + public function shardSizeOf($dbname){ + $format = $this->layout->readFormatRecord($dbname); + return $format !== null ? (int)$format['shardSize'] : (int)$this->shardSize; + } + + /** + * Segment ids of a database: [null] when not sharded, the shard ids otherwise, [] if it doesn't exist + */ + public function listSegments($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + return array_map('intval', array_column($meta['shards'], 'id')); + } + return $this->layout->dbPresent($dbname) ? [null] : []; + } + + /** Segments of a database without listing them (0: no database) */ + public function segmentCount($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + return $meta === null ? 0 : count($meta['shards']); + } + return $this->layout->dbPresent($dbname) ? 1 : 0; + } + + /** A database of at most this many segments decodes the indexes it uses at once (every segment is hot) */ + public const FEW_SEGMENTS = 16; + + /** + * Segment that holds a key, or false if that shard doesn't exist + * @return int|null|false + */ + public function segmentForKey($dbname, $key){ + if(!$this->isSharded($dbname)){ + return null; + } + $shardId = intdiv($key, $this->shardSizeOf($dbname)); + return in_array($shardId, $this->listSegments($dbname), true) ? $shardId : false; + } +} +} +namespace noneDB\Internal\Storage { + +use function array_is_list, array_slice, array_unique, array_values, bin2hex, chr, clearstatcache, count, + dirname, fclose, fflush, file_get_contents, file_put_contents, flock, fopen, fwrite, hash, hash_equals, + hash_hmac, implode, in_array, is_array, is_dir, is_file, is_int, is_link, is_string, json_decode, json_encode, + ord, preg_match, random_bytes, scandir, sort, strlen, substr, time, unlink; +use const FILE_APPEND, JSON_PRESERVE_ZERO_FRACTION, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE, LOCK_EX, + LOCK_NB, LOCK_SH, LOCK_UN, PREG_OFFSET_CAPTURE, SORT_STRING; + +/** + * @internal + */ +final class StoreRecord { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $dbDir; + private $hashCache; + private $maintenanceDepth; + private $metrics; + private $noWrites; + private $storeLocks; + private $storeState; + private $cache; + private $config; + private $io; + private $locks; + private $recovery; + private $sync; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->config, $this->io, $this->locks, $this->recovery, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->hashCache = &$this->k->hashCache; + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->storeLocks = &$this->rt->storeLocks; + $this->storeState = &$this->rt->storeState; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->config = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * keyCheck of the configured key: identifies it in STORE without revealing it + */ + public function keyCheck(){ + return substr(hash_hmac('sha256', 'nonedb4/key', (string)$this->config->getSecretKey()), 0, 32); + } + + public static function keyCheckOf($key){ + return substr(hash_hmac('sha256', 'nonedb4/key', (string)$key), 0, 32); + } + + /** + * Directory name of a database for the configured key (stable, unpredictable without the key) + */ + public function locator($dbname){ + $cacheKey = $this->cache->getKeyFingerprint() . '|' . $dbname; + return $this->hashCache[$cacheKey] ?? ($this->hashCache[$cacheKey] = self::locatorOf($dbname, (string)$this->config->getSecretKey())); + } + + public static function locatorOf($dbname, $key){ + return substr(hash_hmac('sha256', 'nonedb4/db/' . $dbname, (string)$key), 0, 32); + } + + /** + * Encode a metadata record with its checksum (§2) + */ + public static function sealRecord(array $record){ + unset($record['checksum']); + $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION; + $json = json_encode($record, $flags); + if($json === false){ + throw new \RuntimeException('noneDB: cannot encode a metadata record'); + } + $record['checksum'] = hash('sha256', $json . "\n"); + return json_encode($record, $flags) . "\n"; + } + + /** + * Decode and verify a metadata record: null when it is not a valid record of $kind. The checksum is + * checked on the bytes as written (the record without its last member "checksum"), not on a + * re-encoding, which could differ (an empty object decodes to an empty array). + */ + public static function openRecord($raw, $kind){ + if(!is_string($raw) || $raw === '' || substr($raw, -1) !== "\n" + || preg_match('/,"checksum":"([0-9a-f]{64})"\}\n\z/', $raw, $m, PREG_OFFSET_CAPTURE) !== 1){ + return null; + } + $body = substr($raw, 0, $m[0][1]) . "}\n"; + if(!hash_equals(hash('sha256', $body), $m[1][0])){ + return null; + } + $record = json_decode(substr($raw, 0, -1), true); + if(!is_array($record) || ($record['magic'] ?? null) !== 'NONEDB4' || ($record['kind'] ?? null) !== $kind){ + return null; + } + unset($record['checksum']); + return $record; + } + + /** + * One STORE copy: [exists, record or null, raw bytes or null] + */ + public function readStoreCopy($name){ + $path = $this->dbDir . $name; + $raw = @file_get_contents($path); + if($raw === false){ + clearstatcache(true, $path); + return [is_file($path), null, null]; + } + return [true, $this->validStoreRecord($raw), $raw]; + } + + private function validStoreRecord($raw){ + $record = self::openRecord($raw, 'store'); + $valid = $record !== null && ($record['storageFormat'] ?? null) === \noneDB::STORAGE_FORMAT + && is_string($record['storeId'] ?? null) && is_int($record['seq'] ?? null) && $record['seq'] >= 1 + && is_array($record['keys'] ?? null) && array_is_list($record['keys']); + if($valid){ + $keys = $record['keys']; + foreach($keys as $keyCheck){ + if(!is_string($keyCheck) || preg_match('/^[0-9a-f]{32}$/', $keyCheck) !== 1){ + $valid = false; + } + } + $sorted = array_values(array_unique($keys)); + sort($sorted, SORT_STRING); + $valid = $valid && $sorted === $keys; + } + return $valid ? $record : null; + } + + /** + * The store record by §3.1, null when there is no store. One valid copy repairs the other (under + * STORE.LOCK; not while this process holds it shared: the next open repairs it); two valid copies that + * differ, or none valid, are FORMAT_DAMAGED. + */ + public function loadStoreRecord(){ + [$exists1, $copy1, $raw1] = $this->readStoreCopy('STORE'); + $raw2 = $raw1 === null ? false : @file_get_contents($this->dbDir . 'STORE.2'); + if($raw1 !== null && $raw2 === $raw1){ + $copy2 = $copy1; // the same bytes: one check + $exists2 = true; + } else { + [$exists2, $copy2] = $raw2 === false ? $this->readStoreCopy('STORE.2') : [true, $this->validStoreRecord($raw2)]; + } + if(!$exists1 && !$exists2){ + return null; + } + if($copy1 !== null && $copy2 !== null){ + if($copy1 === $copy2){ + return $copy1; + } + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged (its two copies differ)", \noneDB::FORMAT_DAMAGED); + } + $valid = $copy1 ?? $copy2; + if($valid === null){ + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged; see repairStore()", \noneDB::FORMAT_DAMAGED); + } + $held = $this->storeLocks[$this->dbDir]['mode'] ?? null; + if($this->noWrites === 0 && $held !== LOCK_SH){ + // The copies are identical outside maintenance (§3.1): the valid one restores the other + $this->withStoreLock(LOCK_EX, function() use ($valid){ + [, $again1] = $this->readStoreCopy('STORE'); + [, $again2] = $this->readStoreCopy('STORE.2'); + if($again1 === null || $again2 === null){ + $this->writeStoreCopies($valid); + $this->logStoreEvent(['action' => 'store-copy-repaired', 'seq' => $valid['seq']]); + } + }); + } + return $valid; + } + + /** + * Write both STORE copies (STORE.2 first) by atomic replacement; caller holds STORE.LOCK exclusively + */ + public function writeStoreCopies(array $record){ + $content = self::sealRecord($record); + foreach(['STORE.2', 'STORE'] as $name){ + if(!$this->writeStoreFile($this->dbDir . $name, $content)){ + throw new \RuntimeException("noneDB: cannot write the store record {$name} in {$this->dbDir}"); + } + } + } + + /** + * Atomic replacement of a store-level file (STORE copies, journal): temp + rename, synced in durable + * mode (temp file, then the directory after the rename: the commit point of the file). + * Store files belong to no database: STORE.LOCK, not a database lock, protects them. + * @return bool False when the file could not be written (nothing changed) + * @throws \RuntimeException DURABILITY_FAILED (temp file sync, nothing changed), OUTCOME_UNKNOWN + * (directory sync after the rename) + */ + public function writeStoreFile($path, $content){ + $this->recovery->removeStaleTemp($path); // a crashed replacement's temp (§2): STORE.LOCK is held exclusively + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + $fp = @fopen($tmp, 'xb'); + if($fp === false){ + return false; + } + $this->io->setPermissions($tmp); + $ok = fwrite($fp, $content) === strlen($content) && fflush($fp); + $synced = $ok && $this->sync->trySync($fp, $tmp); + fclose($fp); + if(!$ok || !$synced){ + @unlink($tmp); + if($ok){ + throw $this->sync->failure('file', $path, false); + } + return false; + } + if(!$this->sync->renameWithRetry($tmp, $path)){ + @unlink($tmp); + return false; + } + clearstatcache(true, $path); + $this->sync->requireSync(null, dirname($path), 'dir', true); + return true; + } + + /** + * Store-level recovery events (STORE copy repairs, manual rebuilds): .nonedb-maintenance/log.jsonl + */ + public function logStoreEvent(array $event){ + $dir = $this->dbDir . '.nonedb-maintenance'; + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + return; + } + $created = !is_file($dir . '/log.jsonl'); + if(@file_put_contents($dir . '/log.jsonl', json_encode(['t' => time()] + $event, JSON_UNESCAPED_SLASHES) . "\n", FILE_APPEND | LOCK_EX) !== false){ + if($created){ + $this->io->setPermissions($dir . '/log.jsonl'); + } + $this->metrics['recoveryEvents']++; + $this->metrics['recoveryActions'][$event['action']] = ($this->metrics['recoveryActions'][$event['action']] ?? 0) + 1; + } + } + + /** + * Run $fn holding STORE.LOCK (re-entrant within the process; a shared hold is never upgraded) + */ + public function withStoreLock($mode, callable $fn, $nonBlocking = false){ + $dir = $this->dbDir; + if(isset($this->storeLocks[$dir])){ + if($mode === LOCK_EX && $this->storeLocks[$dir]['mode'] !== LOCK_EX){ + throw new \LogicException('noneDB: STORE.LOCK is held shared and cannot be upgraded'); + } + $this->storeLocks[$dir]['depth']++; + try { + return $fn(); + } finally { + $this->storeLocks[$dir]['depth']--; + } + } + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + throw new \RuntimeException("noneDB: cannot create database directory {$dir}"); + } + $created = !is_file($dir . 'STORE.LOCK'); + $fp = @fopen($dir . 'STORE.LOCK', 'c+be'); + if($fp !== false && $created){ + $this->io->setPermissions($dir . 'STORE.LOCK'); + } + if($fp === false){ + throw new \RuntimeException("noneDB: cannot open the store lock in {$dir}", \noneDB::LOCK_FAILED); + } + try { + if($nonBlocking){ + if(!flock($fp, $mode | LOCK_NB)){ + throw new \RuntimeException("noneDB: the store in {$dir} is in use", \noneDB::MAINTENANCE_BUSY); + } + } else { + $this->locks->acquireFlock($fp, $mode, 'STORE'); + } + } catch(\Throwable $e){ + fclose($fp); + throw $e; + } + $this->storeLocks[$dir] = ['fp' => $fp, 'mode' => $mode, 'depth' => 1]; + try { + return $fn(); + } finally { + unset($this->storeLocks[$dir]); + flock($fp, LOCK_UN); + fclose($fp); + } + } + + /** + * Entries of dbDir that are not noneDB 4 objects (§3.4). $withStore: the allow-list of a store that + * has a STORE record; without one only the files an empty location may hold are allowed (§3.2). + * @return string[] + */ + private function foreignEntries($withStore){ + $foreign = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(in_array($entry, ['.htaccess', 'index.html', '.gitkeep', '.gitignore', '.DS_Store', 'STORE.LOCK'], true) + || preg_match('/^STORE(\.2)?\.tmp[0-9a-f]{8}$/', $entry) === 1){ + continue; + } + if($withStore && (in_array($entry, ['STORE', 'STORE.2', '.nonedb-maintenance', '.nonedb-retained'], true) + || (preg_match('/^[0-9a-f]{32}\.ndb4$/', $entry) === 1 && is_dir($this->dbDir . $entry) && !is_link($this->dbDir . $entry)))){ + continue; // a database directory; a link or a file of that name is foreign (§3.4) + } + $foreign[] = $entry; + } + return $foreign; + } + + /** + * MAINTENANCE_IN_PROGRESS while a maintenance journal is open (except in the process running it) + */ + public function checkMaintenanceJournal(){ + if($this->maintenanceDepth > 0){ + return; + } + $journal = $this->dbDir . '.nonedb-maintenance/journal.json'; + clearstatcache(true, $journal); + if(is_file($journal)){ + throw new \RuntimeException("noneDB: maintenance of the store in {$this->dbDir} is in progress or was interrupted; run it again to finish", \noneDB::MAINTENANCE_IN_PROGRESS); + } + } + + /** + * Open the store for this key (§3.2). $create: create the store when the location is empty. + * @return bool false when there is no store (and $create is false): nothing exists yet + */ + public function openStore($create){ + $dir = $this->dbDir; + $keyCheck = $this->keyCheck(); + if(isset($this->storeState[$dir][$keyCheck])){ + return true; + } + $this->checkMaintenanceJournal(); + $record = $this->loadStoreRecord(); + if($record === null && !empty($this->foreignEntries(false))){ + // The entries may belong to a store another process created while this one looked: decided + // again under STORE.LOCK, which every store creation holds + $record = $this->storeRecordUnderExistingLock(); + if($record === null){ + throw $this->missingStoreError($this->foreignEntries(false)); + } + } + if($record === null){ + if(!$create || $this->noWrites > 0){ + return false; + } + $this->withStoreLock(LOCK_EX, function() use ($keyCheck, &$record){ + $record = $this->loadStoreRecord(); + if($record !== null){ + return; + } + $foreign = $this->foreignEntries(false); + if(!empty($foreign)){ + throw $this->missingStoreError($foreign); + } + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^STORE(\.2)?\.tmp[0-9a-f]{8}$/', $entry) === 1){ + @unlink($this->dbDir . $entry); + } + } + $record = ['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT, + 'storeId' => self::uuid4(), 'seq' => 1, 'keys' => [$keyCheck]]; + $this->writeStoreCopies($record); + foreach(['.htaccess' => "Require all denied\nDeny from all\n", 'index.html' => ''] as $name => $content){ + if(@file_put_contents($this->dbDir . $name, $content) !== false){ + $this->io->setPermissions($this->dbDir . $name); + } + } + }); + } + if(!in_array($keyCheck, $record['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$dir}", \noneDB::KEY_NOT_REGISTERED); + } + $this->storeState[$dir][$keyCheck] = true; + return true; + } + + /** + * The store record read under STORE.LOCK held shared, when STORE.LOCK exists (it is never created + * here: a refused location stays unchanged); null without a store + */ + private function storeRecordUnderExistingLock(){ + $dir = $this->dbDir; + if(isset($this->storeLocks[$dir])){ + return $this->loadStoreRecord(); + } + $fp = @fopen($dir . 'STORE.LOCK', 'rb'); + if($fp === false){ + return null; + } + try { + $this->locks->acquireFlock($fp, LOCK_SH, 'STORE'); + $this->storeLocks[$dir] = ['fp' => $fp, 'mode' => LOCK_SH, 'depth' => 1]; + try { + return $this->loadStoreRecord(); + } finally { + unset($this->storeLocks[$dir]); + flock($fp, LOCK_UN); + } + } finally { + fclose($fp); + } + } + + /** + * The missing-database path (§3.3): run $fn holding STORE.LOCK shared after the store was re-read, + * the key re-checked and dbDir listed for foreign entries. $fn decides with that view. + */ + public function withStoreRevalidated($create, callable $fn){ + if(!$this->storeExists() && !$this->openStore($create)){ + return $fn(); // no store and nothing in dbDir: nothing exists + } + return $this->withStoreLock(LOCK_SH, function() use ($fn){ + // Checked again under the lock: a maintenance may have changed the store since it was opened + $this->checkMaintenanceJournal(); + $record = $this->loadStoreRecord(); + if($record === null || !in_array($this->keyCheck(), $record['keys'], true)){ + unset($this->storeState[$this->dbDir]); + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + $foreign = $this->foreignEntries(true); + if(!empty($foreign)){ + throw $this->unsupportedFormat($foreign); + } + return $fn(); + }); + } + + /** + * No STORE copy but entries: noneDB 4 objects without their store record are a damaged store, + * anything else is not a noneDB 4 store + */ + private function missingStoreError(array $foreign){ + foreach($foreign as $entry){ + if(preg_match('/^[0-9a-f]{32}\.ndb4$/', $entry) === 1 || $entry === '.nonedb-maintenance' || $entry === 'STORE' || $entry === 'STORE.2'){ + return new \RuntimeException("noneDB: the store record in {$this->dbDir} is missing; see repairStore()", \noneDB::FORMAT_DAMAGED); + } + } + return $this->unsupportedFormat($foreign); + } + + /** + * True when this process opened the store for its key (it exists and the key is registered) + */ + public function storeExists(){ + return isset($this->storeState[$this->dbDir][$this->keyCheck()]); + } + + private function unsupportedFormat(array $foreign){ + $shown = array_slice($foreign, 0, 5); + return new \RuntimeException("noneDB: {$this->dbDir} contains files that are not noneDB 4 objects (" + . implode(', ', $shown) . (count($foreign) > 5 ? ', ...' : '') . "): no database is created or answered as missing while they are there (docs/STORAGE_FORMAT.md §3.4)", + \noneDB::UNSUPPORTED_FORMAT); + } + + public static function uuid4(){ + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } +} +} +namespace noneDB\Internal\Storage { + +use function array_intersect_key, array_keys, array_values, count, max; + +/** + * @internal + */ +final class Writer { + /** @var \noneDB\Internal\Core\Kernel */ + private $k; + /** @var \noneDB\Internal\Cache\Runtime */ + private $rt; + + private $autoCreateDB; + private $shardingEnabled; + private $checkpoints; + private $compaction; + private $db; + private $findFilter; + private $layoutSwitch; + private $records; + private $scope; + private $segments; + private $shardMeta; + private $shardRouter; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->compaction, $this->db, $this->findFilter, $this->layoutSwitch, $this->records, $this->scope, $this->segments, $this->shardMeta, $this->shardRouter); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->shardingEnabled = &$this->k->shardingEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->compaction = null; + $this->db = null; + $this->findFilter = null; + $this->layoutSwitch = null; + $this->records = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->shardRouter = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE PATHS (callers hold the exclusive lock) + // ========================================== + + /** + * Insert validated records. Assigns keys, writes data + indexes, updates field/spatial indexes + * and switches the database to the sharded layout once it has assigned shardSize keys (shardingEnabled). + * @param array $items Records without "key" + * @return array ['n' => int] or ['n' => 0, 'error' => string] + */ + public function insertRecords($dbname, array $items, $startKey = null){ + if(empty($items)){ + return ["n" => 0]; + } + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->db->createDBUnlocked($dbname); + } + + // With a start key the records get keys startKey, startKey+1, ... and keys that are already + // stored are skipped, so replaying the same insert after a crash never duplicates anything. + $keyed = null; + $endKey = null; + if($startKey !== null){ + $keyed = []; + foreach(array_values($items) as $i => $item){ + $keyed[(int)$startKey + $i] = $item; + } + $endKey = (int)$startKey + count($items); + foreach(array_keys($this->findFilter->selectRecords($dbname, ['key' => array_keys($keyed)])) as $existing){ + unset($keyed[$existing]); + } + } + + if($this->segments->isSharded($dbname)){ + $this->shardRouter->insertShardedRecords($dbname, $items, $keyed, $endKey); + return ["n" => count($items)]; + } + + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + if($index === null){ + $this->records->createJsonlDatabase($dbname, null); + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + } + + if($keyed !== null){ + $records = $keyed; + $index['n'] = max((int)$index['n'], $endKey); + } else { + $records = []; + foreach($items as $item){ + $records[(int)$index['n']] = $item; + $index['n']++; + } + } + $this->records->appendSegmentRecords($dbname, null, $index, $records); + $this->checkpoints->storeSegmentIndex($dbname, null, $index); + + if($this->shardingEnabled && $index['n'] >= $this->segments->shardSizeOf($dbname)){ + // Replaces the data file: after this write committed + $this->scope->runIndependent($dbname, 'convertToSharded', function() use ($dbname){ + return $this->layoutSwitch->convertToSharded($dbname); + }); + } + return ["n" => count($items)]; + } + + /** + * Replace records ([key => full new record]) in one segment + * @return int Number of records written + */ + public function updateSegmentRecords($dbname, $shardId, array $newRecords){ + if(empty($newRecords)){ + return 0; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + foreach(array_keys($newRecords) as $key){ + if(!KeyIndex::has($index, $key)){ + unset($newRecords[$key]); + } + } + if(empty($newRecords)){ + $this->checkpoints->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $this->records->appendSegmentRecords($dbname, $shardId, $index, $newRecords); // counts the replaced lines + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + $this->compaction->deferCompaction($dbname, $shardId, $index); + return count($newRecords); + } + + /** + * Delete records ([key => record]) from one segment + * @return int Number of records deleted + */ + public function deleteSegmentRecords($dbname, $shardId, array $oldRecords){ + if(empty($oldRecords)){ + return 0; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + if($index['kb'] === ''){ + $oldRecords = array_intersect_key($oldRecords, $index['p']); // a decoded index: one call + } else { + foreach(array_keys($oldRecords) as $key){ + if(!KeyIndex::has($index, $key)){ + unset($oldRecords[$key]); + } + } + } + if(empty($oldRecords)){ + $this->checkpoints->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $keys = array_keys($oldRecords); + $this->records->appendTombstones($dbname, $shardId, $index, $keys); // drops the keys, counts the lines + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + + if($shardId !== null){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + if((int)$shard['id'] === $shardId){ + $meta['shards'][$i]['count'] = max(0, $meta['shards'][$i]['count'] - count($keys)); + $meta['shards'][$i]['deleted'] += count($keys); + } + } + $meta['totalRecords'] = max(0, (int)$meta['totalRecords'] - count($keys)); + $meta['deletedCount'] = (int)($meta['deletedCount'] ?? 0) + count($keys); + $this->shardMeta->writeMeta($dbname, $meta); + } + + $this->compaction->deferCompaction($dbname, $shardId, $index); + return count($keys); + } +} +} +namespace { +class noneDB { + /** This library's version */ + public const VERSION = '4.0.0-rc.6'; + /** The storage format it reads and writes (docs/STORAGE_FORMAT.md) */ + public const STORAGE_FORMAT = 4; + + // Exception codes (RuntimeException::getCode(), docs/STORAGE_FORMAT.md §11, docs/DURABILITY.md §2) + /** A write failed after its commit point, or undoing it failed: the outcome is unknown; read the state again (the next access recovers) */ + public const OUTCOME_UNKNOWN = 3301; + /** The database lock could not be taken within lockTimeout, or was lost */ + public const LOCK_FAILED = 3302; + /** dbDir holds entries that are not noneDB 4 objects: no database is created or answered as missing while they are there */ + public const UNSUPPORTED_FORMAT = 3304; + /** A store record, FORMAT, CURRENT or generation is damaged and can't be repaired with proof */ + public const FORMAT_DAMAGED = 3305; + /** 'durability' => 'durable' on a platform that can't provide it (only Linux can) */ + public const UNSUPPORTED_DURABILITY = 3306; + /** A sync durable mode requires failed before the commit point: the write was rolled back */ + public const DURABILITY_FAILED = 3307; + /** The configured secretKey is not registered for the store */ + public const KEY_NOT_REGISTERED = 3308; + /** A key maintenance of the store is running or was interrupted (run it again) */ + public const MAINTENANCE_IN_PROGRESS = 3309; + /** restore() refused the backup (incomplete or damaged, of another database without force, another shard size): nothing was changed */ + public const RESTORE_REFUSED = 3310; + /** A key maintenance found a database in use: nothing was changed */ + public const MAINTENANCE_BUSY = 3311; + + /** @var int key of this instance's kernel in Kernel::$instances */ + private $id; + + /** + * @param array|null $config Settings (docs/CONFIGURATION.md), merged over a .nonedb file when one is found; null: + * the file alone (or dev mode) + * @throws \InvalidArgumentException for an invalid setting or secretKey + * @throws \RuntimeException UNSUPPORTED_DURABILITY for 'durable' where it can't be provided + */ + public function __construct($config = null){ + $kernel = new \noneDB\Internal\Core\Kernel($config); + $this->id = spl_object_id($this); + \noneDB\Internal\Core\Kernel::$instances[$this->id] = $kernel; + } + + /** The main settings for var_dump() and print_r(), the secret masked */ + public function __debugInfo(){ + $kernel = \noneDB\Internal\Core\Kernel::$instances[$this->id] ?? null; + return $kernel === null ? [] : $kernel->config->debugInfo(); + } + + /** + * Instances hold a secret and open resources; they can't be serialized + * @throws \LogicException + */ + public function __serialize(): array { + throw new \LogicException('noneDB instances cannot be serialized'); + } + + public function __unserialize(array $data): void { + throw new \LogicException('noneDB instances cannot be unserialized'); + } + + public function __clone(){ + $source = \noneDB\Internal\Core\Kernel::$instances[$this->id]; + $this->id = spl_object_id($this); + \noneDB\Internal\Core\Kernel::$instances[$this->id] = $source->cloneKernel(); + } + + public function __destruct(){ + $kernel = \noneDB\Internal\Core\Kernel::$instances[$this->id] ?? null; + if($kernel !== null){ + unset(\noneDB\Internal\Core\Kernel::$instances[$this->id]); + $kernel->config->destroyed(); + $kernel->release(); + } + } + + // ========================================== + // QUERY BUILDER (FLUENT INTERFACE) + // ========================================== + + /** + * Create a new query builder for fluent interface + * @param string $dbname + * @return noneDBQuery + */ + public function query(string $dbname): noneDBQuery { + return new noneDBQuery($this, \noneDB\Internal\Core\Kernel::$instances[$this->id]->layout->normalizeDbName($dbname)); + } + + /** + * Check if config file exists + * @return bool + */ + public static function configExists(): bool{ + return \noneDB\Internal\Config\Configuration::configExists(); + } + + + /** + * Get the config file template path + * @return string|null + */ + public static function getConfigTemplate(): ?string{ + return \noneDB\Internal\Config\Configuration::getConfigTemplate(); + } + + + /** + * Clear config cache (useful for testing) + * @return void + */ + public static function clearConfigCache(): void{ + \noneDB\Internal\Config\Configuration::clearConfigCache(); + } + + + /** + * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) + * @return bool + */ + public static function isDevMode(): bool{ + return \noneDB\Internal\Config\Configuration::isDevMode(); + } + + + /** + * Set development mode programmatically + * Useful for testing or when env vars are not available + * @param bool $enabled + * @return void + */ + public static function setDevMode(bool $enabled): void{ + \noneDB\Internal\Config\Configuration::setDevMode($enabled); + } + + + /** + * Clear all static caches (useful for testing or memory management) + * @return void + */ + public static function clearStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + } + + + /** + * Disable static caching (each instance uses its own cache) + * @return void + */ + public static function disableStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::disableStaticCache(); + } + + + /** + * Enable static caching (default) + * @return void + */ + public static function enableStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::enableStaticCache(); + } + + + /** + * Durability of this instance and of the last write operation of the process (docs/DURABILITY.md §6): + * - mode: 'performance' or 'durable' + * - platform: ['os', 'durableSupported', 'notes'] + * - lastOperation: null before the first write, else ['op', 'db', 'outcome' => 'committed'|'rolled_back'|'unknown', + * 'required' => syncs the mode needed, 'completed' => syncs that succeeded, 'reason' => ?string] + * - lastSyncError: null or ['path' (relative to dbDir), 'op' => 'file'|'dir', 'time'] + * - syncs: ['file', 'dir', 'failed'] counters of the process + * @return array + */ + public function getDurability(): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->sync->getDurability(); + } + + // ========================================== + // PUBLIC INDEX API + // ========================================== + + /** + * Make the key indexes (.kix checkpoints) of a database again from its data files, then its field and + * spatial indexes where they changed. + * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. + * @param string $dbname + * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] + */ + public function rebuildIndex($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->rebuildIndex($dbname); + } + + + /** + * Get index information for a database + * @param string $dbname + * @return array|null + */ + public function getIndexInfo($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->getIndexInfo($dbname); + } + + + /** + * Check a database against its own data, read-only: every line of every data file is + * decoded; the key index, field indexes, spatial indexes and shard meta are compared with what the + * data says. Nothing is written, repaired, flushed or recovered (rebuildIndex() repairs). + * @param string $dbname + * @return array ['ok' => bool, 'segments', 'lines', 'records', 'badLines' => [[segment, offset, length]], + * 'tornTail', 'indexMismatches', 'bufferedRecords', 'pendingRecovery', 'quarantineFiles', + * 'recoveryLog' => last entries] + */ + public function checkIntegrity($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->integrity->checkIntegrity($dbname); + } + + + /** + * Cheap counters of the process and, for a database, its metadata (docs/OPERATIONS.md §1): index cache + * hits, misses and evictions, index pages read, checkpoints, compactions, lock waits, recovery events, + * durability, the last checkIntegrity(); a database's counters, sizes and state. Nothing is scanned or written. + * @return array ['process' => [...]] (+ 'database' => [...] for $dbname) + */ + public function getMetrics(?string $dbname = null): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->metrics->getMetrics($dbname); + } + + + /** + * Recovery log of a database: what crash recovery and maintenance did to its files, oldest first. Every + * entry has "t" (time) and "action" (docs/API.md "getRecoveryLog()" lists them). What reads of this + * process found and no write session logged yet is included with "persisted" => false (reads never write). + * @param string $dbname + * @param int $limit Newest entries to return + * @return array + */ + public function getRecoveryLog($dbname, $limit = 100){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->recoveryLog->getRecoveryLog($dbname, $limit); + } + + + /** + * True when the database exists. With autoCreateDB a missing database is created; without it nothing is + * created and the answer is false. + * @param string $dbname + * @return bool + * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] + */ + public function checkDB($dbname=null){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->checkDB($dbname); + } + + + /** + * Create a database (STORAGE_FORMAT.md §8.1) + * @param string $dbname + * @return bool True if created, false if it already existed + */ + public function createDB($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->createDB($dbname); + } + + + /** + * Delete a database (STORAGE_FORMAT.md §8.2): every file but its lock file + * @param string $dbname + * @return array ['success' => bool, 'files' => int deleted file count] + */ + public function dropDB($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->dropDB($dbname); + } + + + /** + * The databases of this key: their names, sorted (false); the same with each one's information (true); or + * one database's information, false when it doesn't exist (its name) + * @param bool|string $info + * @return array|false + */ + public function getDBs($info=false){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->getDBs($info); + } + + + /** + * The first $limit records of a list of records (a result of find()) + * @param array $array + * @param int $limit At least 1 + * @return array|false False for invalid arguments + */ + public function limit($array, $limit=0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->limit($array, $limit); + } + + /** + * The records matching the filters, in key order (a "key" filter: in its order) + * @param string $dbname + * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); + * a "key" entry selects by record key (int or list of ints) + * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + */ + public function find($dbname, $filters=0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->find($dbname, $filters); + } + + + /** + * Insert one record or a list of records: each gets the next key + * @param string $dbname + * @param array $data One record, or a list of records + * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] + */ + public function insert($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->insert($dbname, $data); + } + + + /** + * Delete the records matching the filters + * @param string $dbname + * @param array $data Filters (same rules as find); [] deletes every record + * @return array ['n' => deleted count] + */ + public function delete($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->delete($dbname, $data); + } + + + /** + * Set fields of the records matching the filters + * @param string $dbname + * @param array $data [filters, ['set' => [field => value, ...]]] + * @return array ['n' => updated count] + */ + public function update($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->update($dbname, $data); + } + + // ========================================== + // QUERY METHODS + // ========================================== + + /** + * Get unique values for a field + * @param string $dbname + * @param string $field + * @return array|false + */ + public function distinct($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->distinct($dbname, $field); + } + + + /** + * Sort array by field + * @param array $array Result from find() + * @param string $field Field to sort by + * @param string $order 'asc' or 'desc' + * @return array|false + */ + public function sort($array, $field, $order = 'asc'){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->sort($array, $field, $order); + } + + + /** + * Count records matching filter + * @param string $dbname + * @param mixed $filter + * @return int + */ + public function count($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->count($dbname, $filter); + } + + + /** + * Pattern matching search (LIKE) + * @param string $dbname + * @param string $field + * @param string $pattern Use ^ for starts with, $ for ends with + * @return array|false + */ + public function like($dbname, $field, $pattern){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->like($dbname, $field, $pattern); + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Sum numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function sum($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->sum($dbname, $field, $filter); + } + + + /** + * Average of numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function avg($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->avg($dbname, $field, $filter); + } + + + /** + * Get minimum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function min($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->min($dbname, $field, $filter); + } + + + /** + * Get maximum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function max($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->max($dbname, $field, $filter); + } + + // ========================================== + // UTILITY METHODS + // ========================================== + + /** + * Get first matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function first($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->first($dbname, $filter); + } + + + /** + * Get last matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function last($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->last($dbname, $filter); + } + + + /** + * Check if records exist matching filter + * @param string $dbname + * @param mixed $filter + * @return bool + */ + public function exists($dbname, $filter){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->exists($dbname, $filter); + } + + + /** + * Range query (min <= value <= max) + * @param string $dbname + * @param string $field + * @param mixed $min + * @param mixed $max + * @param mixed $filter Additional filter + * @return array|false + */ + public function between($dbname, $field, $min, $max, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->between($dbname, $field, $min, $max, $filter); + } + + // ========================================== + // SHARDING PUBLIC METHODS + // ========================================== + + /** + * Get sharding information for a database + * @param string $dbname + * @return array|false + */ + public function getShardInfo($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->shardInfo->getShardInfo($dbname); + } + + // ========================================== + // FIELD INDEX PUBLIC API + // ========================================== + + /** + * Create an index on a field for faster equality lookups (find / where with field => value) + * @param string $dbname Database name + * @param string $field Field name (any name except "key") + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] (+ 'shards' => int when sharded) + */ + public function createFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->createFieldIndex($dbname, $field); + } + + + /** + * Drop a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => string|null] + */ + public function dropFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->dropFieldIndex($dbname, $field); + } + + + /** + * Check if a field has an index + * @param string $dbname Database name + * @param string $field Field name + * @return bool + */ + public function hasFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->hasFieldIndex($dbname, $field); + } + + + /** + * Get list of field indexes for a database + * @param string $dbname Database name + * @return array ['fields' => array, 'sharded' => bool] + */ + public function getFieldIndexes($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->getFieldIndexes($dbname); + } + + + /** + * Rebuild a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createFieldIndex + */ + public function rebuildFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->rebuildFieldIndex($dbname, $field); + } + + + /** + * Get keys matching a field value using field index + * Returns null if no index exists (caller should fall back to scan) + * @param string $dbname Database name + * @param string $field Field name + * @param mixed $value Value to match + * @param int|null $shardId Shard ID for sharded databases (null = all shards) + * @return array|null Array of matching keys, or null if no index + */ + public function getKeysFromFieldIndex($dbname, $field, $value, $shardId = null){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->getKeysFromFieldIndex($dbname, $field, $value, $shardId); + } + + // ========================================== + // SPATIAL INDEX - GEOJSON SUPPORT + // ========================================== + + /** + * Validate a GeoJSON geometry object + * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection + * + * @param array $geometry GeoJSON geometry object + * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] + */ + public function validateGeoJSON($geometry): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->geoJson->validateGeoJSON($geometry); + } + + + /** + * Calculate Haversine distance between two points (in meters). Not memoized: a key for two points costs + * more than the formula + * @param float $lon1 Longitude of point 1 + * @param float $lat1 Latitude of point 1 + * @param float $lon2 Longitude of point 2 + * @param float $lat2 Latitude of point 2 + * @return float Distance in meters + */ + public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->geometry->haversineDistance($lon1, $lat1, $lon2, $lat2); + } + + + /** + * Check if a spatial index exists + * @param string $dbname Database name + * @param string $field Field name + * @param int|null $shardId Shard ID + * @return bool + */ + public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->rtree->hasSpatialIndex($dbname, $field, $shardId); + } + + // ========================================== + // SPATIAL INDEX - PUBLIC API + // ========================================== + + /** + * Create a spatial index on a GeoJSON field + * @param string $dbname Database name + * @param string $field Field name containing GeoJSON + * @param array $options Options ['nodeSize' => 32] + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + */ + public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->createSpatialIndex($dbname, $field, $options); + } + + + /** + * Drop a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => ?string] + */ + public function dropSpatialIndex(string $dbname, string $field): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->dropSpatialIndex($dbname, $field); + } + + + /** + * Get list of spatial indexes for a database + * @param string $dbname Database name + * @return array List of indexed field names + */ + public function getSpatialIndexes(string $dbname): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->getSpatialIndexes($dbname); + } + + + /** + * Rebuild a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createSpatialIndex + */ + public function rebuildSpatialIndex(string $dbname, string $field): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->rebuildSpatialIndex($dbname, $field); } + /** - * All vertices of a path inside the polygon and no segment crossing any of its rings + * Find records within a distance from a point (distance to the nearest point of the geometry) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @param array $options ['includeDistance' => false] + * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) */ - private function pathWithinPolygon(array $points, array $polygonCoords): bool { - foreach ($points as $point) { - if (!$this->pointInPolygon((float)$point[0], (float)$point[1], $polygonCoords)) { - return false; - } - } - $n = count($points); - for ($i = 0; $i < $n - 1; $i++) { - foreach ($polygonCoords as $ring) { - $m = count($ring); - for ($j = 0; $j < $m - 1; $j++) { - if ($this->segmentsCrossProperly($points[$i], $points[$i + 1], $ring[$j], $ring[$j + 1])) { - return false; - } - } - } - } - return true; + public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinDistance($dbname, $field, $lon, $lat, $distanceMeters, $options); } + /** - * True when two segments cross at a single interior point (touching endpoints don't count) + * Find records within a bounding box (geometries whose bounding box overlaps it). + * minLon > maxLon means the box crosses the antimeridian. + * @return array Matching records */ - private function segmentsCrossProperly(array $p1, array $p2, array $p3, array $p4): bool { - $d1 = $this->crossProductDirection($p3, $p4, $p1); - $d2 = $this->crossProductDirection($p3, $p4, $p2); - $d3 = $this->crossProductDirection($p1, $p2, $p3); - $d4 = $this->crossProductDirection($p1, $p2, $p4); - return (($d1 > 0 && $d2 < 0) || ($d1 < 0 && $d2 > 0)) && (($d3 > 0 && $d4 < 0) || ($d3 < 0 && $d4 > 0)); + public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinBBox($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); + } + + + /** + * Find K nearest records to a point + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] + * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + */ + public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->nearest($dbname, $field, $lon, $lat, $k, $options); + } + + + /** + * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon or MultiPolygon geometry + * @return array Matching records + */ + public function withinPolygon(string $dbname, string $field, array $polygon): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinPolygon($dbname, $field, $polygon); } // ========================================== @@ -6484,118 +19857,74 @@ private function segmentsCrossProperly(array $p1, array $p2, array $p3, array $p * @return array ['success' => bool, 'flushed' => int, 'error' => string|null] */ public function flush($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->flushUnlocked($dbname); - }); + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->flush($dbname); } - private function flushUnlocked($dbname){ - $flushed = $this->flushPendingBuffers($dbname); - return ['success' => true, 'flushed' => $flushed, 'error' => null]; - } /** * Flush the write buffers of every database (of this secretKey) in dbDir * @return array ['databases' => int, 'flushed' => int] */ public function flushAllBuffers(){ - $names = []; - if(is_dir($this->dbDir)){ - foreach(scandir($this->dbDir) ?: [] as $file){ - if(preg_match("/^([0-9a-f]{20})-([A-Za-z0-9' -]+)(?:_s\\d+)?\\.nonedb\\.buffer(?:\\.flushing)?$/", $file, $m) - && $this->hashDBName($m[2]) === $m[1]){ - $names[$m[2]] = true; - } - } - } - $total = 0; - foreach(array_keys($names) as $dbname){ - $result = $this->flush((string)$dbname); - $total += $result['flushed']; - } - return ['databases' => count($names), 'flushed' => $total]; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->flushAllBuffers(); } + /** * Get buffer information for a database * @param string $dbname * @return array */ public function getBufferInfo($dbname){ - $dbname = $this->normalizeDbName($dbname); - $fn = function() use ($dbname){ - return $this->getBufferInfoUnlocked($dbname); - }; - return $this->dbFilesExist($dbname) ? $this->withLock($dbname, false, $fn) : $fn(); + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->getBufferInfo($dbname); } - private function getBufferInfoUnlocked($dbname){ - $info = [ - 'enabled' => $this->bufferEnabled, - 'sizeLimit' => $this->bufferSizeLimit, - 'countLimit' => $this->bufferCountLimit, - 'flushInterval' => $this->bufferFlushInterval, - 'buffers' => [] - ]; - $describe = function($path){ - clearstatcache(true, $path); - $exists = file_exists($path) && filesize($path) > 0; - return [ - 'exists' => $exists, - 'size' => $exists ? (int)filesize($path) : 0, - 'records' => $exists ? $this->getBufferRecordCount($path) : 0 - ]; - }; - $info['buffers']['main'] = $describe($this->getBufferPath($dbname)); - foreach($this->legacyShardBufferFiles($dbname) as $file){ - if(preg_match('/_s(\d+)\.nonedb\.buffer$/', $file, $m)){ - $info['buffers']['shard_' . $m[1]] = $describe($file); - } - } - return $info; - } /** * Enable or disable write buffering * @param bool $enable */ public function enableBuffering($enable = true){ - $this->bufferEnabled = (bool)$enable; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->enableBuffering($enable); } + /** * Check if buffering is enabled * @return bool */ public function isBufferingEnabled(){ - return $this->bufferEnabled; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->isBufferingEnabled(); } + /** * Set buffer size limit (in bytes) * @param int $bytes */ public function setBufferSizeLimit($bytes){ - $this->bufferSizeLimit = max(1024, (int)$bytes); // Minimum 1KB + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferSizeLimit($bytes); } + /** * Set buffer flush interval (in seconds) * @param int $seconds 0 to disable time-based flush */ public function setBufferFlushInterval($seconds){ - $this->bufferFlushInterval = max(0, (int)$seconds); + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferFlushInterval($seconds); } + /** * Set buffer count limit * @param int $count */ public function setBufferCountLimit($count){ - $this->bufferCountLimit = max(10, (int)$count); // Minimum 10 records + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferCountLimit($count); } + /** * Compact a database: drop old record versions and deleted records from the data files. * Keys are preserved (sharded and non-sharded alike); indexes stay valid. @@ -6603,154 +19932,112 @@ public function setBufferCountLimit($count){ * @return array ['success' => bool, 'freedSlots' => int, 'totalRecords' => int, 'sharded' => bool] */ public function compact($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->compactUnlocked($dbname); - }); - } - - private function compactUnlocked($dbname){ - $result = array("success" => false, "freedSlots" => 0); - $this->flushPendingBuffers($dbname); - if(!$this->dbExists($dbname)){ - $result['status'] = 'database_not_found'; - return $result; - } - - $sharded = $this->isSharded($dbname); - $freed = 0; - if($sharded){ - $meta = $this->getCachedMeta($dbname, true); - $freed = (int)($meta['deletedCount'] ?? 0); - } - $total = 0; - foreach($this->listSegments($dbname) as $segment){ - $compacted = $this->compactJsonl($dbname, $segment); - if(!$sharded){ - $freed += $compacted['freed']; - } - $total += $compacted['compacted']; - } - if($sharded){ - $this->removeEmptyShards($dbname); - $meta = $this->getCachedMeta($dbname, true); - foreach($meta['shards'] as $i => $shard){ - $meta['shards'][$i]['deleted'] = 0; - } - $meta['deletedCount'] = 0; - $meta['totalRecords'] = $total; - $this->writeMeta($dbname, $meta); - } - - $result['success'] = true; - $result['freedSlots'] = $freed; - $result['totalRecords'] = $total; - $result['sharded'] = $sharded; - if($sharded){ - $result['newShardCount'] = count($this->listSegments($dbname)); - } - return $result; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->compact($dbname); } /** - * Remove shards without records (keys never move, so nothing else changes). The shard that holds - * the newest key is kept: new records go there. - */ - private function removeEmptyShards($dbname){ - $meta = $this->getCachedMeta($dbname, true); - if($meta === null || count($meta['shards']) < 2){ - return; - } - $lastShard = intdiv(max(0, (int)$meta['nextKey'] - 1), (int)$meta['shardSize']); - $kept = []; - $removed = []; - foreach($meta['shards'] as $shard){ - $shardId = (int)$shard['id']; - $index = $this->readJsonlIndex($dbname, $shardId); - if($shardId !== $lastShard && ($index === null || empty($index['o']))){ - $removed[] = $shardId; - } else { - $kept[] = $shard; - } - } - if(empty($removed)){ - return; - } - $meta['shards'] = $kept; - $this->writeMeta($dbname, $meta); - - $manifest = $this->getIndexManifest($dbname); - foreach($removed as $shardId){ - $dataPath = $this->getShardPath($dbname, $shardId); - foreach($manifest['fields'] as $field){ - $this->deleteFieldIndexFile($dbname, $field, $shardId); - } - $this->removeSpatialShardFiles($dbname, $shardId); - @unlink($dataPath . '.jidx'); - @unlink($dataPath); - $this->markFileNotExists($dataPath); - unset($this->indexCache[$dataPath . '.jidx'], $this->jsonlFormatCache[$dataPath]); - } - foreach($manifest['fields'] as $field){ - $this->rebuildGlobalFieldIndex($dbname, $field); - } - } - /** - * Manually trigger migration to sharded format + * Switch a database of the single layout to the sharded layout (STORAGE_FORMAT.md §5.2). A database + * switches by itself once it has assigned shardSize keys (shardingEnabled) * @param string $dbname * @return array ["success" => bool, "status" => string] */ - public function migrate($dbname){ - $dbname = $this->normalizeDbName($dbname); - return $this->withLock($dbname, true, function() use ($dbname){ - return $this->migrateUnlocked($dbname); - }); + public function convertToSharded($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->convertToSharded($dbname); } - private function migrateUnlocked($dbname){ - if($this->isSharded($dbname)){ - return array("success" => true, "status" => "already_sharded"); - } - $this->flushPendingBuffers($dbname); - if($this->isSharded($dbname)){ - return array("success" => true, "status" => "migrated"); - } - if(!$this->cachedFileExists($this->getDataPath($dbname))){ - return array("success" => false, "status" => "database_not_found"); - } - if($this->migrateToSharded($dbname)){ - return array("success" => true, "status" => "migrated"); - } - return array("success" => false, "status" => "migration_failed"); - } /** * Check if sharding is enabled * @return bool */ public function isShardingEnabled(){ - return $this->shardingEnabled; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->isShardingEnabled(); } + /** * Get current shard size setting * @return int */ public function getShardSize(){ - return $this->shardSize; + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->getShardSize(); } // ========================================== - // QUERY BUILDER (FLUENT INTERFACE) + // BACKUP AND RESTORE (docs/STORAGE_FORMAT.md §10) // ========================================== /** - * Create a new query builder for fluent interface - * @param string $dbname - * @return noneDBQuery + * Copy a database, as it is at one moment, into $dir: a directory outside dbDir that doesn't exist or is + * empty (0700, files 0600). Its manifest.json is written last; store.json is the STORE record. Writers of + * the database wait while it runs, key maintenance fails with MAINTENANCE_BUSY; nothing of the database is + * written (the buffer is copied, not flushed). + * @return array ['database', 'dir', 'databaseId', 'generation', 'nextKey', 'records', 'bufferedRecords', 'files', 'bytes'] + * @throws \InvalidArgumentException $dir inside dbDir or not empty + * @throws \RuntimeException the database does not exist; MAINTENANCE_IN_PROGRESS, KEY_NOT_REGISTERED, LOCK_FAILED */ - public function query(string $dbname): noneDBQuery { - return new noneDBQuery($this, $this->normalizeDbName($dbname)); + public function backup(string $dbname, string $dir): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->backupRestore->backup($dbname, $dir); + } + + + /** + * Make the backup in $dir the content of database $dbname, with one commit: the database opens as it was + * or as restored after any crash. A backup of another database is refused unless ['force' => true], which + * loads it under this database's identity. Keys never go back: new records get keys above every key the + * database handed out. Also the way out of FORMAT_DAMAGED: the damaged generations are kept. + * @return array ['database', 'generation', 'nextKey', 'records', 'bufferedRecords', 'target' => 'present'|'absent'|'damaged', 'foreign' => bool] + * @throws \RuntimeException RESTORE_REFUSED (nothing changed), LOCK_FAILED, MAINTENANCE_IN_PROGRESS + */ + public function restore(string $dir, string $dbname, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->backupRestore->restore($dir, $dbname, $options); + } + + // ========================================== + // KEY MAINTENANCE (docs/STORAGE_FORMAT.md §9) + // ========================================== + // + // Every operation follows the maintenance order: STORE.LOCK (exclusive, non-blocking), then every + // target database's LOCK (non-blocking; one busy lock: nothing written, MAINTENANCE_BUSY), then the + // journal .nonedb-maintenance/journal.json, then "m" and a new epoch in every target's state record. + // The logical commit point is the journal in state COMMITTED with the complete target STORE record; + // the STORE copies, clearing "m" and closing the journal follow and are repeated by a re-run. + + /** + * Rotate the secret key of every database in $dbDir that belongs to $oldKey: their directories are + * renamed to the new key's locators (contents unchanged: FORMAT and metadata hold no key) and STORE + * registers $newKey instead of $oldKey. Application processes must be stopped. Re-running after an + * interruption finishes the rotation; ['abort' => true] undoes one that did not commit yet. + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'databases' => string[], 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function migrateKey(string $dbDir, string $oldKey, string $newKey, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::migrateKey($dbDir, $oldKey, $newKey, $options); + } + + + /** + * Register a further key for the store in $dbDir; $registeredKey must already be registered. + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function registerKey(string $dbDir, string $registeredKey, string $newKey, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::registerKey($dbDir, $registeredKey, $newKey, $options); + } + + + /** + * Rebuild a store record whose two copies are both invalid (STORAGE_FORMAT.md §9.4). Every database + * directory must belong to one of $keys. ['manifest' => path or array]: a verified STORE record (a + * backup's store.json, a maintenance result's storeManifest) is restored (complete). Without one, only + * ['confirmManualRebuild' => true] rebuilds the record from $keys (not complete: keys without + * databases that were registered before are lost). + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'mode' => 'manifest'|'manual'|null, 'complete' => bool, 'storeManifest' => array|null, + * 'error' => ?string (lists databases no key owns), 'code' => ?int] + */ + public static function repairStore(string $dbDir, array $keys, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::repairStore($dbDir, $keys, $options); } } @@ -6789,9 +20076,9 @@ class noneDBQuery { // Spatial query support private array $spatialFilters = []; private ?array $distanceConfig = null; + /** @var array|null Statistics of the last scan (explain()) */ + private ?array $lastScan = null; - /** @var \Closure Calls noneDB internals without Reflection */ - private \Closure $internal; private const OPERATORS = ['$gt', '$gte', '$lt', '$lte', '$ne', '$eq', '$in', '$nin', '$exists', '$like', '$regex', '$contains']; @@ -6802,9 +20089,13 @@ class noneDBQuery { public function __construct(noneDB $db, string $dbname) { $this->db = $db; $this->dbname = $dbname; - $this->internal = \Closure::bind(function (string $method, array $args) { - return $this->$method(...$args); - }, $db, noneDB::class); + } + + /** + * The engine of the database instance (looked up, not held: a dump of the query shows no internals) + */ + private function kernel(): \noneDB\Internal\Core\Kernel { + return \noneDB\Internal\Core\Kernel::of($this->db); } /** @@ -6825,16 +20116,6 @@ public function __debugInfo() { // FILTER HELPERS // ========================================== - /** - * Call a private method of the parent noneDB instance - * @param string $methodName - * @param mixed ...$args - * @return mixed - */ - private function callPrivateMethod(string $methodName, ...$args) { - return ($this->internal)($methodName, $args); - } - /** * True for a filter value that is an operator spec: a non-empty array whose keys all start with "$" * @throws \InvalidArgumentException when operators and plain keys are mixed @@ -6870,6 +20151,9 @@ private function compileGroups(array $groups): array { foreach ($value as $operator => $operand) { $conditions[] = [(string)$field, $operator, $this->compileOperand($operator, $operand)]; } + } elseif ((string)$field === 'key') { + // As find(): key 5, "5" or a list of them (only one plain where() went through find()) + $conditions[] = ['key', '#keys', array_flip($this->kernel()->findFilter->normalizeKeyFilter($value))]; } else { $conditions[] = [(string)$field, null, $value]; } @@ -6886,9 +20170,9 @@ private function compileOperand(string $operator, $operand) { if (!is_array($operand)) { throw new \InvalidArgumentException("noneDB: {$operator} needs an array"); } - return $operand; + return $this->membership($operand); case '$like': - return $this->likeRegex($operand); + return $this->likeSpec($operand); case '$regex': return self::regexOperand($operand); case '$gt': @@ -6915,6 +20199,10 @@ private function matchesGroup(array $record, array $conditions): bool { if (!array_key_exists($field, $record) || $record[$field] !== $operand) { return false; } + } elseif ($operator === '#keys') { + if (!isset($record['key']) || !is_int($record['key']) || !isset($operand[$record['key']])) { + return false; + } } elseif (!$this->matchesOperator($record, $field, $operator, $operand)) { return false; } @@ -6938,64 +20226,24 @@ private function matchesWhereLogic(array $record, array $where, array $orWhere): } if ($all) { return true; - } - } - foreach ($orWhere as $conditions) { - if ($this->matchesGroup($record, $conditions)) { - return true; - } - } - return false; - } - - /** - * Compare two values for ordering operators ($gt, between, ...). - * Numbers (and numeric strings) compare numerically, other strings lexically; anything else - * (null, booleans, arrays, number vs. non-numeric string) is not comparable. - * @internal also used by noneDB::between(); not part of the public API - * @return int|null - */ - public static function compareValues($a, $b): ?int { - if ($a === null || $b === null || is_bool($a) || is_bool($b) || is_array($a) || is_array($b) || is_object($a) || is_object($b)) { - return null; - } - $numA = is_int($a) || is_float($a) || (is_string($a) && is_numeric($a)); - $numB = is_int($b) || is_float($b) || (is_string($b) && is_numeric($b)); - if ($numA && $numB) { - return ($a + 0) <=> ($b + 0); + } } - if (is_string($a) && is_string($b)) { - return strcmp($a, $b) <=> 0; + foreach ($orWhere as $conditions) { + if ($this->matchesGroup($record, $conditions)) { + return true; + } } - return null; + return false; } /** - * Regex for a like()/$like pattern ("^" start anchor, trailing "$" end anchor, rest literal) + * Compiled like()/$like pattern ("^" start anchor, trailing "$" end anchor, rest literal) */ - private function likeRegex($pattern): string { + private function likeSpec($pattern): array { if (!is_scalar($pattern)) { throw new \InvalidArgumentException('noneDB: like pattern must be a string'); } - return $this->callPrivateMethod('likePatternToRegex', (string)$pattern); - } - - /** - * Run preg_match, turning compile errors and backtrack-limit failures into exceptions - */ - private static function regexMatch(string $regex, string $subject): bool { - $result = @preg_match($regex, $subject); - if ($result === false) { - $error = preg_last_error(); - if ($error === PREG_BAD_UTF8_ERROR || $error === PREG_BAD_UTF8_OFFSET_ERROR) { - return false; // subject isn't UTF-8: can't match a UTF-8 pattern - } - if ($error === PREG_NO_ERROR) { - throw new \InvalidArgumentException('noneDB: invalid regular expression'); - } - throw new \RuntimeException('noneDB: regular expression failed (' . (function_exists('preg_last_error_msg') ? preg_last_error_msg() : 'error ' . $error) . ')'); - } - return $result === 1; + return \noneDB\Internal\Query\Values::compileLike((string)$pattern); } /** @@ -7005,7 +20253,9 @@ private static function regexOperand($pattern): string { if (!is_string($pattern) || $pattern === '') { throw new \InvalidArgumentException('noneDB: $regex needs a non-empty string pattern'); } - $body = preg_replace('/(? 0; case '$gte': - $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; return $c !== null && $c >= 0; case '$lt': - $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; return $c !== null && $c < 0; case '$lte': - $c = $hasField ? self::compareValues($fieldValue, $operand) : null; + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; return $c !== null && $c <= 0; case '$ne': return !$hasField || $fieldValue !== $operand; case '$eq': return $hasField && $fieldValue === $operand; case '$in': - return $hasField && in_array($fieldValue, $operand, true); + return $hasField && $this->isMember($fieldValue, $operand); case '$nin': - return !$hasField || !in_array($fieldValue, $operand, true); + return !$hasField || !$this->isMember($fieldValue, $operand); case '$exists': return $operand ? $hasField : !$hasField; case '$like': if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { return false; } - return self::regexMatch($operand, (string)$fieldValue); + return \noneDB\Internal\Query\Values::likeMatch($operand, (string)$fieldValue); case '$regex': if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { return false; } - return self::regexMatch($operand, (string)$fieldValue); + return \noneDB\Internal\Query\Values::regexMatch($operand, (string)$fieldValue); case '$contains': if (!$hasField) { return false; @@ -7074,9 +20324,9 @@ private function matchesOperator(array $record, string $field, string $operator, * between, notBetween, search) */ private function matchesAdvancedFilters(array $record): bool { - foreach ($this->whereNotFilters as $group) { - foreach ($group as $field => $value) { - if (array_key_exists($field, $record) && $record[$field] === $value) { + foreach ($this->whereNotFilters as $fields) { + foreach ($fields as $conditions) { + if ($this->matchesGroup($record, $conditions)) { return false; } } @@ -7084,11 +20334,11 @@ private function matchesAdvancedFilters(array $record): bool { foreach ($this->whereInFilters as $filter) { if (!array_key_exists($filter['field'], $record)) return false; - if (!in_array($record[$filter['field']], $filter['values'], true)) return false; + if (!$this->isMember($record[$filter['field']], $filter['values'])) return false; } foreach ($this->whereNotInFilters as $filter) { - if (array_key_exists($filter['field'], $record) && in_array($record[$filter['field']], $filter['values'], true)) { + if (array_key_exists($filter['field'], $record) && $this->isMember($record[$filter['field']], $filter['values'])) { return false; } } @@ -7096,12 +20346,12 @@ private function matchesAdvancedFilters(array $record): bool { foreach ($this->likeFilters as $like) { $value = $record[$like['field']] ?? null; if (!is_scalar($value) || is_bool($value)) return false; - if (!self::regexMatch($like['regex'], (string)$value)) return false; + if (!\noneDB\Internal\Query\Values::likeMatch($like['like'], (string)$value)) return false; } foreach ($this->notLikeFilters as $notLike) { $value = $record[$notLike['field']] ?? null; - if (is_scalar($value) && !is_bool($value) && self::regexMatch($notLike['regex'], (string)$value)) { + if (is_scalar($value) && !is_bool($value) && \noneDB\Internal\Query\Values::likeMatch($notLike['like'], (string)$value)) { return false; } } @@ -7109,16 +20359,16 @@ private function matchesAdvancedFilters(array $record): bool { foreach ($this->betweenFilters as $between) { if (!array_key_exists($between['field'], $record)) return false; $value = $record[$between['field']]; - $lo = self::compareValues($value, $between['min']); - $hi = self::compareValues($value, $between['max']); + $lo = \noneDB\Internal\Query\Values::compareValues($value, $between['min']); + $hi = \noneDB\Internal\Query\Values::compareValues($value, $between['max']); if ($lo === null || $hi === null || $lo < 0 || $hi > 0) return false; } foreach ($this->notBetweenFilters as $notBetween) { if (!array_key_exists($notBetween['field'], $record)) continue; $value = $record[$notBetween['field']]; - $lo = self::compareValues($value, $notBetween['min']); - $hi = self::compareValues($value, $notBetween['max']); + $lo = \noneDB\Internal\Query\Values::compareValues($value, $notBetween['min']); + $hi = \noneDB\Internal\Query\Values::compareValues($value, $notBetween['max']); if ($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0) return false; } @@ -7128,7 +20378,7 @@ private function matchesAdvancedFilters(array $record): bool { $searchFields = empty($search['fields']) ? array_keys($record) : $search['fields']; $found = false; foreach ($searchFields as $field) { - if ($field === 'key' && empty($search['fields'])) continue; // the record key is not content + if (empty($search['fields']) && ($field === 'key' || $field === '_distance')) continue; // not content $value = $record[$field] ?? null; if (!is_scalar($value) || is_bool($value)) continue; if (self::containsIgnoreCase((string)$value, $term)) { @@ -7143,23 +20393,15 @@ private function matchesAdvancedFilters(array $record): bool { } /** - * Case-insensitive substring test, multibyte aware when mbstring is available + * Case-insensitive substring test with Unicode case folding when mbstring is available */ private static function containsIgnoreCase(string $haystack, string $needle): bool { - if (function_exists('mb_strtolower') && preg_match('//u', $haystack) === 1 && preg_match('//u', $needle) === 1) { - return strpos(self::foldCase($haystack), self::foldCase($needle)) !== false; + if (\noneDB\Internal\Query\Values::hasMbstring() && preg_match('//u', $haystack) === 1 && preg_match('//u', $needle) === 1) { + return strpos(\noneDB\Internal\Query\Values::foldCase($haystack), \noneDB\Internal\Query\Values::foldCase($needle)) !== false; } return stripos($haystack, $needle) !== false; } - /** - * Lower-case for matching. Unicode lower-cases "İ" to "i" + combining dot (U+0307); the dot is - * dropped so "İzmir", "IZMIR" and "izmir" all match each other. - */ - private static function foldCase(string $s): string { - return str_replace("\u{0307}", '', mb_strtolower($s, 'UTF-8')); - } - private function hasAdvancedFilters(): bool { return !empty($this->whereNotFilters) || !empty($this->whereInFilters) || !empty($this->whereNotInFilters) || !empty($this->likeFilters) || !empty($this->notLikeFilters) || !empty($this->betweenFilters) @@ -7193,7 +20435,7 @@ public function where(array $filters): self { * @return self */ public function like(string $field, string $pattern): self { - $this->likeFilters[] = ['field' => $field, 'pattern' => $pattern, 'regex' => $this->likeRegex($pattern)]; + $this->likeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; return $this; } @@ -7279,10 +20521,34 @@ public function orWhere(array $filters): self { * @return self */ public function whereIn(string $field, array $values): self { - $this->whereInFilters[] = ['field' => $field, 'values' => array_values($values)]; + $this->whereInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values)), 'raw' => array_values($values)]; return $this; } + /** + * Value list for strict membership tests: longer lists become a hash set of exact identities + * (noneDB::valueKey: same result as in_array(..., true), without scanning the list per record) + * @return array|\stdClass + */ + private function membership(array $values) { + if (count($values) <= 8) { + return $values; + } + $valueKey = $this->kernel()->values->valueKeyFunction(); + $set = []; + foreach ($values as $value) { + $set[$valueKey($value)] = true; + } + return (object)['set' => $set, 'key' => $valueKey]; + } + + private function isMember($value, $membership): bool { + if ($membership instanceof \stdClass) { + return isset($membership->set[($membership->key)($value)]); + } + return in_array($value, $membership, true); + } + /** * Filter where field value is NOT in given array (strict comparison) * @param string $field @@ -7290,7 +20556,7 @@ public function whereIn(string $field, array $values): self { * @return self */ public function whereNotIn(string $field, array $values): self { - $this->whereNotInFilters[] = ['field' => $field, 'values' => array_values($values)]; + $this->whereNotInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values))]; return $this; } @@ -7301,7 +20567,13 @@ public function whereNotIn(string $field, array $values): self { */ public function whereNot(array $filters): self { if (!empty($filters)) { - $this->whereNotFilters[] = $filters; + // Per field: the field matches when all its conditions match (AND, like in where()); + // a record is excluded when any field matches. Operators are checked now, like where(). + $fields = []; + foreach ($filters as $field => $value) { + $fields[] = $this->compileGroups([[$field => $value]])[0]; + } + $this->whereNotFilters[] = $fields; } return $this; } @@ -7313,7 +20585,7 @@ public function whereNot(array $filters): self { * @return self */ public function notLike(string $field, string $pattern): self { - $this->notLikeFilters[] = ['field' => $field, 'pattern' => $pattern, 'regex' => $this->likeRegex($pattern)]; + $this->notLikeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; return $this; } @@ -7523,6 +20795,9 @@ private function spatialCandidates(?float $nearestRadius = null, ?float &$neares } if ($result === null) { $result = $keyed; + } elseif ($filter['type'] === 'nearest') { + // nearest() orders and cuts by its own distance: its order and _distance win + $result = array_intersect_key($keyed, $result); } else { // Intersect by key, keep the order and fields of the earlier result $result = array_intersect_key($result, $keyed); @@ -7537,43 +20812,127 @@ private function spatialCandidates(?float $nearestRadius = null, ?float &$neares } /** - * Base records: pushes a single plain where() group down to find() so field indexes and key - * lookups are used; otherwise all records. - * @return array [records, bool whereAlreadyApplied] + * The plan of the query's filters (see noneDB's Planner): what field indexes, global field indexes and + * key lookups can narrow down. Every record read is still checked against every filter. + * @return array */ - private function baseRecords(): array { - if (count($this->whereFilters) === 1 && empty($this->orWhereFilters)) { - $group = $this->whereFilters[0]; - $plain = true; - foreach ($group as $value) { - if ($this->isOperatorSpec($value)) { - $plain = false; - break; + private function buildPlan(): array { + $where = []; + foreach ($this->whereFilters as $group) { + $where[] = $this->groupPlan($group); + } + $node = count($where) === 1 ? $where[0] : (empty($where) ? null : ['and', $where]); + if (!empty($this->orWhereFilters)) { + // where groups (all) OR any orWhere group; an empty orWhere() group matches everything + $branches = $node === null ? [] : [$node]; + foreach ($this->orWhereFilters as $group) { + $branches[] = $this->groupPlan($group); + } + $node = ['or', $branches]; + } + $and = $node === null ? [] : [$node]; + foreach ($this->whereInFilters as $filter) { + $and[] = $this->inPlan($filter['field'], $filter['raw'] ?? []); + } + foreach ($this->betweenFilters as $between) { + $and[] = ['range', $between['field'], '$gte', $between['min']]; + $and[] = ['range', $between['field'], '$lte', $between['max']]; + } + return self::simplifyPlan(['and', $and]); + } + + /** + * Plan of one where()/orWhere() group (all its conditions) + */ + private function groupPlan(array $group): array { + $and = []; + foreach ($group as $field => $value) { + $field = (string)$field; + if (!$this->isOperatorSpec($value)) { + if ($field === 'key') { + $and[] = ['keys', $this->kernel()->findFilter->normalizeKeyFilter($value)]; + } elseif (!is_array($value)) { + $and[] = ['eq', $field, $value]; + } + continue; + } + foreach ($value as $operator => $operand) { + if ($operator === '$eq' && !is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->kernel()->findFilter->normalizeKeyFilter($operand)] : ['eq', $field, $operand]; + } elseif ($operator === '$in' && is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->kernel()->findFilter->normalizeKeyFilter(array_values($operand))] : $this->inPlan($field, $operand); + } elseif (in_array($operator, ['$gt', '$gte', '$lt', '$lte'], true) && $field !== 'key' && !is_array($operand)) { + $and[] = ['range', $field, $operator, $operand]; + } + // $ne, $nin, $exists, $like, $regex, $contains: checked on the records only + } + } + return self::simplifyPlan(['and', $and]); + } + + private function inPlan(string $field, array $values): array { + foreach ($values as $value) { + if (is_array($value) || is_object($value)) { + return ['any']; // indexes hold no arrays + } + } + return ['in', $field, array_values($values)]; + } + + private static function simplifyPlan(array $node): array { + if ($node[0] !== 'and' && $node[0] !== 'or') { + return $node; + } + $children = []; + foreach ($node[1] as $child) { + $child = self::simplifyPlan($child); + if ($child[0] === 'any') { + if ($node[0] === 'or') { + return ['any']; // a branch that matches anything } + continue; // AND with "anything" is the rest } - if ($plain) { - $records = $this->db->find($this->dbname, $group); - return [$records === false ? [] : $records, true]; + $children[] = $child; + } + if (empty($children)) { + return ['any']; + } + return count($children) === 1 ? $children[0] : [$node[0], $children]; + } + + /** + * True when the filters are one plain where() group with a key filter: find() keeps the order of + * the keys asked for, which a scan in key order wouldn't + */ + private function keyOrderLookup(): bool { + if (count($this->whereFilters) !== 1 || !empty($this->orWhereFilters) || !array_key_exists('key', $this->whereFilters[0])) { + return false; + } + foreach ($this->whereFilters[0] as $value) { + if ($this->isOperatorSpec($value)) { + return false; } } - $records = $this->db->find($this->dbname, 0); - return [$records === false ? [] : $records, false]; + return true; } /** * Filtered records (no join, grouping or projection) * @param bool $earlyExit Stop once offset+limit records are collected (only when order can't change) */ - private function filteredRecords(bool $earlyExit): array { + private function filteredRecords(bool $earlyExit, ?int $target = null, ?int $topK = null): array { // One shared lock for every read the query makes (count, spatial rounds, records), so they // all see the same state of the database - $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); - return $this->callPrivateMethod('withReadLock', $dbname, function () use ($earlyExit) { - return $this->filteredRecordsLocked($earlyExit); + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withReadLock($dbname, function () use ($earlyExit, $target, $topK) { + return $this->filteredRecordsLocked($earlyExit, $target, $topK); }); } - private function filteredRecordsLocked(bool $earlyExit): array { + /** + * @param int|null $target Matches enough to stop early (exists()); default: offset + limit without sort + */ + private function filteredRecordsLocked(bool $earlyExit, ?int $target = null, ?int $topK = null): array { $nearest = null; $nearestFilters = 0; foreach ($this->spatialFilters as $filter) { @@ -7587,20 +20946,19 @@ private function filteredRecordsLocked(bool $earlyExit): array { $nearestRadius = null; $total = 0; if ($nearestFilters === 1 && $this->spatialFilters[0]['type'] === 'nearest' && $nearest > 0 - && $this->callPrivateMethod('usesSpatialIndex', $this->dbname, $this->spatialFilters[0]['field'])) { + && $this->kernel()->spatialQueries->usesSpatialIndex($this->dbname, $this->spatialFilters[0]['field'])) { $nearestRadius = 1000.0; $total = $this->db->count($this->dbname); } - $needWhere = !empty($this->whereFilters) || !empty($this->orWhereFilters); - $where = $needWhere ? $this->compileGroups($this->whereFilters) : []; - $orWhere = $needWhere ? $this->compileGroups($this->orWhereFilters) : []; - $needAdvanced = $this->hasAdvancedFilters(); - $target = ($earlyExit && $nearest === null && empty($this->sortKeys) && $this->limitCount !== null) - ? $this->offsetCount + $this->limitCount - : PHP_INT_MAX; - // The usual case, one where() group: no need to go through the AND/OR logic per record - $single = ($needWhere && count($where) === 1 && empty($orWhere)) ? $where[0] : null; + [$single, $where, $orWhere, $needWhere, $needAdvanced] = $this->compiledFilters(); + if ($target !== null && $earlyExit && $nearest === null) { + // exists(): independent of sort + } else { + $target = ($earlyExit && $nearest === null && empty($this->sortKeys) && $this->limitCount !== null) + ? $this->offsetCount + $this->limitCount + : PHP_INT_MAX; + } while (true) { $boundary = null; @@ -7608,8 +20966,14 @@ private function filteredRecordsLocked(bool $earlyExit): array { if ($spatial !== null) { $records = array_values($spatial); $whereApplied = false; + } elseif ($this->keyOrderLookup()) { + $records = $this->db->find($this->dbname, $this->whereFilters[0]); + $records = $records === false ? [] : $records; + $whereApplied = true; } else { - [$records, $whereApplied] = $this->baseRecords(); + // Indexes narrow the segments and keys down; the records are read in batches and the scan + // stops once enough match. A sort with a small limit keeps only the best ones (top-k). + return $this->scanFiltered($target, $topK); } $filtered = []; @@ -7663,6 +21027,104 @@ private function filteredRecordsLocked(bool $earlyExit): array { return $filtered; } + /** + * The filters compiled once per query: [the one where() group when it is the only one (no AND/OR logic + * per record) or null, where() groups, orWhere() groups, any where/orWhere, any other filter] + */ + private function compiledFilters(): array { + $needWhere = !empty($this->whereFilters) || !empty($this->orWhereFilters); + $where = $needWhere ? $this->compileGroups($this->whereFilters) : []; + $orWhere = $needWhere ? $this->compileGroups($this->orWhereFilters) : []; + $single = ($needWhere && count($where) === 1 && empty($orWhere)) ? $where[0] : null; + return [$single, $where, $orWhere, $needWhere, $this->hasAdvancedFilters()]; + } + + /** + * Visit the records matching the filters in key order through the plan's scan, none kept: $visit($key, + * $record) returns false to stop; the first $offset matches are skipped and at most $limit visited. One + * segment's records are held at a time. Not for spatial filters and key lookups (filteredRecordsLocked()). + * Caller holds the lock. + * @param bool $whole The visitor never stops: each segment is read in one go + * @return array The scan's statistics (explain()) + */ + private function eachMatch(callable $visit, bool $whole, int $offset = 0, ?int $limit = null): array { + [$single, $where, $orWhere, $needWhere, $needAdvanced] = $this->compiledFilters(); + $left = $limit ?? PHP_INT_MAX; + if ($left <= 0) { + return []; + } + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->planner->scanRecords($dbname, $this->buildPlan(), + function (array $records) use ($visit, $single, $where, $orWhere, $needWhere, $needAdvanced, &$offset, &$left) { + foreach ($records as $key => $record) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) { + continue; + } + if ($needAdvanced && !$this->matchesAdvancedFilters($record)) continue; + if ($offset > 0) { + $offset--; + continue; + } + if ($visit($key, $record) === false || --$left === 0) { + return false; + } + } + return true; + }, $whole); + } + + /** + * filteredRecordsLocked() without spatial filters: the plan's candidates, every filter checked + * @param int $target Stop after this many matches + * @param int|null $topK Keep only the best $topK by the sort keys (records arrive in key order) + */ + private function scanFiltered(int $target, ?int $topK): array { + $filtered = []; + $prune = $topK === null ? PHP_INT_MAX : max(4 * $topK, 1024); + $this->lastScan = $this->eachMatch(function ($key, array $record) use (&$filtered, $target, $topK, $prune) { + $filtered[] = $record; + if ($topK !== null) { + if (count($filtered) > $prune) { + $filtered = array_slice($this->applySort($filtered), 0, $topK); + } + } elseif (count($filtered) >= $target) { + return false; + } + return true; + }, $target === PHP_INT_MAX); + return $filtered; + } + + /** + * True when the results are the matches in key order (no groupBy(), join, sort, spatial filter, + * withDistance() or key lookup): aggregates and last() then read them through eachMatch(), one segment + * held at a time, instead of collecting them + */ + private function streamsResults(): bool { + return $this->groupByField === null && empty($this->joinConfigs) && empty($this->sortKeys) && empty($this->spatialFilters) + && $this->distanceConfig === null && !$this->keyOrderLookup(); + } + + /** + * Every record of the results in order, without projection (the aggregates): $visit($record) + */ + private function eachResult(callable $visit): void { + if (!$this->streamsResults()) { + foreach ($this->results(false) as $record) { + $visit($record); + } + return; + } + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + $this->kernel()->locks->withReadLock($dbname, function () use ($visit) { + $this->eachMatch(function ($key, array $record) use ($visit) { + $visit($record); + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + }); + } + /** * Add _distance for withDistance() */ @@ -7676,8 +21138,10 @@ private function applyDistanceCalculation(array $records): array { if (!is_array($geometry) || !$this->db->validateGeoJSON($geometry)['valid']) { continue; } - $record['_distance'] = round($this->callPrivateMethod('distanceToGeometry', - (float)$this->distanceConfig['lon'], (float)$this->distanceConfig['lat'], $geometry), 4); + $distance = $this->kernel()->spatialQueries->distanceToGeometry((float)$this->distanceConfig['lon'], (float)$this->distanceConfig['lat'], $geometry); + if (is_finite($distance)) { // a geometry without any position has no distance + $record['_distance'] = round($distance, 4); + } } unset($record); return $records; @@ -7685,31 +21149,93 @@ private function applyDistanceCalculation(array $records): array { private function applyJoins(array $records): array { foreach ($this->joinConfigs as $join) { - $foreignData = $this->db->find($join['foreignDb'], 0); + if ($join['foreignKey'] === 'key') { + // Joining on record keys: read only the keys asked for + $keys = []; + foreach ($records as $record) { + $lk = self::joinKey($record[$join['localKey']] ?? null); + if (is_int($lk) && $lk >= 0) { + $keys[$lk] = $lk; + } + } + $foreignData = empty($keys) ? [] : $this->db->find($join['foreignDb'], ['key' => array_values($keys)]); + } else { + // Only the foreign records whose value can join: the local raw values (a whole number also as + // its float and canonical string, which join it too), through the foreign field index when + // there is one. The join identity (joinKey) is only compared, never used as a filter value. + $values = []; + foreach ($records as $record) { + $value = $record[$join['localKey']] ?? null; + $lk = self::joinKey($value); + if ($lk === null) { + continue; + } + if (is_int($lk)) { + $values['i' . $lk] = $lk; + $values['d' . $lk] = (float)$lk; + $values['s' . $lk] = (string)$lk; + } else { + $values[$lk] = $value; + } + } + $foreignData = empty($values) ? [] : $this->db->query($join['foreignDb'])->whereIn($join['foreignKey'], array_values($values))->get(); + } if ($foreignData === false) { $foreignData = []; } $foreignIndexed = []; foreach ($foreignData as $fRecord) { - $fk = $fRecord[$join['foreignKey']] ?? null; - if (is_scalar($fk)) { - $foreignIndexed[$fk] = $fRecord; + $fk = self::joinKey($fRecord[$join['foreignKey']] ?? null); + if ($fk !== null) { + $foreignIndexed[$fk] = $fRecord; // the last record with a key wins } } foreach ($records as &$record) { - $local = $record[$join['localKey']] ?? null; - $record[$join['alias']] = is_scalar($local) ? ($foreignIndexed[$local] ?? null) : null; + $lk = self::joinKey($record[$join['localKey']] ?? null); + $record[$join['alias']] = $lk !== null ? ($foreignIndexed[$lk] ?? null) : null; } unset($record); } return $records; } + /** + * Join identity of a value (only to compare, never to query): whole numbers in the int64 range, + * 5.0 and the canonical string "5" are the same as 5 (record keys join with user_id values); + * other floats by their bits (a float used as an array key was truncated: 1.5 joined 1); other + * strings as themselves ("007" is not 7, "1.5" is not 1.5); booleans only booleans (true joined 1 + * and "1"); null and arrays never join. + * @return int|string|null + */ + private static function joinKey($v) { + if (is_int($v)) { + return $v; + } + if (is_float($v)) { + if (is_finite($v) && floor($v) === $v && $v >= -9.2233720368547758E18 && $v < 9.2233720368547758E18) { + return (int)$v; + } + return 'd' . bin2hex(pack('E', $v + 0.0)); + } + if (is_string($v)) { + if (preg_match('/\A(0|-?[1-9][0-9]*)\z/', $v) === 1 && (string)(int)$v === $v) { + return (int)$v; + } + return 's' . $v; + } + if (is_bool($v)) { + return $v ? 'b1' : 'b0'; + } + return null; + } + private function applyGroupBy(array $records): array { $groups = []; + // Exact identity (as distinct()): json_encode() followed serialize_precision and split -0.0 from 0.0 + $valueKey = $this->kernel()->values->valueKeyFunction(); foreach ($records as $record) { $value = $record[$this->groupByField] ?? null; - $groupKey = json_encode($value, JSON_PRESERVE_ZERO_FRACTION | JSON_PARTIAL_OUTPUT_ON_ERROR); + $groupKey = $valueKey(is_float($value) ? $value + 0.0 : $value); if (!isset($groups[$groupKey])) { $groups[$groupKey] = ['_group' => $value, '_items' => [], '_count' => 0]; } @@ -7718,6 +21244,9 @@ private function applyGroupBy(array $records): array { } foreach ($this->havingFilters as $having) { + if (!in_array($having['operator'], ['>', '<', '>=', '<=', '=', '!='], true)) { + continue; // unknown operator: the condition is ignored (for every group) + } $groups = array_filter($groups, function ($group) use ($having) { if ($having['aggregate'] === 'count') { $aggValue = $group['_count']; @@ -7727,8 +21256,9 @@ private function applyGroupBy(array $records): array { [$aggType, $aggField] = explode(':', $having['aggregate'], 2); $values = []; foreach ($group['_items'] as $item) { - if (isset($item[$aggField]) && is_numeric($item[$aggField]) && !is_bool($item[$aggField])) { - $values[] = $item[$aggField] + 0; + $number = \noneDB\Internal\Query\Values::numberOf($item[$aggField] ?? null); + if ($number !== null) { + $values[] = $number; } } switch ($aggType) { @@ -7763,7 +21293,7 @@ private function applySort(array $records): array { // array_multisort with the same comparison as <=>. The original position is the last column, // so records that compare equal keep their order (a stable sort on every PHP version). $records = array_values($records); - $sortKey = $this->callPrivateMethod('sortKeyFunction'); + $sortKey = $this->kernel()->values->sortKeyFunction(); $columns = []; foreach ($this->sortKeys as $sk) { $kinds = []; @@ -7787,6 +21317,40 @@ private function applyProjection(array $records): array { if (empty($this->selectFields) && empty($this->exceptFields)) { return $records; } + if ($this->groupByField !== null) { + // Group rows: _group, _count and _items in select()/except() apply to the group rows; other + // fields apply to the items of each group (select(['name']) must not empty a group) + $groupFields = ['_group', '_count', '_items']; + $select = $this->selectFields; + $except = $this->exceptFields; + $this->selectFields = array_values(array_diff($select, $groupFields)); + $this->exceptFields = array_values(array_diff($except, $groupFields)); + $groupBy = $this->groupByField; + $this->groupByField = null; + try { + foreach ($records as &$group) { + if (!empty($this->selectFields) || !empty($this->exceptFields)) { + $group['_items'] = $this->applyProjection($group['_items']); + } + $keep = array_values(array_intersect($select, $groupFields)); + if (!empty($keep)) { + if (!empty($this->selectFields)) { + $keep[] = '_items'; + } + $group = array_intersect_key($group, array_flip($keep)); + } + foreach (array_intersect($except, $groupFields) as $field) { + unset($group[$field]); + } + } + unset($group); + } finally { + $this->groupByField = $groupBy; + $this->selectFields = $select; + $this->exceptFields = $except; + } + return $records; + } return array_map(function ($record) { if (!empty($this->selectFields)) { $filtered = []; @@ -7810,15 +21374,101 @@ private function applyProjection(array $records): array { }, $records); } + /** + * How the query reads its records: per segment whether it is skipped (a global field index or + * the keys rule it out), read through an index (candidates) or scanned; how it sorts (top-k, full) and + * whether it stops early. Every record read is checked against every filter ("residual": always). + * Nothing is built or written; an index a query would build first counts as a scan. With $analyze the + * query runs too and "actual" reports what it read and returned. + * @return array + */ + public function explain(bool $analyze = false): array { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + $plan = $this->buildPlan(); + $info = $this->kernel()->planner->explainPlan($dbname, $plan); + $segments = $info['segments']; + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $topK = $this->topKSize($lateJoins); + if (!empty($this->spatialFilters)) { + $access = 'spatial'; + } elseif ($segments['total'] === 0 || $segments['skipped'] === $segments['total']) { + $access = 'none'; + } elseif ($segments['scanned'] === 0) { + $access = 'index'; + } elseif ($segments['indexed'] === 0) { + $access = 'scan'; + } else { + $access = 'mixed'; + } + $joins = []; + foreach ($this->joinConfigs as $join) { + $joins[] = ['db' => $join['foreignDb'], 'on' => $join['localKey'] . ' = ' . $join['foreignKey'], + 'strategy' => $join['foreignKey'] === 'key' ? 'key-lookup' : 'in-lookup', 'when' => $lateJoins ? 'after-slice' : 'before-sort']; + } + $result = [ + 'explain' => 1, + 'db' => $this->dbname, + 'sharded' => $info['sharded'], + 'access' => $access, + 'segments' => $segments, + 'plan' => $info['plan'], + 'planTruncated' => $info['planTruncated'] ?? false, + 'estimatedRead' => $info['estimatedRead'], + 'filter' => $this->keyOrderLookup() ? ['keys-in-filter-order'] : $plan, + 'residual' => true, + 'spatial' => empty($this->spatialFilters) ? null : array_column($this->spatialFilters, 'type'), + 'sort' => empty($this->sortKeys) ? 'none' : ($topK !== null ? 'top-k' : 'full'), + 'earlyExit' => empty($this->sortKeys) && $this->groupByField === null && $this->limitCount !== null + && (empty($this->joinConfigs) || $lateJoins) && empty($this->spatialFilters), + 'joins' => $joins, + 'groupBy' => $this->groupByField, + ]; + if ($analyze) { + $this->lastScan = null; + $start = microtime(true); + $rows = $this->get(); + $result['actual'] = [ + 'recordsRead' => $this->lastScan['recordsRead'] ?? null, + 'segmentsRead' => $this->lastScan === null ? null : $this->lastScan['segments'] - $this->lastScan['skipped'], + 'returned' => count($rows), + 'ms' => round((microtime(true) - $start) * 1000, 3), + ]; + } + return $result; + } + /** * Execute query and get all results * @return array */ public function get(): array { - $canEarlyExit = empty($this->joinConfigs) && $this->groupByField === null && empty($this->sortKeys); - $results = $this->filteredRecords($canEarlyExit); + return $this->results(true); + } + + /** + * get() with or without select()/except() (aggregates read the fields themselves) + */ + private function results(bool $project): array { + if (!empty($this->joinConfigs)) { + // Every database of the query read in one consistent state (shared locks held together) + $databases = array_merge([$this->dbname], array_column($this->joinConfigs, 'foreignDb')); + return $this->kernel()->snapshot->withSnapshot($databases, function () use ($project) { + return $this->resultsLocked($project); + }); + } + return $this->resultsLocked($project); + } + + private function resultsLocked(bool $project): array { + // Joins only add fields: without groupBy() and without sorting by a join alias they run on the + // records left after sort, offset and limit, which also allows stopping early + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $canEarlyExit = (empty($this->joinConfigs) || $lateJoins) && $this->groupByField === null && empty($this->sortKeys); + $results = $this->filteredRecords($canEarlyExit, null, $this->topKSize($lateJoins)); $results = $this->applyDistanceCalculation($results); - $results = $this->applyJoins($results); + if (!$lateJoins) { + $results = $this->applyJoins($results); + } if ($this->groupByField !== null) { $results = $this->applyGroupBy($results); // having() only applies to groups } @@ -7826,18 +21476,79 @@ public function get(): array { if ($this->offsetCount > 0 || $this->limitCount !== null) { $results = array_slice($results, $this->offsetCount, $this->limitCount); } - return $this->applyProjection($results); + if ($lateJoins) { + $results = $this->applyJoins($results); + } + return $project ? $this->applyProjection($results) : $results; } /** - * Keys of the records a write (update/delete/removeFields) applies to: - * filters, sort, offset and limit apply; select/except/join don't matter. - * @return int[] + * Size of the top-k a sorted query with a small limit needs (offset + limit), or null: no limit, a + * large one, grouping, joins before the sort, or a sort by withDistance()'s _distance */ - private function targetKeys(): array { + private function topKSize(bool $lateJoins): ?int { + if (empty($this->sortKeys) || $this->limitCount === null || $this->groupByField !== null + || (!empty($this->joinConfigs) && !$lateJoins) || $this->distanceConfig !== null) { + return null; + } + $k = $this->offsetCount + $this->limitCount; + return $k <= 1000 ? max(1, $k) : null; + } + + private function sortsByJoinAlias(): bool { + $aliases = array_column($this->joinConfigs, 'alias'); + foreach ($this->sortKeys as $sk) { + if (in_array($sk['field'], $aliases, true)) { + return true; + } + } + return false; + } + + /** + * The records a write (update/delete/removeFields) applies to, one segment at a time: $apply($dbname, + * segment, [key => record]) once a segment's matches are complete. Filters, sort, offset and limit apply; + * select/except/join don't matter. Without sort, spatial filters and a key lookup the matches stream from + * the scan (one segment held); otherwise the target keys come first and their records are read segment by + * segment. Caller holds the exclusive lock. + */ + private function eachTargetSegment(callable $apply): void { if ($this->groupByField !== null) { throw new \LogicException('noneDB: update(), delete() and removeFields() cannot be combined with groupBy()'); } + $kernel = $this->kernel(); + $dbname = $kernel->layout->normalizeDbName($this->dbname); + $kernel->buffer->flushPendingBuffers($dbname); + // A missing database has no segments: nothing is written, and the filters are still checked + if (!empty($this->sortKeys) || !empty($this->spatialFilters) || $this->keyOrderLookup()) { + foreach ($kernel->findFilter->segmentMatches($dbname, ['key' => $this->targetKeys()]) as $segment => $records) { + $apply($dbname, $segment, $records); + } + return; + } + $size = $kernel->segments->isSharded($dbname) ? $kernel->segments->shardSizeOf($dbname) : null; + $segment = null; + $held = []; + $this->eachMatch(function ($key, array $record) use ($apply, $dbname, $size, &$segment, &$held) { + $of = $size === null ? null : intdiv($key, $size); + if (!empty($held) && ($of !== $segment || count($held) >= \noneDB\Internal\Storage\RecordIO::BATCH)) { + $apply($dbname, $segment, $held); // the scan is past that segment, or holds a batch of it + $held = []; + } + $segment = $of; + $held[$key] = $record; + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + if (!empty($held)) { + $apply($dbname, $segment, $held); + } + } + + /** + * Keys of the records a write applies to when sort, spatial filters or a key lookup decide (see + * eachTargetSegment()) + * @return int[] + */ + private function targetKeys(): array { $records = $this->applySort($this->applyDistanceCalculation($this->filteredRecords(empty($this->sortKeys)))); if ($this->offsetCount > 0 || $this->limitCount !== null) { $records = array_slice($records, $this->offsetCount, $this->limitCount); @@ -7853,10 +21564,11 @@ private function targetKeys(): array { /** * Run a write under the database's exclusive lock, so selecting and writing are one atomic step + * ($op: the operation, for getDurability()) */ - private function withWriteLock(callable $fn) { - $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); - return $this->callPrivateMethod('withLock', $dbname, true, $fn); + private function withWriteLock(callable $fn, string $op) { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withLock($dbname, true, $fn, $op); } /** @@ -7865,7 +21577,7 @@ private function withWriteLock(callable $fn) { */ public function first(): ?array { $originalLimit = $this->limitCount; - $this->limitCount = 1; + $this->limitCount = $originalLimit === null ? 1 : min(1, $originalLimit); // limit(0): nothing try { $results = $this->get(); } finally { @@ -7879,8 +21591,15 @@ public function first(): ?array { * @return array|null */ public function last(): ?array { - $results = $this->get(); - return count($results) > 0 ? $results[count($results) - 1] : null; + if (!$this->streamsResults()) { + $results = $this->get(); + return count($results) > 0 ? $results[count($results) - 1] : null; + } + $last = null; + $this->eachResult(function (array $record) use (&$last) { + $last = $record; + }); + return $last === null ? null : $this->applyProjection([$last])[0]; } /** @@ -7894,6 +21613,9 @@ public function count(): int { if ($this->groupByField !== null) { return count($this->get()); } + if (empty($this->spatialFilters) && !$this->keyOrderLookup()) { + return $this->countMatches(); // record by record, none kept in memory + } $results = $this->filteredRecords(true); if ($this->offsetCount > 0 || $this->limitCount !== null) { $results = array_slice($results, $this->offsetCount, $this->limitCount); @@ -7901,6 +21623,17 @@ public function count(): int { return count($results); } + private function countMatches(): int { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withReadLock($dbname, function () { + $count = 0; + $this->eachMatch(function () use (&$count) { + $count++; + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + return $count; + }); + } + /** * True when count() can use the stored counters * @return bool @@ -7921,7 +21654,14 @@ private function isUnfiltered(): bool { * @return bool */ public function exists(): bool { - return $this->first() !== null; + if ($this->limitCount === 0) { + return false; + } + if ($this->groupByField !== null || !empty($this->havingFilters)) { + return $this->first() !== null; // having() may drop every group: no early exit + } + // Sort, joins and projection don't change whether a record exists: stop at offset + 1 matches + return count($this->filteredRecords(true, $this->offsetCount + 1)) > $this->offsetCount; } // ========================================== @@ -7929,17 +21669,20 @@ public function exists(): bool { // ========================================== /** - * Numeric values of a field in the results (booleans and non-numeric values are ignored) + * Sum and number of the numeric values of a field in the results (booleans and non-numeric values are ignored) + * @return array [sum, count] */ - private function numericValues(string $field): array { - $values = []; - foreach ($this->get() as $record) { - $value = $record[$field] ?? null; - if ((is_int($value) || is_float($value) || (is_string($value) && is_numeric($value)))) { - $values[] = $value + 0; + private function numericTotal(string $field): array { + $sum = 0; + $count = 0; + $this->eachResult(function (array $record) use ($field, &$sum, &$count) { + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if ($number !== null) { + $sum += $number; + $count++; } - } - return $values; + }); + return [$sum, $count]; } /** @@ -7948,7 +21691,7 @@ private function numericValues(string $field): array { * @return float */ public function sum(string $field): float { - return (float)array_sum($this->numericValues($field)); + return (float)$this->numericTotal($field)[0]; } /** @@ -7957,8 +21700,8 @@ public function sum(string $field): float { * @return float */ public function avg(string $field): float { - $values = $this->numericValues($field); - return count($values) > 0 ? array_sum($values) / count($values) : 0.0; + [$sum, $count] = $this->numericTotal($field); + return $count > 0 ? $sum / $count : 0.0; } /** @@ -7969,13 +21712,12 @@ public function avg(string $field): float { public function min(string $field) { $min = null; $found = false; - foreach ($this->get() as $record) { - if (!isset($record[$field])) continue; - if (!$found || $this->callPrivateMethod('compareSortValues', $record[$field], $min) < 0) { + $this->eachResult(function (array $record) use ($field, &$min, &$found) { + if (isset($record[$field]) && (!$found || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $min) < 0)) { $min = $record[$field]; $found = true; } - } + }); return $min; } @@ -7987,13 +21729,12 @@ public function min(string $field) { public function max(string $field) { $max = null; $found = false; - foreach ($this->get() as $record) { - if (!isset($record[$field])) continue; - if (!$found || $this->callPrivateMethod('compareSortValues', $record[$field], $max) > 0) { + $this->eachResult(function (array $record) use ($field, &$max, &$found) { + if (isset($record[$field]) && (!$found || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $max) > 0)) { $max = $record[$field]; $found = true; } - } + }); return $max; } @@ -8003,12 +21744,11 @@ public function max(string $field) { * @return array */ public function distinct(string $field): array { + $seen = []; $values = []; - foreach ($this->get() as $record) { - if (isset($record[$field]) && !in_array($record[$field], $values, true)) { - $values[] = $record[$field]; - } - } + $this->eachResult(function (array $record) use ($field, &$seen, &$values) { + \noneDB\Internal\Query\Values::collectUnique([$record], $field, $seen, $values); + }); return $values; } @@ -8022,13 +21762,17 @@ public function distinct(string $field): array { * @return array ["n" => count] */ public function update(array $set): array { + $error = $this->kernel()->crud->updateSetError($set); + if ($error !== null) { + return $error; + } return $this->withWriteLock(function () use ($set) { - $keys = $this->targetKeys(); - if (empty($keys)) { - return ['n' => 0]; - } - return $this->db->update($this->dbname, [['key' => $keys], ['set' => $set]]); - }); + $n = 0; + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use ($set, &$n) { + $n += $this->kernel()->crud->updateRecords($dbname, $segment, $records, $set); + }); + return ['n' => $n]; + }, 'update'); } /** @@ -8037,12 +21781,12 @@ public function update(array $set): array { */ public function delete(): array { return $this->withWriteLock(function () { - $keys = $this->targetKeys(); - if (empty($keys)) { - return ['n' => 0]; - } - return $this->db->delete($this->dbname, ['key' => $keys]); - }); + $n = 0; + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use (&$n) { + $n += $this->kernel()->writer->deleteSegmentRecords($dbname, $segment, $records); + }); + return ['n' => $n]; + }, 'delete'); } /** @@ -8060,34 +21804,31 @@ public function removeFields(array $fields): array { } return $this->withWriteLock(function () use ($fieldsToRemove) { - $keys = $this->targetKeys(); - if (empty($keys)) { - return ['n' => 0, 'fields_removed' => $fieldsToRemove]; - } - // Fields are removed from the stored records, never from a projected/joined result - $stored = $this->db->find($this->dbname, ['key' => $keys]); - $replacements = []; + $n = 0; $removed = []; - foreach ($stored as $record) { - $changed = false; - foreach ($fieldsToRemove as $field) { - if (array_key_exists($field, $record)) { - unset($record[$field]); - $removed[$field] = true; - $changed = true; + // Fields are removed from the stored records, never from a projected/joined result + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use ($fieldsToRemove, &$n, &$removed) { + $replacements = []; + foreach ($records as $key => $record) { + $changed = false; + foreach ($fieldsToRemove as $field) { + if (array_key_exists($field, $record)) { + unset($record[$field]); + $removed[$field] = true; + $changed = true; + } + } + if ($changed) { + $replacements[$key] = $record; } } - if ($changed) { - $key = $record['key']; - unset($record['key']); - $replacements[$key] = $record; - } - } - $dbname = $this->callPrivateMethod('normalizeDbName', $this->dbname); + $n += $this->kernel()->writer->updateSegmentRecords($dbname, $segment, $replacements); + }); return [ - 'n' => $this->callPrivateMethod('replaceRecordsByKey', $dbname, $replacements), + 'n' => $n, 'fields_removed' => array_values(array_filter($fieldsToRemove, function ($f) use ($removed) { return isset($removed[$f]); })) ]; - }); + }, 'removeFields'); } } +} diff --git a/phpunit.xml b/phpunit.xml index a6c1fee..117eee0 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -11,17 +11,29 @@ convertDeprecationsToExceptions="true"> + tests/Unit - - tests/Feature - + tests/Integration - - tests/Regression + + + tests/Crash + + + + tests/Concurrency + + + + tests/Differential + + + + tests/Migration diff --git a/src/Internal/Cache/CacheManager.php b/src/Internal/Cache/CacheManager.php new file mode 100644 index 0000000..1230345 --- /dev/null +++ b/src/Internal/Cache/CacheManager.php @@ -0,0 +1,456 @@ +k = $k; + $this->rt = $k->rt; + unset($this->config, $this->layout, $this->locks); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->keyFingerprint = &$this->k->keyFingerprint; + $this->metaCache = &$this->k->metaCache; + $this->segWrites = &$this->k->segWrites; + $this->shardFilesUpTo = &$this->k->shardFilesUpTo; + $this->shardedCache = &$this->k->shardedCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->bufferCounts = &$this->rt->bufferCounts; + $this->bufferEnds = &$this->rt->bufferEnds; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->sessionNear = &$this->rt->sessionNear; + $this->indexStats = &$this->rt->indexStats; // @test + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->staticFieldIndexCache = &$this->rt->staticFieldIndexCache; + $this->staticFileExistsCache = &$this->rt->staticFileExistsCache; + $this->staticGlobalSpatialIndexCache = &$this->rt->staticGlobalSpatialIndexCache; + $this->staticIndexCache = &$this->rt->staticIndexCache; + $this->staticSpatialIndexCache = &$this->rt->staticSpatialIndexCache; + $this->staticVerifiedIndexes = &$this->rt->staticVerifiedIndexes; + $this->staticWritableIndexes = &$this->rt->staticWritableIndexes; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->config = null; + $this->layout = null; + $this->locks = null; + $this->k = null; + $this->rt = null; + } + + /** + * HMAC fingerprint of the secret; binds caches to the key without revealing it + */ + public function getKeyFingerprint(){ + if($this->keyFingerprint === null){ + $this->keyFingerprint = substr(hash_hmac('sha256', 'nonedb4/cache-key', (string)$this->config->getSecretKey()), 0, 16); + } + return $this->keyFingerprint; + } + + /** + * Clear all static caches (useful for testing or memory management) + * @return void + */ + public static function clearStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticVerifiedIndexes = []; + \noneDB\Internal\Cache\Runtime::get()->uncheckpointed = []; + \noneDB\Internal\Cache\Runtime::get()->decodeDue = []; + \noneDB\Internal\Cache\Runtime::get()->staticShardedCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticMetaCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticHashCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticFileExistsCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticSanitizeCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticFieldIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticSpatialIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticGlobalSpatialIndexCache = []; + \noneDB\Internal\Cache\Runtime::get()->staticGenerations = []; + \noneDB\Internal\Cache\Runtime::get()->staticCurrentGenerations = []; + \noneDB\Internal\Cache\Runtime::get()->staticFormatRecords = []; + \noneDB\Internal\Cache\Runtime::get()->storeState = []; + \noneDB\Internal\Cache\Runtime::get()->parsedLockStates = []; + \noneDB\Internal\Cache\Runtime::get()->bufferCounts = []; + \noneDB\Internal\Cache\Runtime::get()->bufferEnds = []; + \noneDB\Internal\Cache\Runtime::get()->staticWritableIndexes = []; + \noneDB\Internal\Cache\Runtime::get()->staticDataSizes = []; + \noneDB\Internal\Cache\Runtime::get()->staticGlobalSkip = []; + \noneDB\Internal\Cache\Runtime::get()->staticSegWrites = []; + \noneDB\Internal\Cache\Runtime::get()->staticDataEpochs = []; + \noneDB\Internal\Cache\Runtime::get()->staticShardFilesUpTo = []; + \noneDB\Internal\Cache\Runtime::get()->secondaryChecked = []; + \noneDB\Internal\Cache\Runtime::get()->cacheTouch = []; + \noneDB\Internal\Cache\Runtime::get()->tailCache = []; + foreach(\noneDB\Internal\Cache\Runtime::get()->lockHandles as $handle){ + @fclose($handle['fp']); + } + \noneDB\Internal\Cache\Runtime::get()->lockHandles = []; + } + + /** + * Disable static caching (each instance uses its own cache) + * @return void + */ + public static function disableStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticCacheEnabled = false; + } + + /** + * Enable static caching (default) + * @return void + */ + public static function enableStaticCache(){ + \noneDB\Internal\Cache\Runtime::get()->staticCacheEnabled = true; + } + + /** + * Cached file_exists check. Entries are dropped whenever another process writes the database + * (see withLock), so the cache can't go stale across processes. + * + * @param string $path File path to check + * @return bool True if file exists + */ + public function cachedFileExists($path){ + if(!$this->staticCacheEnabled){ + clearstatcache(true, $path); + return file_exists($path); + } + if(isset($this->staticFileExistsCache[$path])){ + return $this->staticFileExistsCache[$path]; + } + clearstatcache(true, $path); + $exists = file_exists($path); + $this->staticFileExistsCache[$path] = $exists; + return $exists; + } + + /** + * Mark file as existing in cache (call after creating file) + * @param string $path File path + */ + public function markFileExists($path){ + if($this->staticCacheEnabled){ + $this->staticFileExistsCache[$path] = true; + } + } + + /** + * Mark file as not existing in cache (call after deleting file) + * @param string $path File path + */ + public function markFileNotExists($path){ + if($this->staticCacheEnabled){ + $this->staticFileExistsCache[$path] = false; + } + } + + /** + * The cache limit in bytes, 0 = none: the cacheMemoryLimit setting, or by default half of PHP's + * memory_limit (no limit without one): the cache doesn't make a process run out of + * memory, and doesn't slow down one that may use all the memory it needs + */ + public static function cacheLimitBytes(){ + $rt = \noneDB\Internal\Cache\Runtime::get(); + return $rt->cacheMemoryLimit ?? self::memoryLimitHalf(); + } + + /** + * Half of PHP's memory_limit in bytes, 0 without one + */ + private static function memoryLimitHalf(){ + $rt = \noneDB\Internal\Cache\Runtime::get(); + $setting = (string)ini_get('memory_limit'); + if($setting !== $rt->memoryLimitSetting){ + // Worked out once per value of memory_limit: this runs between the segments of every operation + $ini = strtolower(trim($setting)); + $bytes = (int)$ini; + $unit = substr($ini, -1); + $bytes *= $unit === 'g' ? 1073741824 : ($unit === 'm' ? 1048576 : ($unit === 'k' ? 1024 : 1)); + $rt->memoryLimitSetting = $setting; + $rt->memoryLimitHalf = $bytes <= 0 ? 0 : intdiv($bytes, 2); + } + return $rt->memoryLimitHalf; + } + + /** + * Keep the process under cacheMemoryLimit by dropping cached indexes: the process's memory + * (memory_get_usage()) is what counts, the caches are what can be dropped. Runs when the last lock of the + * process is released and between the segments of an operation (evictBetweenSegments()); while the process + * uses less memory than the limit it does nothing. Least recently used entries go first, until the process + * is under the limit (three quarters of it between segments) or nothing is left to drop. Every evicted + * entry is loaded again when needed. A global index a write session changed stays: it is written when the + * session ends (FieldIndex::deferGlobalWrite()). + * @param bool $within Between the segments of an operation (evictBetweenSegments()) + */ + public function evictCaches($within = false){ + $limit = self::cacheLimitBytes(); + $target = $within ? intdiv(3 * $limit, 4) : $limit; + if(memory_get_usage() <= $target){ + return; + } + $pending = []; + foreach($this->lockRegistry as $entry){ + $pending += $entry['globals']; + } + $entries = []; + foreach(['staticIndexCache', 'staticFieldIndexCache', 'staticSpatialIndexCache', 'staticGlobalSpatialIndexCache'] as $cache => $property){ + foreach(array_keys($this->$property) as $key){ + if(!isset($pending[$key])){ + $entries[] = [$this->cacheTouch[$key] ?? 0, $cache, $key]; + } + } + } + usort($entries, function($a, $b){ return $a[0] <=> $b[0]; }); + foreach($entries as [, $cache, $key]){ + if($cache === 0){ + unset($this->staticIndexCache[$key], $this->staticVerifiedIndexes[$key], $this->staticWritableIndexes[$key]); + } elseif($cache === 1){ + unset($this->staticFieldIndexCache[$key]); + } elseif($cache === 2){ + unset($this->staticSpatialIndexCache[$key]); + } else { + unset($this->staticGlobalSpatialIndexCache[$key]); + } + unset($this->cacheTouch[$key]); + $this->metrics['evictions']++; + if(memory_get_usage() <= $target){ + break; + } + } + } + + /** + * Bytes read from an index file, and the 4 KiB pages they are (getMetrics(), docs/OPERATIONS.md §1.1) + */ + public function noteIndexRead($bytes, $pages = null){ + $this->metrics['bytes'] += $bytes; + $this->metrics['pages'] += $pages ?? intdiv($bytes + 4095, 4096); + } + + /** + * True when an index may be decoded into a larger in-memory form of about $bytes: no limit, or the + * process stays under half of cacheMemoryLimit with it (room left before eviction starts) + */ + public static function decodeFits($bytes){ + $limit = self::cacheLimitBytes(); + return $limit <= 0 || memory_get_usage() + $bytes <= intdiv($limit, 2); + } + + /** Memory of the process after the last eviction between segments */ + private $betweenFloor = 0; + + /** Memory PHP held for the process after it */ + private $heldFloor = 0; + + /** + * Between the segments of an operation over many segments (a read, an aggregation, a write by filter, an + * index build, compaction), once a segment is done: when the process uses more memory than cacheMemoryLimit, + * least recently used entries go until the estimate is under three quarters of the limit; the next eviction waits + * until the process has grown by another quarter (counted from where it is now if it has freed memory since). A + * process whose peak came within an eighth of memory_limit (what PHP held for it, what memory_limit counts: a + * segment's batches leave more of it behind than the process uses) measures that too, and gives back what it + * freed. Not sooner: a warm query loses what it would load again. One operation over a large database holds the + * cache and the segment it works on (the segments it finished go first), without a scan of the cache at every + * segment. Under the exclusive lock too: what a write session changed is in its files, except the + * global indexes it defers (they stay); a segment index it changed that goes is checkpointed by the + * next reader that loads it (Checkpoints::checkpointTouchedSegments()). + */ + public function evictBetweenSegments(){ + if(!$this->staticCacheEnabled){ + return; + } + $limit = $this->sessionLimit ?? ($this->sessionLimit = self::cacheLimitBytes()); + if($limit <= 0){ + return; + } + $used = memory_get_usage(); + $this->betweenFloor = min($this->betweenFloor, $used); + $quarter = intdiv($limit, 4); + $over = $used > $limit && $used > $this->betweenFloor + $quarter; + $near = memory_get_peak_usage(true) > ($this->sessionNear ?? ($this->sessionNear = self::nearMemoryLimit())); + if($near){ + $held = memory_get_usage(true); + $this->heldFloor = min($this->heldFloor, $held); + $over = $over || ($held > $limit && $held > $this->heldFloor + $quarter); + } + if(!$over){ + return; + } + $this->cacheGrew = false; + $this->indexStats['evictionsBetween']++; // @test + $this->evictCaches(true); + $this->betweenFloor = memory_get_usage(); + if($near){ + gc_mem_caches(); + $this->heldFloor = memory_get_usage(true); + } + } + + /** + * Where a process's peak is near memory_limit: seven eighths of it. Never without one + */ + private static function nearMemoryLimit(){ + $half = self::memoryLimitHalf(); + return $half > 0 ? intdiv(7 * $half, 4) : PHP_INT_MAX; + } + + /** + * Drop every cached entry that belongs to a database (index, meta, field/spatial indexes, file-exists) + */ + public function invalidateDbCaches($dbname){ + $prefix = $this->layout->dbHome($dbname); + $len = strlen($prefix); + $caches = [ + &$this->indexCache, &$this->verifiedIndexes, &$this->shardedCache, &$this->metaCache, + &$this->fieldIndexCache, &$this->spatialIndexCache, + &$this->globalSpatialIndexCache, &$this->dataSizes, &$this->dataEpochs, &$this->shardFilesUpTo, + &$this->secondaryChecked, &$this->tailCache, + ]; + foreach($caches as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($cache[$key]); + } + } + } + unset($cache); + foreach(array_keys($this->staticFileExistsCache) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->staticFileExistsCache[$key]); + } + } + $lockPath = $this->locks->getLockPath($dbname); + unset($this->globalSkip[$lockPath], $this->segWrites[$lockPath]); + foreach(array_keys($this->bufferCounts) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->bufferCounts[$key], $this->bufferEnds[$key]); + } + } + foreach(array_keys($this->writableIndexes) as $key){ + if(strncmp((string)$key, $prefix, $len) === 0){ + unset($this->writableIndexes[$key]); + } + } + } + + /** + * Caches of a database after another process wrote it without replacing a data file (same epoch): + * everything is dropped but the key indexes, which are checked again (the data only grew: the new lines + * are replayed on the next read) + */ + public function refreshDbCaches($dbname){ + $prefix = $this->layout->dbHome($dbname); + $len = strlen($prefix); + // Kept (they describe data bytes that didn't change, the new lines are replayed when used): key + // indexes, field indexes and R-trees, what was checked against the data, parsed tails, epochs + $keep = []; + foreach(['indexCache' => &$this->indexCache, 'fieldIndexCache' => &$this->fieldIndexCache, + 'spatialIndexCache' => &$this->spatialIndexCache, 'dataEpochs' => &$this->dataEpochs, + 'secondaryChecked' => &$this->secondaryChecked, 'tailCache' => &$this->tailCache] as $name => &$cache){ + foreach($cache as $key => $value){ + if(strncmp((string)$key, $prefix, $len) === 0 + && ($name !== 'fieldIndexCache' && $name !== 'spatialIndexCache' || (is_array($value) && isset($value['kind'])))){ + $keep[$name][$key] = $value; + } + } + } + unset($cache); + $this->invalidateDbCaches($dbname); + foreach($keep['indexCache'] ?? [] as $key => $value){ + $this->indexCache[$key] = $value; + } + foreach($keep['fieldIndexCache'] ?? [] as $key => $value){ + $this->fieldIndexCache[$key] = $value; + } + foreach($keep['spatialIndexCache'] ?? [] as $key => $value){ + $this->spatialIndexCache[$key] = $value; + } + foreach($keep['dataEpochs'] ?? [] as $key => $value){ + $this->dataEpochs[$key] = $value; + } + foreach($keep['secondaryChecked'] ?? [] as $key => $value){ + $this->secondaryChecked[$key] = $value; + } + foreach($keep['tailCache'] ?? [] as $key => $value){ + $this->tailCache[$key] = $value; + } + } +} diff --git a/src/Internal/Cache/Runtime.php b/src/Internal/Cache/Runtime.php new file mode 100644 index 0000000..a24d663 --- /dev/null +++ b/src/Internal/Cache/Runtime.php @@ -0,0 +1,122 @@ + resolved entry script directory + + // Placeholder keys shipped in the docs/templates; never accepted as a real secret + public $placeholderKeys = [ + 'CHANGE_THIS_TO_A_SECURE_RANDOM_STRING', + 'YOUR_SECURE_RANDOM_STRING', + 'YOUR_SECURE_RANDOM_STRING_HERE', + 'GENERATE_UNIQUE_64_CHAR_STRING', + 'your_secure_key', + 'your_key', + 'nonedb_123', // a default key that was published: as public as the examples + ]; + + // secretKey values live outside the object so print_r/var_export/serialize can't leak them + public $secretKeys = []; + public $bufferCounts=[]; // bufferPath => [size, count] so limits don't need a re-read per insert + public $bufferEnds=[]; // bufferPath => size the buffer had after this process's last append + public $flushingNow=[]; // database base path => true while its buffer is being flushed + + // Static caches for cross-instance sharing + public $staticIndexCache=[]; + public $staticVerifiedIndexes=[]; + public $pendingRecoveryEvents=[]; // lock path -> event id -> event found by a reader, logged by the next writer + public $uncheckpointed=[]; // lock path -> segments loaded without a checkpoint, written by the next writer + public $decodeDue=[]; // lock path -> key index paths decoded at the start of the database's next operation + public $indexStats=['loads' => 0, 'replayedLines' => 0, 'checkpoints' => 0, 'pointLookups' => 0, 'pagesRead' => 0, 'bodiesRead' => 0, 'evictionsBetween' => 0, 'bodiesBuilt' => 0, 'filtersBuilt' => 0, 'bodiesDecoded' => 0, 'keyDecodes' => 0, 'bodyLookups' => 0, 'dataHeld' => 0, 'heldReads' => 0, 'liveReads' => 0, 'liveTries' => 0]; // tests count the work // @test + public $staticShardedCache=[]; + public $staticMetaCache=[]; + public $staticHashCache=[]; + public $staticFileExistsCache=[]; + public $staticSanitizeCache=[]; + public $staticFieldIndexCache=[]; + public $staticSpatialIndexCache=[]; + public $staticGlobalSpatialIndexCache=[]; + public $staticGenerations=[]; // lock path -> generation the caches were validated against + public $staticCacheEnabled=true; // Enable/disable static caching + + // Process-wide lock registry (shared by every instance so two instances never self-deadlock) + public $lockRegistry=[]; // lock path -> ['fp', 'mode' => LOCK_SH|LOCK_EX, 'depth', 'dbname', 'gen', 'flags', 'bumped', 'pending', 'scope', 'rolledBack', 'sid'] while locked + public $lockHandles=[]; // lock path -> ['fp' => resource, 'pid' => int]: unlocked handles kept open for the next operation + public $sessionCounter=0; // id of the latest lock session ('sid' in the registry) + public $sessionLimit=null; // the cache limit in that session, null until it is needed (segments read it, memory_limit once) + public $sessionNear=null; // where the process's peak is near memory_limit in that session, null until it is needed + public $shutdownCleanupRegistered=false; + public $segmentPairTests=0; // edge pairs compared by ringSelfIntersects() (tests check the complexity) // @test + public $geometryValidations=0; // validateGeoJSON() calls (tests check that indexed candidates skip it) // @test + public $segmentReads=0; // readSegmentRecords() calls (tests check shard skipping) // @test + public $directoryListings=0; // listFilesWithPrefix() calls (tests check reads don't list the directory) // @test + public $pointEdgeTests=0; // point-edge tests of the polygon hole check (tests check the complexity) // @test + public $sweepBudget=null; // tests only: work the quick polygon checks may do before the sweep line (0 = sweep at once) // @test + public $mbstringFn=null; // tests only: callable() overriding "is mbstring available" // @test + public $exactMapRecords=null; // tests only: the record limit of an exact map (null: FieldIndex::EXACT_MAP_RECORDS) // @test + public $sapi=null; // tests only: the SAPI an exact map is decided by (null: PHP_SAPI) // @test + public $syncFn=null; // tests only: callable($handle, 'file'|'dir', $path): bool, replaces fsync() // @test + public $flushFn=null; // tests only: callable($handle, $path): bool, replaces fflush() of appends and replacements // @test + public $writeFn=null; // tests only: callable($handle, $bytes, $path): int|false, replaces fwrite() of appends and replacements // @test + public $renameFn=null; // tests only: callable($from, $to): bool, replaces rename() (and enables the retry) // @test + public $osFamily=null; // tests only: the platform durable mode is checked against (null: PHP_OS_FAMILY) // @test + public $syncStats=['file' => 0, 'dir' => 0, 'failed' => 0]; + // getMetrics() (docs/OPERATIONS.md §1): counters of the process, each counted where its work is done + public $metrics=['hits' => 0, 'misses' => 0, 'evictions' => 0, 'pages' => 0, 'bytes' => 0, 'checkpoints' => 0, + 'checkpointSeconds' => 0.0, 'compactions' => 0, 'compactionSeconds' => 0.0, 'locks' => 0, 'lockWaits' => 0, + 'lockWaitSeconds' => 0.0, 'lockTimeouts' => 0, 'recoveryEvents' => 0, 'recoveryActions' => []]; + public $integrityChecks=[]; // databaseId -> summary of its last checkIntegrity() + public $lastIntegrity=null; // summary of the process's last checkIntegrity() + public $lastSyncError=null; + public $lastOperation=null; // outcome of the last write operation of the process (getDurability()) + public $noWrites=0; // > 0 while checkIntegrity() runs: no recovery, no repair, no file written + public $multiLock=0; // > 0 inside withSnapshot(): no exclusive lock may be waited for + + public $faultHook=null; // tests only: callable($point, array $context), may throw or exit // @test + // Static caches are dropped while the process uses more memory than this: when the last lock is released + // and between the segments of an operation (CacheManager::evictCaches()) + public $cacheMemoryLimit = null; // bytes, 0 = no limit, null = automatic (cacheLimitBytes); the last instance's setting counts + public $memoryLimitSetting = null; // the memory_limit value memoryLimitHalf was worked out from + public $memoryLimitHalf = 0; // half of it in bytes (the automatic cache limit), 0 = no limit + public $cacheTouch = []; // cache key -> tick of its last use + public $cacheTick = 0; + public $cacheGrew = false; // something was loaded into a cache since evictCaches() last looked + public $exactFloats = false; // serialize_precision is -1 for the batch being encoded (withExactFloats) + public $staticDataSizes = []; + public $staticGlobalSkip = []; + public $staticSegWrites = []; + public $staticDataEpochs = []; + public $staticShardFilesUpTo = []; + public $secondaryChecked = []; // data path -> "epoch:bytes:crc" a secondary index or global entry was checked against + public $tailCache = []; // data path -> [from, to, rows] of the last parsed data tail + public $indexTokens = []; // field -> its index file name part (indexToken) + public $staticWritableIndexes = []; + + public $devModeNoticeShown = false; + + public $storeState = []; // dbDir => keyCheck => true: the key is registered (store opened) + public $storeLocks = []; // dbDir => ['fp' => resource, 'mode' => LOCK_SH|LOCK_EX, 'depth' => int] + public $maintenanceDepth = 0; // > 0 while this process runs maintenance (its own journal is open) + public $staticCurrentGenerations = []; + public $staticFormatRecords = []; + + public $parsedLockStates = []; // lock path => [raw record, state, flags, damaged] of its last read + +} diff --git a/src/Internal/Concurrency/LockManager.php b/src/Internal/Concurrency/LockManager.php new file mode 100644 index 0000000..1cf75b9 --- /dev/null +++ b/src/Internal/Concurrency/LockManager.php @@ -0,0 +1,868 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->recovery, $this->rtree, $this->scope, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->dbDir = &$this->k->dbDir; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->formatRecords = &$this->k->formatRecords; + $this->durable = &$this->k->durable; + $this->generations = &$this->k->generations; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->lockPaths = &$this->k->lockPaths; + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->segWrites = &$this->k->segWrites; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->decodeDue = &$this->rt->decodeDue; + $this->lockHandles = &$this->rt->lockHandles; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->parsedLockStates = &$this->rt->parsedLockStates; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->sessionNear = &$this->rt->sessionNear; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->storeLocks = &$this->rt->storeLocks; + $this->storeState = &$this->rt->storeState; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->recovery = null; + $this->rtree = null; + $this->scope = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + // Rollback scope. The outermost exclusive withLock() call of a database opens one scope; nested + // exclusive calls on the same database join it. There are no nested scopes: cutting a file back to an + // earlier size can't drop an earlier outer append and keep a later inner one. Every append to a data or + // buffer file records the size the file had before the scope's first append to it (a "mark"). When an + // exception leaves the outermost call, every marked file is cut back to its mark. + // Work that must not depend on the caller's success (buffer flush, the switch to shards) runs in place only + // while the scope has no appends yet, committing its own appends step by step (runIndependent); + // requested later, it is deferred until the scope committed, like compaction, which replaces whole files. + // A deferred job that fails doesn't fail the operation that committed before it: it is logged and runs + // again on a later operation. + + public function getLockPath($dbname){ + if(!isset($this->lockPaths[$dbname])){ + $this->layout->dbHome($dbname); + } + return $this->lockPaths[$dbname]; + } + + /** + * Run $fn while holding the database lock: shared for reads, exclusive for writes. + * + * The lock is process-wide and re-entrant. A nested exclusive request upgrades a shared lock. + * + * @param string $dbname Database name (already normalized) + * @param bool $exclusive + * @param callable $fn + * @param string $op The operation an exclusive call is (getDurability()['lastOperation']) + * @return mixed Whatever $fn returns + */ + public function withLock($dbname, $exclusive, callable $fn, $op = 'write'){ + $path = $this->getLockPath($dbname); + if($exclusive && $this->multiLock > 0){ + // A snapshot holds shared locks of several databases: waiting for an exclusive one here could + // deadlock with another snapshot. withSnapshot() does that work alone first and tries again. + throw new \RuntimeException("noneDB: database '{$dbname}' needs maintenance before the snapshot", self::SNAPSHOT_RETRY); + } + + if(isset($this->lockRegistry[$path])){ + if(!empty($this->lockRegistry[$path]['lost'])){ + throw new \RuntimeException("noneDB: the lock on database '{$dbname}' was lost (timed out while upgrading)", \noneDB::LOCK_FAILED); + } + if($exclusive && $this->lockRegistry[$path]['mode'] !== LOCK_EX){ + // flock can't upgrade atomically. On BSD/macOS a failed non-blocking upgrade keeps the shared + // lock, so two processes upgrading at once would wait for each other until lockTimeout: release + // first, then wait for the exclusive lock. Another writer may run in between, so the state is + // read again and whatever it left half-done is recovered. + $this->flushPendingGlobals($dbname, $path); // while what they describe still holds + flock($this->lockRegistry[$path]['fp'], LOCK_UN); + $this->recovery->fault('upgrade:released', ['db' => $dbname]); // @test + try { + $this->acquireFlock($this->lockRegistry[$path]['fp'], LOCK_EX, $dbname); + } catch(\Throwable $e){ + $this->lockRegistry[$path]['lost'] = true; // never continue reading without a lock + throw $e; + } + $this->lockRegistry[$path]['mode'] = LOCK_EX; + $this->loadLockState($dbname, $path); + $this->prepareDb($dbname, $path); + } + $this->lockRegistry[$path]['depth']++; + try { + if($exclusive && $this->lockRegistry[$path]['scope'] === null){ + return $this->scope->runWriteScope($dbname, $path, $fn, $op); + } + return $fn(); + } finally { + $this->lockRegistry[$path]['depth']--; + } + } + + // Outermost call. A database this process doesn't know to be present takes the missing-database + // path (STORAGE_FORMAT.md §3.3): store re-validated, dbDir listed for foreign entries, STORE.LOCK + // held shared, so no maintenance can run while it is created or answered as missing. + if(!isset($this->formatRecords[$path]) && !isset($this->storeLocks[$this->dbDir])){ + $this->store->openStore(false); + clearstatcache(true, dirname($path) . '/FORMAT'); + if(!is_file(dirname($path) . '/FORMAT')){ + return $this->withMissingDatabase($dbname, $exclusive, $fn, $op); + } + } + try { + return $this->lockSession($dbname, $exclusive, $fn, !isset($this->storeLocks[$this->dbDir]), $op); + } catch(\RuntimeException $e){ + if($e->getCode() !== self::DB_GONE && $e->getCode() !== self::DB_ABSENT){ + throw $e; + } + return $this->withMissingDatabase($dbname, $exclusive, $fn, $op); + } + } + + /** + * The missing-database path: under STORE.LOCK (shared) after the store was re-read and dbDir checked + * for foreign entries. A read of a database without a directory runs $fn without any lock (nothing is + * created); a write creates the directory and locks it. + */ + private function withMissingDatabase($dbname, $exclusive, callable $fn, $op){ + $path = $this->getLockPath($dbname); + return $this->store->withStoreRevalidated($exclusive, function() use ($dbname, $exclusive, $fn, $path, $op){ + clearstatcache(true, dirname($path)); + if(!is_dir(dirname($path))){ + if(!$exclusive || !$this->store->storeExists()){ + return $fn(); // absent: reads create nothing + } + if(!@mkdir(dirname($path), 0750) && !is_dir(dirname($path))){ + throw new \RuntimeException("noneDB: cannot create the directory of database '{$dbname}'"); + } + $this->io->setPermissions(dirname($path), true); + } + return $this->lockSession($dbname, $exclusive, $fn, false, $op); + }); + } + + /** + * One lock session of a database (outermost withLock call). $recheck: the session didn't come through + * the missing-database path, so finding the database absent means going there (DB_ABSENT). + */ + private function lockSession($dbname, $exclusive, callable $fn, $recheck, $op){ + $path = $this->getLockPath($dbname); + $mode = $exclusive ? LOCK_EX : LOCK_SH; + $fp = $this->lockFile($path, $mode, $dbname); + $this->sessionLimit = null; + $this->sessionNear = null; + $this->lockRegistry[$path] = [ + 'fp' => $fp, 'mode' => $mode, 'depth' => 1, 'dbname' => $dbname, + 'gen' => 0, 'epoch' => null, 'flags' => [], 'bumped' => false, 'pending' => [], + 'scope' => null, 'rolledBack' => false, 'sid' => ++$this->sessionCounter, 'globals' => [], 'wrote' => false, + ]; + + $ok = false; + try { + $this->loadLockState($dbname, $path); + try { + $this->prepareDb($dbname, $path); + } catch(\RuntimeException $e){ + // restore() is the way out of a damaged database (STORAGE_FORMAT.md §10.3 step 2): what recovery + // can't resolve is handed to it instead of failing + if($op !== 'restore' || $e->getCode() !== \noneDB::FORMAT_DAMAGED){ + throw $e; + } + } + if(isset($this->decodeDue[$path]) && $op !== 'rebuildIndex'){ + // §6.4: loaded by an earlier operation, decoded now (not by one that makes them again) + $this->checkpoints->decodeDueIndexes($path); + } + if($recheck && !isset($this->formatRecords[$path]) && !$this->layout->dbPresent($dbname)){ // a cached FORMAT: present + throw new \RuntimeException("noneDB: database '{$dbname}' is absent", self::DB_ABSENT); + } + $result = $exclusive ? $this->scope->runWriteScope($dbname, $path, $fn, $op) : $fn(); + $ok = true; + return $result; + } finally { + try { + if($this->lockRegistry[$path]['mode'] === LOCK_EX){ + $this->finishWriteSession($dbname, $path, $ok); + } + $this->flushPendingGlobals($dbname, $path, $ok); + } finally { + $fp = $this->lockRegistry[$path]['fp']; + unset($this->lockRegistry[$path]); + flock($fp, LOCK_UN); + $this->keepLockHandle($path, $fp); + // The cache size is looked at only after something new was cached + if($this->cacheGrew && empty($this->lockRegistry)){ + $this->cacheGrew = false; + if($this->staticCacheEnabled && \noneDB\Internal\Cache\CacheManager::cacheLimitBytes() > 0){ + $this->cache->evictCaches(); + } + } + } + } + } + + /** + * End of an exclusive session: write pending spatial indexes, close the journal entries of this + * session, and make sure the generation moved so other processes drop their caches + */ + private function finishWriteSession($dbname, $path, $ok){ + if($ok){ + if(empty($this->lockRegistry[$path]['rolledBack'])){ + $this->checkpoints->checkpointTouchedSegments($dbname, $path); + } + // After a rollback (of a deferred job) the journal stays: the next lock holder rebuilds + if(!empty($this->lockRegistry[$path]['flags']['j']) && empty($this->lockRegistry[$path]['rolledBack'])){ + // Data, indexes and meta of every journaled segment are written + unset($this->lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + } else { + // Partial failure: forget in-memory state, the files are the source of truth. + // A journal entry stays, so the next lock holder rebuilds what may be half-written. + $this->cache->invalidateDbCaches($dbname); + } + } + + /** + * Open (or reuse) a lock file and lock it. + * A kept handle is only used in the process that opened it (a forked child would share the parent's + * locks), and any handle is only trusted while the path still names the same file: if the file was + * deleted or replaced meanwhile, the file the path names now is locked instead. + * @return resource + */ + private function lockFile($path, $mode, $dbname){ + $pid = getmypid(); + for($attempt = 0; ; $attempt++){ + $kept = $this->lockHandles[$path] ?? null; + unset($this->lockHandles[$path]); + if($kept !== null && $kept['pid'] === $pid){ + $fp = $kept['fp']; + } else { + if($kept !== null){ + @fclose($kept['fp']); + } + // A database directory that is a link is no database: no link is followed (§3.4) + if(is_link(dirname($path))){ + throw new \RuntimeException("noneDB: the directory of database '{$dbname}' is a symbolic link, not a database (docs/STORAGE_FORMAT.md §3.4)", \noneDB::UNSUPPORTED_FORMAT); + } + // "e": close-on-exec. A kept handle must not leak into programs this process starts: + // an inherited copy would keep a lock alive after this process died holding it. + $created = !is_file($path); + $fp = is_dir(dirname($path)) ? @fopen($path, 'c+be') : false; + if($fp !== false && $created){ + $this->io->setPermissions($path); + } + if($fp === false){ + clearstatcache(true, dirname($path)); + if(!is_dir(dirname($path))){ + // The database directory is gone (dropped store, rotated key): the caller takes + // the missing-database path, which re-validates the store + throw new \RuntimeException("noneDB: database '{$dbname}' has no directory", self::DB_GONE); + } + throw new \RuntimeException("noneDB: cannot open lock file for database '{$dbname}'", \noneDB::LOCK_FAILED); + } + } + try { + $this->acquireFlock($fp, $mode, $dbname); + } catch(\Throwable $e){ + fclose($fp); + throw $e; + } + $own = fstat($fp); + clearstatcache(true, $path); + $named = @stat($path); + if($named !== false && ($own === false || !$own['ino'] || !$named['ino'] + || ($own['ino'] === $named['ino'] && $own['dev'] === $named['dev']))){ + return $fp; + } + flock($fp, LOCK_UN); + fclose($fp); + if($attempt >= 10){ + throw new \RuntimeException("noneDB: the lock file of database '{$dbname}' keeps being replaced", \noneDB::LOCK_FAILED); + } + } + } + + /** + * Keep an unlocked lock file handle open for the next operation (the oldest is closed past the limit) + */ + private function keepLockHandle($path, $fp){ + $this->lockHandles[$path] = ['fp' => $fp, 'pid' => getmypid()]; + if(count($this->lockHandles) > self::LOCK_HANDLE_LIMIT){ + reset($this->lockHandles); + $oldest = key($this->lockHandles); + @fclose($this->lockHandles[$oldest]['fp']); + unset($this->lockHandles[$oldest]); + } + } + + /** + * Parse a state record by the grammar of STORAGE_FORMAT.md §4.2: [state, flags, damaged]. An empty + * record is a new one (state ''); anything that doesn't match the grammar is damaged and read as J. + */ + private static function parseLockState($raw){ + $tokens = explode(' ', trim($raw, " \0")); + $state = (string)array_shift($tokens); // "{generation}:{epoch}" + $flags = []; + $damaged = false; + if($state === ''){ + $tokens = []; // a new record: written with an epoch by the next exclusive session + } elseif(preg_match('/^[0-9]{1,18}:[0-9a-f]{8}$/', $state) !== 1){ + $damaged = true; + } + foreach($tokens as $token){ + if($token === ''){ + continue; + } + if($token === 'b' || $token === 'c' || $token === 'd' || $token === 'm'){ + $flags[$token] = true; + } elseif($token === 'J'){ + $flags['j']['*'] = true; + } elseif(strncmp($token, 'j=', 2) === 0 && preg_match('/^j=(main|s[0-9]{1,9})(,(main|s[0-9]{1,9}))*$/', $token) === 1){ + foreach(explode(',', substr($token, 2)) as $segment){ + $flags['j'][$segment] = true; + } + } elseif(preg_match('/^g=([0-9]{1,9})>([0-9]{1,9})$/', $token, $m) === 1){ + $flags['g'] = [(int)$m[1], (int)$m[2]]; + } else { + $damaged = true; + } + } + if($damaged){ + // Not a record of this format: everything it could have said is recovered from the data (J); + // a generation that isn't CURRENT is never removed on this path (STORAGE_FORMAT.md §4.2) + $state = 'damaged'; + $flags = ['j' => ['*' => true], 'damaged' => true]; + } + return [$state, $flags, $damaged]; + } + + /** + * Read the state record (caller holds the lock). If another process wrote the database since this + * process last looked, all cached state of the database is dropped. + */ + private function loadLockState($dbname, $path){ + $fp = $this->lockRegistry[$path]['fp']; + fseek($fp, 0); + $raw = (string)fread($fp, self::LOCK_STATE_SIZE); + // The last record parsed per lock file: an unchanged record (the usual case) is not parsed again + $parsed = $this->parsedLockStates[$path] ?? null; + if($parsed !== null && $parsed[0] === $raw){ + [, $state, $flags, $damaged] = $parsed; + } else { + [$state, $flags, $damaged] = self::parseLockState($raw); + if($parsed === null && count($this->parsedLockStates) >= 4096){ + $this->parsedLockStates = []; + } + $this->parsedLockStates[$path] = [$raw, $state, $flags, $damaged]; + } + // The caches were checked against this exact state: nothing to do (the usual case, one comparison; + // the epoch is parsed when needed: false = the valid one of the state) + $seen = $this->generations[$path] ?? null; + $epoch = false; + if($seen !== $state){ + $colon = strpos($state, ':'); + $epoch = $colon === false ? null : substr($state, $colon + 1); + $seenEpoch = $seen === null || strpos($seen, ':') === false ? null : substr($seen, strpos($seen, ':') + 1); + if($seen === null || $epoch === null || $seenEpoch !== $epoch){ + $this->cache->invalidateDbCaches($dbname); + $this->layout->forgetDatabaseLayout($dbname); + // Another process replaced files or ran maintenance: the store is read again (§3.3) + unset($this->storeState[$this->dbDir]); + if($seen !== null && $this->maintenanceDepth === 0){ + $this->checkStoreAfterEpochChange(); + } + } else { + $this->cache->refreshDbCaches($dbname); // same epoch, another generation: the files only grew + } + $this->generations[$path] = $epoch === null ? null : $state; + } + $this->lockRegistry[$path]['gen'] = $damaged ? 0 : (int)$state; + $this->lockRegistry[$path]['state'] = $state; + $this->lockRegistry[$path]['epoch'] = $epoch; + $this->lockRegistry[$path]['flags'] = $flags; + if($this->durable){ + // Durable mode: flags read from the record are synced again with this session's first record + // (another process may have written them unsynced); the epoch protects nothing of this session + $current = $this->registryEpoch($path); + $this->lockRegistry[$path]['synced'] = $current === null ? [] : ['e' . $current => true]; + } + } + + /** + * A database's epoch changed under its lock: re-read the store (read only; no repair, which would + * take STORE.LOCK after a database lock) and re-check the key and the maintenance journal + */ + private function checkStoreAfterEpochChange(){ + $this->store->checkMaintenanceJournal(); + [, $copy1] = $this->store->readStoreCopy('STORE'); + [, $copy2] = $this->store->readStoreCopy('STORE.2'); + $record = $copy1 ?? $copy2; + if($record === null || ($copy1 !== null && $copy2 !== null && $copy1 !== $copy2)){ + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged", \noneDB::FORMAT_DAMAGED); + } + if(!in_array($this->store->keyCheck(), $record['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + $this->storeState[$this->dbDir][$this->store->keyCheck()] = true; + } + + /** + * Write the state record of a database held exclusively. $bump: a file is about to be written, so the + * generation moves (once per session; noteWrite()): even a writer that dies half-way makes other + * processes check their caches. A record without epoch gets one. + * Durable mode (DURABILITY.md §3): a flag or an epoch that protects the change that follows is synced + * before the record returns (clearing one needs no sync); $sync forces a sync. + * @throws \RuntimeException DURABILITY_FAILED when that sync fails + */ + public function saveLockState($path, $bump = false, $sync = false){ + if($bump && !$this->lockRegistry[$path]['bumped']){ + $this->lockRegistry[$path]['gen']++; + $this->lockRegistry[$path]['bumped'] = true; + } + $this->lockRegistry[$path]['wrote'] = true; + $epoch = $this->registryEpoch($path); + if($epoch === null){ + $epoch = $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + } + $gen = max(1, (int)$this->lockRegistry[$path]['gen']); + $this->lockRegistry[$path]['gen'] = $gen; + $state = $gen . ':' . $epoch; + $this->generations[$path] = $state; + $this->lockRegistry[$path]['state'] = $state; + $flags = $this->lockRegistry[$path]['flags']; + unset($this->lockRegistry[$path]['flags']['damaged']); + foreach(['b', 'c', 'd', 'm'] as $flag){ + if(!empty($flags[$flag])){ + $state .= ' ' . $flag; + } + } + if(!empty($flags['g'])){ + $state .= ' g=' . $flags['g'][0] . '>' . $flags['g'][1]; + } + if(!empty($flags['j'])){ + $segments = ' j=' . implode(',', array_keys($flags['j'])); + $state .= (isset($flags['j']['*']) || strlen($state . $segments) > self::LOCK_STATE_SIZE) ? ' J' : $segments; + } + $fp = $this->lockRegistry[$path]['fp']; + $raw = str_pad($state, self::LOCK_STATE_SIZE); + fseek($fp, 0); + if(fwrite($fp, $raw) !== self::LOCK_STATE_SIZE || !fflush($fp)){ + unset($this->parsedLockStates[$path]); + throw new \RuntimeException("noneDB: cannot write the lock file of database '" . $this->lockRegistry[$path]['dbname'] . "'"); + } + // The next session of this process reads this record: parsed already (what parseLockState() would + // return; "J" alone stands for all segments, so such a record is parsed again) + if(strpos($state, ' J') === false){ + unset($flags['damaged']); + $this->parsedLockStates[$path] = [$raw, $gen . ':' . $epoch, $flags, false]; + } else { + unset($this->parsedLockStates[$path]); + } + if($this->durable){ + // What this record protects: its epoch and flags; what is durable already needs no new sync + $tokens = ['e' . $epoch => true]; + foreach(explode(' ', $state) as $i => $token){ + if($i > 0){ + $tokens[$token] = true; + } + } + $synced = $this->lockRegistry[$path]['synced'] ?? []; + if(!$sync){ + foreach($tokens as $token => $unused){ + if(!isset($synced[$token]) && !($token[0] === 'j' && isset($synced['J']))){ + $sync = true; + break; + } + } + } + if($sync){ + if(!$this->sync->trySync($fp, $path)){ + throw $this->sync->failure('file', $path, false); + } + $this->lockRegistry[$path]['synced'] = $tokens; + } + } + } + + /** + * Change the state flag 'b' (write buffer may hold records); caller holds the exclusive lock + */ + public function setBufferFlag($dbname, $on){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]) || !empty($this->lockRegistry[$path]['flags']['b']) === $on){ + return; + } + if($on){ + $this->lockRegistry[$path]['flags']['b'] = true; + } else { + unset($this->lockRegistry[$path]['flags']['b']); + } + $this->saveLockState($path); + } + + /** + * After taking the lock: maintenance, an interrupted generation switch, creation or drop, a damaged + * directory and a crashed writer's journal are handled before any data is returned (STORAGE_FORMAT.md + * §4.2, §5.2, §8.3). The usual case (a present database, no flag but b) costs one isset. + */ + private function prepareDb($dbname, $path){ + $flags = $this->lockRegistry[$path]['flags']; + if(!empty($flags['m']) && $this->maintenanceDepth === 0){ + throw new \RuntimeException("noneDB: database '{$dbname}' is under maintenance", \noneDB::MAINTENANCE_IN_PROGRESS); + } + $quiet = empty($flags['g']) && empty($flags['c']) && empty($flags['d']) && empty($flags['j']) + && $this->lockRegistry[$path]['epoch'] !== null; + if($quiet && isset($this->formatRecords[$path])){ + return; + } + if($this->noWrites > 0){ + // checkIntegrity(), explain(): nothing is recovered or written; a state that can't be read is reported + if(!empty($flags['g'])){ + throw new \RuntimeException("noneDB: database '{$dbname}' has an interrupted generation switch (recovered by the next write)", \noneDB::FORMAT_DAMAGED); + } + $this->layout->resolveDatabaseState($dbname, $path); + return; + } + if($quiet){ + $this->layout->resolveDatabaseState($dbname, $path); + return; + } + $this->withLock($dbname, true, function() use ($dbname, $path){ + // Re-read under the exclusive lock: another process may have done it meanwhile + if(!empty($this->lockRegistry[$path]['flags']['g'])){ + $this->layout->recoverGenerationSwitch($dbname, $path); + } + $this->layout->resolveDatabaseState($dbname, $path); + if($this->lockRegistry[$path]['epoch'] === null){ + // A new lock record: whatever the lost one said is recovered from the data + if($this->layout->dbPresent($dbname)){ + $this->lockRegistry[$path]['flags']['j']['*'] = true; + } + $this->saveLockState($path, false, true); + // 1:{new epoch} (§4.2): no cache can match a new epoch, so this session's writes need no + // further generation + $this->lockRegistry[$path]['bumped'] = true; + } + if(!empty($this->lockRegistry[$path]['flags']['j']) && $this->layout->dbPresent($dbname)){ + $this->recovery->recoverPendingIndexes($dbname, $path); + } elseif(!empty($this->lockRegistry[$path]['flags']['j'])){ + unset($this->lockRegistry[$path]['flags']['j']); + $this->saveLockState($path); + } + if(!empty($this->lockRegistry[$path]['flags']['damaged'])){ + $this->saveLockState($path, false, true); + } + }, 'recovery'); + } + + /** + * Called before a segment's data changes. The journal is needed only for shard meta counters (sharded) + * and switches to shards ($always): field and spatial indexes follow the data by themselves. + */ + public function markIndexesPending($dbname, $shardId, $always = false){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path])){ + return; + } + if(!$always && $shardId === null){ + return; + } + $segment = \noneDB\Internal\Storage\Layout::segmentName($shardId); + if(isset($this->lockRegistry[$path]['pending'][$segment])){ + return; + } + $this->lockRegistry[$path]['pending'][$segment] = true; + $this->lockRegistry[$path]['flags']['j'][$segment] = true; + $this->saveLockState($path, true); // the data write follows + } + + /** + * flock with timeout (lockTimeout seconds; 0 waits forever) + * @throws \RuntimeException on timeout + */ + public function acquireFlock($fp, $mode, $dbname){ + $this->metrics['locks']++; + if(flock($fp, $mode | LOCK_NB)){ + return; + } + // Held by another process: wait, up to lockTimeout (0: as long as it takes) + $this->metrics['lockWaits']++; + $start = microtime(true); + try { + if($this->lockTimeout <= 0){ + if(!flock($fp, $mode)){ + throw new \RuntimeException("noneDB: could not lock database '{$dbname}'", \noneDB::LOCK_FAILED); + } + return; + } + while(!flock($fp, $mode | LOCK_NB)){ + if((microtime(true) - $start) >= $this->lockTimeout){ + $this->metrics['lockTimeouts']++; + throw new \RuntimeException("noneDB: timed out after {$this->lockTimeout}s waiting for the lock on database '{$dbname}'", \noneDB::LOCK_FAILED); + } + usleep($this->lockRetryDelay); + } + } finally { + $this->metrics['lockWaitSeconds'] += microtime(true) - $start; + } + } + + /** + * A file is about to be written: the first write of an exclusive session bumps the generation of the + * database the file belongs to (see saveLockState). A database file written without its database's + * exclusive lock is a bug: LogicException. Files of no database (dbDir housekeeping) pass. + * @throws \LogicException + */ + public function noteWrite($file){ + $file = (string)$file; + unset($this->dataSizes[$file]); // a size cached for global index checks + $owner = null; + foreach($this->lockRegistry as $path => $entry){ + // Files of a database are inside its directory: the lock path without "LOCK" + $prefix = substr($path, 0, -4); + if(strncmp($file, $prefix, strlen($prefix)) === 0){ + $owner = $path; + break; + } + } + if($owner !== null){ + if(substr($file, -6) === '.jsonl' && strpos($file, '/segments/') !== false){ + // A data file: global index lookups check its shard's newer lines (globalCheckShards) + $writes = &$this->segWrites[$owner]; + $writes['seq'] = ($writes['seq'] ?? 0) + 1; + $writes['paths'][$file] = $writes['seq']; + unset($writes); + } + if($this->lockRegistry[$owner]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: write to " . basename($file) . " under a shared lock of database '" . $this->lockRegistry[$owner]['dbname'] . "'"); + } + $this->lockRegistry[$owner]['wrote'] = true; + if(!$this->lockRegistry[$owner]['bumped']){ + $this->saveLockState($owner, true); + } + } elseif(preg_match('#/[0-9a-f]{32}\.ndb4/#', $file) === 1){ + throw new \LogicException("noneDB: write to " . basename($file) . " without the lock of its database"); + } + } + + /** + * Epoch of a locked database's state record (null: none or invalid) + */ + public function registryEpoch($path){ + $epoch = $this->lockRegistry[$path]['epoch'] ?? null; + if($epoch === false){ + $state = $this->lockRegistry[$path]['state']; + $epoch = substr($state, strpos($state, ':') + 1); // a state with a valid epoch (see loadLockState) + $this->lockRegistry[$path]['epoch'] = $epoch; + } + return $epoch; + } + + /** + * A data file of the database is about to be replaced or removed (compaction, the switch to shards, drop): new + * epoch, so every other process drops all its caches of the database (caller holds the exclusive lock) + */ + /** + * Global field and spatial index files changed in memory during this lock session (a checkpoint per + * shard updates them): written once here, at the end of the session, at an epoch change or before the + * lock is upgraded, instead of once per shard (O(shards²) for an operation that checkpoints every + * shard). Not written when the session failed or the epoch changed since: a missing entry only means + * the shard is never skipped. + */ + private function flushPendingGlobals($dbname, $path, $ok = true){ + $pending = $this->lockRegistry[$path]['globals'] ?? []; + if(empty($pending)){ + return; + } + $this->lockRegistry[$path]['globals'] = []; + $epoch = $this->registryEpoch($path); + foreach($pending as $file => [$kind, $field, $deferredAt]){ + $global = $kind === 'gfx' ? ($this->fieldIndexCache[$file] ?? null) : ($this->globalSpatialIndexCache[$file] ?? null); + if(!$ok || !empty($this->lockRegistry[$path]['rolledBack']) || $deferredAt !== $epoch || $global === null){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + continue; + } + try { + if($kind === 'gfx'){ + $this->fieldIndex->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->rtree->writeGlobalSpatialIndex($dbname, $field, $global); + } + } catch(\Throwable $e){ + unset($this->fieldIndexCache[$file], $this->globalSpatialIndexCache[$file]); + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: a shard without a fitting entry is never skipped + } + } + } + + public function bumpEpoch($dbname){ + $path = $this->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]) || $this->lockRegistry[$path]['mode'] !== LOCK_EX){ + throw new \LogicException("noneDB: new epoch of database '{$dbname}' without its exclusive lock"); + } + $this->flushPendingGlobals($dbname, $path); // under the epoch their entries belong to + $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + $this->saveLockState($path, true); + unset($this->globalSkip[$path], $this->segWrites[$path]); + // What this process checked against the data files no longer holds for the one being replaced + $prefix = $this->layout->dbHome($dbname); + foreach([&$this->dataSizes, &$this->dataEpochs, &$this->secondaryChecked, &$this->tailCache] as &$cache){ + foreach(array_keys($cache) as $key){ + if(strncmp((string)$key, $prefix, strlen($prefix)) === 0){ + unset($cache[$key]); + } + } + } + unset($cache); + } + + /** + * Run a read operation: shared lock while the database exists, no lock (and no files created) + * when it doesn't (withLock's missing-database path). Buffered records are flushed first. + */ + public function withReadLock($dbname, callable $fn){ + return $this->withLock($dbname, false, function() use ($dbname, $fn){ + if(isset($this->lockRegistry[$this->getLockPath($dbname)])){ + $this->buffer->flushPendingBuffers($dbname); + } + return $fn(); + }); + } + // ========================================== + // DATABASE LOCKING AND STATE + // ========================================== + // + // Every database has a lock file {locator}.ndb4/LOCK. Besides being the flock target it holds a small + // state record (docs/STORAGE_FORMAT.md §4.2). It is read on every lock acquisition anyway, so the + // checks below cost no extra I/O: + // + // "{generation}:{epoch}[ b][ c][ d][ m][ g=N>X][ j=main,s0,s3 | J]" space padded to LOCK_STATE_SIZE bytes + // + // generation bumped by the first file write of a write session (sessions that write nothing keep + // it). A value other than the one this process last saw means another process changed + // the database: its caches are checked again (key indexes: only the new data). + // epoch 8 hex digits, new before a file a process may have cached is replaced or removed + // (compaction, generation switch, drop, maintenance) and for a new lock record: another + // epoch drops every cache of the database and makes the process read the store again. + // b the write buffer may hold records; they are flushed before the next read or write. + // c / d creation / drop in progress (completed or undone by the next exclusive session). + // m maintenance in progress: MAINTENANCE_IN_PROGRESS. + // g=N>X generation switch from N to X in progress: recovered before any data is returned. + // j=... / J journal: shard counters of these segments (J: all) may lag behind the data because a + // writer died; the next lock holder recounts them. Key, field and spatial indexes need + // none: they replay the data after their checkpoint. + // + // The record is overwritten in place with a fixed size (one small write: no truncate, no rename). + // Lock files are never deleted (dropDB keeps them), and their handles stay open between operations. + + public const LOCK_STATE_SIZE = 128; + public const LOCK_HANDLE_LIMIT = 64; + /** Exception code (internal): an exclusive lock was needed while a snapshot holds several shared ones */ + public const SNAPSHOT_RETRY = 3303; + public const DB_GONE = 3399; // internal: the database directory vanished under a kept path + public const DB_ABSENT = 3398; // internal: found absent without the missing-database path +} diff --git a/src/Internal/Concurrency/Snapshot.php b/src/Internal/Concurrency/Snapshot.php new file mode 100644 index 0000000..dcad23a --- /dev/null +++ b/src/Internal/Concurrency/Snapshot.php @@ -0,0 +1,129 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->layout, $this->locks, $this->recovery, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->faultHook = &$this->rt->faultHook; // @test + $this->multiLock = &$this->rt->multiLock; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Run $fn with shared locks on several databases at once (joins): every read sees the same + * state of all of them. Locks are taken in one canonical order (by lock file path), so two snapshots + * never wait for each other in a circle, and no exclusive lock is ever waited for while several are + * held: work that needs one (recovery, an index repair) is done for each database alone first. + */ + public function withSnapshot(array $dbnames, callable $fn){ + $names = []; + foreach($dbnames as $name){ + $name = $this->layout->normalizeDbName($name); + $lock = $this->locks->getLockPath($name); + $dir = realpath(dirname($lock)); + $names[($dir === false ? dirname($lock) : $dir) . '/' . basename($lock)] = $name; + } + ksort($names, SORT_STRING); + $names = array_values($names); + if(count($names) < 2){ + return $fn(); + } + $started = microtime(true); + while(true){ + foreach($names as $name){ + // Alone: recovery, buffer flush and key index repairs happen here, not under the snapshot + $this->locks->withReadLock($name, function() use ($name){ + $done = false; // a segment was read: the cache may drop what it loaded before the next one + foreach($this->segments->listSegments($name) as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $this->checkpoints->readJsonlIndex($name, $segment); + } + }); + } + try { + return $this->withSharedLocks($names, $fn); + } catch(\RuntimeException $e){ + if($e->getCode() !== \noneDB\Internal\Concurrency\LockManager::SNAPSHOT_RETRY){ + throw $e; + } + if($this->lockTimeout > 0 && microtime(true) - $started >= $this->lockTimeout){ + throw new \RuntimeException("noneDB: timed out waiting for a consistent snapshot of " . implode(', ', $names), \noneDB::LOCK_FAILED, $e); + } + usleep($this->lockRetryDelay); + } + } + } + + private function withSharedLocks(array $names, callable $fn, $i = 0){ + if($i >= count($names)){ + $this->multiLock++; + try { + if($this->faultHook !== null){ $this->recovery->fault('snapshot:held', ['dbs' => $names]); } // @test + return $fn(); + } finally { + $this->multiLock--; + } + } + return $this->locks->withReadLock($names[$i], function() use ($names, $fn, $i){ + return $this->withSharedLocks($names, $fn, $i + 1); + }); + } +} diff --git a/src/Internal/Config/Configuration.php b/src/Internal/Config/Configuration.php new file mode 100644 index 0000000..b6f5b61 --- /dev/null +++ b/src/Internal/Config/Configuration.php @@ -0,0 +1,693 @@ +k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferEnabled = &$this->k->bufferEnabled; + $this->cloneSourceId = &$this->k->cloneSourceId; + $this->dbDir = &$this->k->dbDir; + $this->devMode = &$this->k->devMode; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->directoryPermissions = &$this->k->directoryPermissions; + $this->durable = &$this->k->durable; + $this->filePermissions = &$this->k->filePermissions; + $this->autoCompactThreshold = &$this->k->autoCompactThreshold; + $this->keyFingerprint = &$this->k->keyFingerprint; + $this->lockRetryDelay = &$this->k->lockRetryDelay; + $this->lockTimeout = &$this->k->lockTimeout; + $this->shardSize = &$this->k->shardSize; + $this->shardingEnabled = &$this->k->shardingEnabled; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + $this->cacheMemoryLimit = &$this->rt->cacheMemoryLimit; + $this->configData = &$this->rt->configData; + $this->configFileDir = &$this->rt->configFileDir; + $this->configFilePath = &$this->rt->configFilePath; + $this->configLoaded = &$this->rt->configLoaded; + $this->devModeNoticeShown = &$this->rt->devModeNoticeShown; + $this->placeholderKeys = &$this->rt->placeholderKeys; + $this->secretKeys = &$this->rt->secretKeys; + $this->shutdownCleanupRegistered = &$this->rt->shutdownCleanupRegistered; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + + /** + * Load the configuration of the instance (Kernel::__construct, after the caches are linked) + * @param array|null $config Optional config array; merged over the .nonedb file when one exists + * @throws \RuntimeException If no configuration can be found outside dev mode + * @throws \InvalidArgumentException If secretKey is missing, empty, not a string or a placeholder + */ + public function initialize($config = null){ + $this->loadConfig($config); + $this->validateSecretKey(); + + if(!$this->shutdownCleanupRegistered){ + $this->shutdownCleanupRegistered = true; + register_shutdown_function(static function(){ + self::releaseInterruptedSessions(); + }); + } + } + + /** + * Shutdown: a fatal error (memory_limit, max_execution_time) or exit() inside an operation skips + * its finally blocks, so its locks would stay registered and later shutdown code would continue the + * half-done session. Release them and drop every cache instead: the next operation takes the lock + * again and recovers from the files like after a crash. + */ + private static function releaseInterruptedSessions(){ + if(empty(\noneDB\Internal\Cache\Runtime::get()->lockRegistry)){ + return; + } + foreach(\noneDB\Internal\Cache\Runtime::get()->lockRegistry as $entry){ + @flock($entry['fp'], LOCK_UN); + @fclose($entry['fp']); + } + \noneDB\Internal\Cache\Runtime::get()->lockRegistry = []; + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + } + + /** + * Never expose the secret in debug output + * @return array + */ + public function debugInfo(){ + return [ + 'dbDir' => $this->dbDir, + 'secretKey' => '***', + 'autoCreateDB' => $this->autoCreateDB, + 'shardingEnabled' => $this->shardingEnabled, + 'shardSize' => $this->shardSize, + 'bufferEnabled' => $this->bufferEnabled, + 'durability' => $this->durable ? 'durable' : 'performance', + 'devMode' => $this->devMode, + ]; + } + + public function cloned(){ + $source = $this->secretKeys[$this->cloneSourceId] ?? null; + $this->cloneSourceId = spl_object_id($this->k); + if($source !== null){ + $this->secretKeys[$this->cloneSourceId] = $source; + } + } + + public function destroyed(){ + // Anything still dirty belongs to a lock we no longer hold; it was flushed on release. + unset($this->secretKeys[spl_object_id($this->k)]); + } + + public function getSecretKey(){ + return $this->secretKeys[spl_object_id($this->k)] ?? null; + } + + private function setSecretKey($key){ + $this->secretKeys[spl_object_id($this->k)] = $key; + $this->cloneSourceId = spl_object_id($this->k); + $this->keyFingerprint = null; + } + + /** + * Load configuration. + * + * The .nonedb file is searched in: the entry script's directory, noneDB.php's directory, + * the Composer project root (when installed under vendor/) and the current working directory. + * A config array passed to the constructor is merged over the file (file is optional when the + * array already contains secretKey and dbDir). + * + * @param array|null $config Optional config array + * @throws \RuntimeException If config is missing and not in dev mode + */ + private function loadConfig($config = null){ + $arrayConfig = is_array($config) ? $config : []; + // A secretKey given in the array is used (and validated) as given, also null/false/'': + // never replaced by the file's key or the dev mode key + $keyGiven = array_key_exists('secretKey', $arrayConfig); + $arrayIsComplete = $keyGiven && isset($arrayConfig['dbDir']); + + // Load (and cache) the file config once per process + if(!$this->configLoaded){ + $this->configData = null; + $this->configFilePath = null; + $this->configFileDir = null; + + $configPath = self::findConfigFile(); + if($configPath !== null){ + $content = @file_get_contents($configPath); + if($content === false){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Cannot read config file: {$configPath}"); + } + } else { + $data = json_decode($content, true); + if(!is_array($data)){ + if(!$arrayIsComplete){ + throw new \RuntimeException("noneDB: Invalid JSON in config file: {$configPath}"); + } + } else { + $this->configData = $data; + $this->configFilePath = $configPath; + $this->configFileDir = dirname($configPath); + } + } + } + $this->configLoaded = true; + } + + if($this->configData !== null){ + self::checkNames($this->configData, self::SETTINGS, 'setting', " in {$this->configFilePath}"); + $fileConfig = $this->configData; + if(isset($arrayConfig['dbDir'])){ + unset($fileConfig['dbDir']); // overridden below; no need to resolve the file's one + } + $this->applyConfig($fileConfig, $this->configFileDir); + } + self::checkNames($arrayConfig, self::SETTINGS, 'setting'); + if(!empty($arrayConfig)){ + $this->applyConfig($arrayConfig, null); + } + if($this->configData !== null || $arrayIsComplete || ($keyGiven && $this->dbDir !== null)){ + return; + } + + if(self::isDevModeEnabled()){ + $this->devMode = true; + $projectRoot = self::projectRoot(); + if($this->dbDir === null){ + $this->dbDir = $projectRoot . '/db/'; + } + if(!$keyGiven){ + // Deterministic: a development database stays readable from one run to the next + $this->setSecretKey('nonedb_dev_mode_key_' . md5(__DIR__)); + } + if(PHP_SAPI !== 'cli' && !$this->devModeNoticeShown){ + $this->devModeNoticeShown = true; + trigger_error('noneDB: development mode is active (predictable secretKey). Never use it in production.', E_USER_NOTICE); + } + return; + } + + throw new \RuntimeException( + "noneDB: Configuration file not found!\n" . + "Create a '.nonedb' config file in your project root.\n" . + "See '.nonedb.example' for reference.\n" . + "For development, set NONEDB_DEV_MODE=true environment variable or define('NONEDB_DEV_MODE', true);" + ); + } + + /** + * Candidate config file locations, in priority order + * @return array + */ + private static function configSearchPaths(): array { + $dirs = [self::scriptDir(), __DIR__]; + $root = self::projectRoot(); + if($root !== __DIR__){ + $dirs[] = $root; + } + $cwd = getcwd(); + if($cwd !== false){ + $dirs[] = $cwd; + } + $paths = []; + foreach(array_unique($dirs) as $dir){ + $paths[] = rtrim($dir, '/\\') . '/' . \noneDB\Internal\Cache\Runtime::get()->configFile; + } + return $paths; + } + + private static function findConfigFile(): ?string { + foreach(self::configSearchPaths() as $path){ + if(is_file($path)){ + return $path; + } + } + return null; + } + + /** + * Absolute directory of the entry script (falls back to the working directory) + */ + private static function scriptDir(): string { + $script = $_SERVER['SCRIPT_FILENAME'] ?? ''; + $cwd = getcwd(); + // A relative SCRIPT_FILENAME depends on the working directory: both are part of the memo key + $memoKey = $script . "\0" . $cwd; + if(isset(\noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey])){ + return \noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey]; + } + $dir = null; + if($script !== ''){ + $real = realpath($script); + if($real !== false){ + $dir = dirname($real); + } else { + $realDir = realpath(dirname($script)); + if($realDir !== false){ + $dir = $realDir; + } + } + } + if($dir === null){ + $dir = $cwd !== false ? $cwd : __DIR__; + } + if(count(\noneDB\Internal\Cache\Runtime::get()->scriptDirs) > 16){ + \noneDB\Internal\Cache\Runtime::get()->scriptDirs = []; + } + return \noneDB\Internal\Cache\Runtime::get()->scriptDirs[$memoKey] = $dir; + } + + /** + * Project root: the directory that contains vendor/ when installed with Composer, else noneDB.php's directory + */ + private static function projectRoot(): string { + $dir = str_replace('\\', '/', __DIR__); + $pos = strpos($dir, '/vendor/'); + if($pos !== false){ + return substr(__DIR__, 0, $pos); + } + return __DIR__; + } + + private static function isDevModeEnabled(): bool { + return getenv('NONEDB_DEV_MODE') === 'true' + || getenv('NONEDB_DEV_MODE') === '1' + || (defined('NONEDB_DEV_MODE') && NONEDB_DEV_MODE === true); + } + + private static function isAbsolutePath(string $path): bool { + return $path !== '' && ($path[0] === '/' || $path[0] === '\\' || preg_match('/^[A-Za-z]:[\\\\\/]/', $path) === 1); + } + + /** + * Apply configuration values to instance properties + * @param array $config Configuration array + * @param string|null $baseDir Directory that relative dbDir values resolve against (null: the entry script's) + */ + private function applyConfig(array $config, ?string $baseDir){ + if(array_key_exists('secretKey', $config)){ + $this->setSecretKey($config['secretKey']); + } + if(array_key_exists('dbDir', $config)){ + if(!is_string($config['dbDir']) || trim($config['dbDir']) === ''){ + throw new \InvalidArgumentException("noneDB: 'dbDir' must be a directory path (a non-empty string)"); + } + $this->dbDir = $this->resolveDbDir($config['dbDir'], $baseDir); + } + if(array_key_exists('autoCreateDB', $config)){ + $this->autoCreateDB = self::configBool('autoCreateDB', $config['autoCreateDB']); + } + + // Sharding settings + if(array_key_exists('shardingEnabled', $config)){ + $this->shardingEnabled = self::configBool('shardingEnabled', $config['shardingEnabled']); + } + if(array_key_exists('shardSize', $config)){ + $shardSize = self::configNumber($config['shardSize'], true) ?? 0; + if($shardSize < 1){ + throw new \InvalidArgumentException("noneDB: 'shardSize' must be a positive integer"); + } + $this->shardSize = $shardSize; + } + + // Compaction settings + if(array_key_exists('autoCompactThreshold', $config)){ + $threshold = self::configNumber($config['autoCompactThreshold'], false) ?? 0.0; + if($threshold <= 0){ + throw new \InvalidArgumentException("noneDB: 'autoCompactThreshold' must be greater than 0"); + } + $this->autoCompactThreshold = $threshold; + } + + // Lock settings + if(array_key_exists('lockTimeout', $config)){ + // Seconds, fractions allowed (0.5 used to become 0 = wait forever) + $timeout = $config['lockTimeout']; + if(!is_int($timeout) && !is_float($timeout) && !(is_string($timeout) && is_numeric($timeout))){ + $timeout = null; + } + $timeout = $timeout === null ? NAN : (float)$timeout; + if(!is_finite($timeout) || $timeout < 0){ + throw new \InvalidArgumentException("noneDB: 'lockTimeout' must be 0 (wait forever) or a positive number of seconds"); + } + $this->lockTimeout = $timeout; + } + if(array_key_exists('lockRetryDelay', $config)){ + $delay = self::configNumber($config['lockRetryDelay'], true) ?? 0; + if($delay < 1){ + throw new \InvalidArgumentException("noneDB: 'lockRetryDelay' must be a positive number of microseconds"); + } + $this->lockRetryDelay = $delay; + } + + // Write buffer + if(array_key_exists('bufferEnabled', $config)){ + $this->bufferEnabled = self::configBool('bufferEnabled', $config['bufferEnabled']); + } + + // Use indexes for queries (indexes are always kept up to date) + if(array_key_exists('fieldIndexEnabled', $config)){ + $this->fieldIndexEnabled = self::configBool('fieldIndexEnabled', $config['fieldIndexEnabled']); + } + if(array_key_exists('spatialIndexEnabled', $config)){ + $this->spatialIndexEnabled = self::configBool('spatialIndexEnabled', $config['spatialIndexEnabled']); + } + if(array_key_exists('durability', $config)){ + $this->durable = self::durabilityMode($config['durability']); + } + if(array_key_exists('filePermissions', $config)){ + $this->filePermissions = self::permissions('filePermissions', $config['filePermissions'], 0600); + } + if(array_key_exists('directoryPermissions', $config)){ + $this->directoryPermissions = self::permissions('directoryPermissions', $config['directoryPermissions'], 0700); + } + if(array_key_exists('cacheMemoryLimit', $config)){ + $limit = $config['cacheMemoryLimit']; + if($limit === 'auto'){ + $this->cacheMemoryLimit = null; + } elseif(!(is_int($limit) && $limit >= 0) && !(is_string($limit) && ctype_digit($limit))){ + throw new \InvalidArgumentException("noneDB: 'cacheMemoryLimit' must be a non-negative integer (bytes) or 'auto'"); + } else { + $this->cacheMemoryLimit = (int)$limit; + } + } + } + + /** + * The 'durability' setting: 'performance' or 'durable' (true). Durable mode is never downgraded: where + * the platform can't provide it, the instance is not created (docs/DURABILITY.md §1). + * @throws \InvalidArgumentException a value other than the two modes + * @throws \RuntimeException UNSUPPORTED_DURABILITY + */ + public static function durabilityMode($value): bool { + self::durabilityValue($value); + if($value === 'durable' && !self::durableSupported()){ + throw new \RuntimeException("noneDB: durable mode is not supported on " . self::osFamily() . ": " + . self::platformNote(self::osFamily()) . " Use 'durability' => 'performance' or run on Linux.", \noneDB::UNSUPPORTED_DURABILITY); + } + return $value === 'durable'; + } + + /** + * A durability value: 'performance' or 'durable', whatever the platform + * @throws \InvalidArgumentException + */ + public static function durabilityValue($value): void { + if($value !== 'performance' && $value !== 'durable'){ + throw new \InvalidArgumentException("noneDB: 'durability' must be 'performance' or 'durable'"); + } + } + + /** The platform durable mode is checked against */ + public static function osFamily(): string { + return PHP_OS_FAMILY; // @prod + return \noneDB\Internal\Cache\Runtime::get()->osFamily ?? PHP_OS_FAMILY; // @test + } + + /** Durable mode is supported on Linux only (docs/DURABILITY.md §1) */ + public static function durableSupported(): bool { + return self::osFamily() === 'Linux'; + } + + /** Why durable mode is not available on a platform ('' where it is) */ + public static function platformNote(string $os): string { + switch($os){ + case 'Linux': + return ''; + case 'Darwin': + return 'fsync() does not flush the drive cache on macOS, and PHP cannot call F_FULLFSYNC.'; + case 'Windows': + return 'a directory cannot be synced on Windows, so a new file or a rename is not durable.'; + } + return 'only Linux is supported for durable mode.'; + } + + /** + * A numeric setting: an int, a finite float or a numeric string (an integer when $integer); anything + * else (booleans, arrays, "10abc", NAN) is null, and the caller rejects it + * @return int|float|null + */ + private static function configNumber($value, bool $integer){ + if(is_string($value) && is_numeric(trim($value))){ + $value = trim($value) + 0; + } + if(is_int($value)){ + return $integer ? $value : (float)$value; + } + if(is_float($value) && is_finite($value) && (!$integer || floor($value) === $value)){ + return $integer ? (int)$value : $value; + } + return null; + } + + /** + * Every key of $given must be one of $known (settings, or a method's options): an unknown one is an error that + * names it, suggests the known name that differs only in letter case or by up to two letters, and lists them + * @param string $noun 'setting' or 'option' + * @param string $where Where they come from: " in {file}", " of {method}()" + * @throws \InvalidArgumentException + */ + public static function checkNames(array $given, array $known, string $noun, string $where = ''): void { + foreach(array_keys($given) as $name){ + if(in_array($name, $known, true)){ + continue; + } + $name = (string)$name; + $closest = null; + $distance = 3; + foreach($known as $candidate){ + $d = levenshtein(strtolower($name), strtolower($candidate)); + if($d < $distance){ + $closest = $candidate; + $distance = $d; + } + } + throw new \InvalidArgumentException("noneDB: unknown {$noun} '{$name}'{$where}" . ($closest === null ? '' : " (did you mean '{$closest}'?)") + . "; the {$noun}s are: " . implode(', ', $known)); + } + } + + /** + * An option that is true or false (an option is given in PHP: a boolean, nothing else) + * @throws \InvalidArgumentException + */ + public static function optionBool(string $name, $value): bool { + if(!is_bool($value)){ + throw new \InvalidArgumentException("noneDB: '{$name}' must be true or false"); + } + return $value; + } + + /** + * A boolean setting: true/false, 1/0, "1"/"0", "true"/"false" (any case). Anything else is an error, + * so that a string like "false" or "no" never means true. + * @throws \InvalidArgumentException + */ + private static function configBool(string $name, $value): bool { + if(is_bool($value)){ + return $value; + } + if($value === 1 || $value === 0){ + return $value === 1; + } + if(is_string($value)){ + $lower = strtolower(trim($value)); + if($lower === '1' || $lower === 'true'){ + return true; + } + if($lower === '0' || $lower === 'false'){ + return false; + } + } + throw new \InvalidArgumentException("noneDB: '{$name}' must be true or false"); + } + + /** + * Permission bits of a setting: an integer or an octal string ('0640'); the owner must have $owner (read and + * write, and search for directories), and the world no write access + * @throws \InvalidArgumentException + */ + private static function permissions(string $name, $value, int $owner): int { + $mode = is_int($value) ? $value : (is_string($value) && preg_match('/^0?[0-7]{3}$/D', $value) === 1 ? (int)octdec($value) : -1); + if($mode < 0 || $mode > 0777 || ($mode & $owner) !== $owner || ($mode & 0002) !== 0){ + throw new \InvalidArgumentException("noneDB: '{$name}' must be permission bits like " . ($owner === 0600 ? '0640' : '0750') + . ' (an integer or an octal string): the owner needs ' . ($owner === 0600 ? 'read and write' : 'read, write and search') . ', the world may not write'); + } + return $mode; + } + + /** + * Resolve a configured dbDir to its canonical absolute path ending with a slash (STORAGE_FORMAT.md §3). + * Relative paths (with or without a leading ./) are relative to the config file's directory + * (file config) or to the entry script's directory (array config). + */ + private function resolveDbDir(string $dbDir, ?string $baseDir): string { + if(!self::isAbsolutePath($dbDir)){ + $relative = preg_replace('#^(\./)+#', '', $dbDir); + $baseDir = $baseDir ?? self::scriptDir(); + $dbDir = rtrim($baseDir, '/\\') . '/' . $relative; + } + return rtrim(self::canonicalPath($dbDir), '/\\') . '/'; + } + + /** + * A path with its links and its "." and ".." resolved: the nearest existing part by realpath(), the parts + * after it (that don't exist yet) by their names + */ + public static function canonicalPath(string $path): string { + $missing = []; + for($at = rtrim($path, '/\\'); $at !== '' && !file_exists($at) && !is_link($at); $at = dirname($at)){ + $missing[] = basename($at); + if(dirname($at) === $at){ + break; + } + } + $resolved = realpath($at === '' ? DIRECTORY_SEPARATOR : $at); + $resolved = $resolved === false ? rtrim($at, '/\\') : rtrim($resolved, '/\\'); + foreach(array_reverse($missing) as $part){ + if($part === '..'){ + $resolved = dirname($resolved); + } elseif($part !== '.' && $part !== ''){ + $resolved .= DIRECTORY_SEPARATOR . $part; + } + } + return $resolved === '' ? DIRECTORY_SEPARATOR : $resolved; + } + + /** + * Reject missing, empty, non-string and placeholder secret keys + * @throws \InvalidArgumentException + */ + private function validateSecretKey(){ + $key = $this->getSecretKey(); + if(!is_string($key) || trim($key) === ''){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' must be a non-empty string. Set it in your .nonedb file or constructor config " + . "(generate one with: php -r \"echo bin2hex(random_bytes(32));\")." + ); + } + if(in_array($key, $this->placeholderKeys, true)){ + throw new \InvalidArgumentException( + "noneDB: 'secretKey' is a publicly known key (an example placeholder or a published default key). " + . "Replace it with a random secret (php -r \"echo bin2hex(random_bytes(32));\"); databases written with it " + . "move to the new key with noneDB::migrateKey() (docs/CONFIGURATION.md)." + ); + } + if($this->dbDir === null){ + throw new \RuntimeException("noneDB: 'dbDir' is not configured"); + } + } + + /** + * Check if config file exists + * @return bool + */ + public static function configExists(): bool { + return self::findConfigFile() !== null; + } + + /** + * Get the config file template path + * @return string|null + */ + public static function getConfigTemplate(): ?string { + $templatePath = __DIR__ . '/.nonedb.example'; + return file_exists($templatePath) ? $templatePath : null; + } + + /** + * Clear config cache (useful for testing) + * @return void + */ + public static function clearConfigCache(): void { + \noneDB\Internal\Cache\Runtime::get()->configLoaded = false; + \noneDB\Internal\Cache\Runtime::get()->configData = null; + \noneDB\Internal\Cache\Runtime::get()->configFilePath = null; + \noneDB\Internal\Cache\Runtime::get()->configFileDir = null; + } + + /** + * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) + * @return bool + */ + public static function isDevMode(): bool { + return self::isDevModeEnabled(); + } + + /** + * Set development mode programmatically + * Useful for testing or when env vars are not available + * @param bool $enabled + * @return void + */ + public static function setDevMode(bool $enabled): void { + if($enabled && !defined('NONEDB_DEV_MODE')){ + define('NONEDB_DEV_MODE', true); + } + } +} diff --git a/src/Internal/Core/Crud.php b/src/Internal/Core/Crud.php new file mode 100644 index 0000000..f0f4659 --- /dev/null +++ b/src/Internal/Core/Crud.php @@ -0,0 +1,264 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->findFilter, $this->io, $this->layout, $this->locks, $this->segments, $this->writer); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferEnabled = &$this->k->bufferEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->findFilter = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->segments = null; + $this->writer = null; + $this->k = null; + $this->rt = null; + } + + /** + * The records matching the filters, in key order (a "key" filter: in its order) + * @param string $dbname + * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); + * a "key" entry selects by record key (int or list of ints) + * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + */ + public function find($dbname, $filters=0){ + $dbname = $this->layout->normalizeDbName($dbname); + if($filters === null || is_int($filters)){ + $filters = []; + } + if(!is_array($filters)){ + return false; + } + return $this->locks->withReadLock($dbname, function() use ($dbname, $filters){ + return $this->findUnlocked($dbname, $filters); + }); + } + + public function findUnlocked($dbname, $filters=0){ + if($filters === null || is_int($filters)){ + $filters = []; + } + if(!$this->segments->dbExists($dbname)){ + // Reads never create databases + return $this->autoCreateDB ? [] : false; + } + return array_values($this->findFilter->selectRecords($dbname, $filters)); + } + + /** + * Check if 'key' exists at top level of data (not nested) + * @param array $data + * @return bool + */ + private function hasReservedKeyField($data){ + return is_array($data) && array_key_exists("key", $data); + } + + /** + * Check if array is a list of records: sequential 0..n-1 keys, every element an array + * @param array $data + * @return bool + */ + private function isRecordList($data){ + if(!is_array($data) || count($data) === 0){ + return false; + } + $i = 0; + foreach($data as $k => $v){ + if($k !== $i++ || !is_array($v)){ + return false; + } + } + return true; + } + + /** + * Validate insert data + * @return array [items, error|null] + */ + private function validateInsertData($data){ + if(!is_array($data)){ + return [[], "insert data must be array"]; + } + if(empty($data)){ + return [[], "insert data must not be empty"]; + } + $keys = array_keys($data); + $looksLikeList = is_int($keys[0]) && is_array($data[$keys[0]]); + if($looksLikeList && !$this->isRecordList($data)){ + return [[], "insert data must be either one record or a list of records"]; + } + $items = $looksLikeList ? array_values($data) : [$data]; + return $this->io->withExactFloats(function() use ($items){ + foreach($items as $item){ + if($this->hasReservedKeyField($item)){ + return [[], "You cannot set key name to key"]; + } + if($this->io->encodeJson($item) === false){ + return [[], "insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"]; + } + } + return [$items, null]; + }); + } + + /** + * Insert one record or a list of records: each gets the next key + * @param string $dbname + * @param array $data One record, or a list of records + * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] + */ + public function insert($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + list($items, $error) = $this->validateInsertData($data); + if($error !== null){ + return array("n" => 0, "error" => $error); + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $items){ + if($this->bufferEnabled){ + return $this->buffer->insertBuffered($dbname, $items); + } + $this->buffer->flushPendingBuffers($dbname); + return $this->writer->insertRecords($dbname, $items); + }, 'insert'); + } + + /** + * Delete the records matching the filters + * @param string $dbname + * @param array $data Filters (same rules as find); [] deletes every record + * @return array ['n' => deleted count] + */ + public function delete($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + if(!is_array($data)){ + return array("n" => 0, "error" => "Please check your delete parameters"); + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $data){ + return $this->deleteUnlocked($dbname, $data); + }, 'delete'); + } + + private function deleteUnlocked($dbname, $data){ + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + return array("n" => 0); + } + $deleted = 0; + foreach($this->findFilter->segmentMatches($dbname, $data) as $segment => $records){ + $deleted += $this->writer->deleteSegmentRecords($dbname, $segment, $records); + } + return array("n" => $deleted); + } + + /** + * Set fields of the records matching the filters + * @param string $dbname + * @param array $data [filters, ['set' => [field => value, ...]]] + * @return array ['n' => updated count] + */ + public function update($dbname, $data){ + $dbname = $this->layout->normalizeDbName($dbname); + if(!is_array($data) || !array_key_exists(0, $data) || !is_array($data[0]) + || !isset($data[1]) || !is_array($data[1]) || !isset($data[1]['set']) || !is_array($data[1]['set'])){ + return array("n" => 0, "error" => "Please check your update parameters"); + } + $error = $this->updateSetError($data[1]['set']); + if($error !== null){ + return $error; + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $data){ + return $this->updateUnlocked($dbname, $data); + }, 'update'); + } + + private function updateUnlocked($dbname, $data){ + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + return array("n" => 0); + } + $updated = 0; + foreach($this->findFilter->segmentMatches($dbname, $data[0]) as $segment => $records){ + $updated += $this->updateRecords($dbname, $segment, $records, $data[1]['set']); + } + return array("n" => $updated); + } + + /** + * The result of update() for fields that can't be set ("key", values JSON can't represent); null if they can + * @return array|null + */ + public function updateSetError(array $set){ + if(array_key_exists("key", $set)){ + return array("n" => 0, "error" => "Please check your update parameters"); + } + if($this->io->encodeJson($set) === false){ + return array("n" => 0, "error" => "update data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)"); + } + return null; + } + + /** + * Set fields of records of one segment as read ([key => record]): update() and the query builder's update(). + * An empty set changes nothing and counts the records. Caller holds the exclusive lock. + * @return int + */ + public function updateRecords($dbname, $segment, array $records, array $set){ + if(empty($set)){ + return count($records); + } + foreach($records as $key => $record){ + foreach($set as $field => $value){ + $record[$field] = $value; + } + $records[$key] = $record; + } + return $this->writer->updateSegmentRecords($dbname, $segment, $records); + } +} diff --git a/src/Internal/Core/DatabaseOps.php b/src/Internal/Core/DatabaseOps.php new file mode 100644 index 0000000..19ad36c --- /dev/null +++ b/src/Internal/Core/DatabaseOps.php @@ -0,0 +1,371 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->layout, $this->locks, $this->scope, $this->segments, $this->shardMeta, $this->store); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->dbDir = &$this->k->dbDir; + $this->indexCache = &$this->k->indexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->directoryListings = &$this->rt->directoryListings; // @test + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->store = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // PUBLIC INDEX API + // ========================================== + + /** + * Make the key indexes (.kix checkpoints) of a database again from its data files, then its field and + * spatial indexes where they changed. + * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. + * @param string $dbname + * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] + */ + public function rebuildIndex($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->rebuildIndexUnlocked($dbname); + }, 'rebuildIndex'); + } + + private function rebuildIndexUnlocked($dbname){ + $start = microtime(true); + $this->buffer->flushPendingBuffers($dbname); + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return ['success' => false, 'error' => 'Database does not exist']; + } + $total = 0; + foreach($segments as $segment){ + $total += \noneDB\Internal\Storage\KeyIndex::count($this->checkpoints->rebuildSegmentIndexFromData($dbname, $segment)); + $this->cache->evictBetweenSegments(); + } + if($this->segments->isSharded($dbname)){ + $this->shardMeta->recountMeta($dbname); + } + // After a manual restore the field and spatial indexes are as stale as the key index + $this->checkpoints->rebuildSecondaryIndexes($dbname, $segments); + return [ + 'success' => true, + 'totalRecords' => $total, + 'time' => round((microtime(true) - $start) * 1000, 2) . 'ms' + ]; + } + + /** + * Get index information for a database + * @param string $dbname + * @return array|null + */ + public function getIndexInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getIndexInfoUnlocked($dbname); + }); + } + + private function getIndexInfoUnlocked($dbname){ + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return null; + } + $total = 0; + $created = null; + $updated = null; + foreach($segments as $segment){ + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + if($index === null){ + continue; + } + $total += \noneDB\Internal\Storage\KeyIndex::count($index); + $created = $created === null ? ($index['created'] ?? null) : min($created, $index['created'] ?? $created); + $updated = max((int)$updated, (int)($index['updated'] ?? 0)); + unset($index); + $this->cache->evictBetweenSegments(); + } + $sharded = $this->segments->isSharded($dbname); + return [ + 'exists' => true, + 'version' => 1, // of the key-index checkpoint (docs/STORAGE_FORMAT.md §6.4) + 'created' => $created, + 'updated' => $updated, + 'totalRecords' => $total, + 'sharded' => $sharded, + 'path' => $sharded ? $this->shardMeta->getMetaPath($dbname) : $this->checkpoints->getJsonlIndexPath($dbname, null) + ]; + } + + /** + * True when the database exists. With autoCreateDB a missing database is created; without it nothing is + * created and the answer is false. + * @param string $dbname + * @return bool + * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] + */ + public function checkDB($dbname=null){ + if($dbname === null || $dbname === '' || $dbname === false){ + return false; + } + $dbname = $this->layout->normalizeDbName($dbname); + if(!$this->autoCreateDB){ + // Only a check: a database that doesn't exist creates nothing (no directory, no lock file) + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->segments->dbExists($dbname); + }); + } + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->checkDBUnlocked($dbname); + }, 'checkDB'); + } + + private function checkDBUnlocked($dbname){ + if($this->segments->dbExists($dbname)){ + return true; + } + if($this->autoCreateDB){ + return $this->createDBUnlocked($dbname); + } + return false; + } + + /** + * Create a database (STORAGE_FORMAT.md §8.1) + * @param string $dbname + * @return bool True if created, false if it already existed + */ + public function createDB($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->createDBUnlocked($dbname); + }, 'createDB'); + } + + /** + * Create a database (STORAGE_FORMAT.md §8.1): generation 1 with an empty main segment and its first + * checkpoint, then CURRENT, then FORMAT. Caller holds the database lock exclusively (and STORE.LOCK + * shared: a database that doesn't exist is only locked through the missing-database path). + */ + public function createDBUnlocked($dbname){ + if($this->segments->dbExists($dbname)){ + return false; + } + $this->layout->createDatabaseFiles($dbname); + $dataPath = $this->segments->getDataPath($dbname); + $this->cache->markFileExists($dataPath); + $index = $this->checkpoints->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] = $index; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] = true; + $this->checkpoints->writeCheckpoint($dbname, null, $index); + return true; + } + + /** + * Delete a database (STORAGE_FORMAT.md §8.2): every file but its lock file + * @param string $dbname + * @return array ['success' => bool, 'files' => int deleted file count] + */ + public function dropDB($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + // Reads of a missing database create nothing: only an existing one is locked exclusively + if(!$this->locks->withLock($dbname, false, function() use ($dbname){ return $this->segments->dbExists($dbname); })){ + return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; + } + return $this->locks->withLock($dbname, true, function() use ($dbname){ + if(!$this->segments->dbExists($dbname)){ + return ['success' => false, 'files' => 0, 'error' => 'Database does not exist']; + } + $this->scope->refuseInUncommittedWrite($dbname, 'dropDB'); + $drop = $this->layout->dropDatabaseFiles($dbname, $this->locks->getLockPath($dbname), true); + if(!$drop['committed']){ + return ['success' => false, 'files' => 0, 'error' => "Database '{$dbname}' could not be dropped: its files can't be deleted (" + . implode(', ', $drop['left']) . ')']; + } + return ['success' => true, 'files' => $drop['removed']]; + }, 'dropDB'); + } + + /** + * Name, creation time and total size of a database (all shards included) + * @return array|false + */ + private function getDBInfo($dbname){ + return $this->locks->withLock($dbname, false, function() use ($dbname){ + $format = $this->layout->readFormatRecord($dbname); + if($format === null){ + return false; + } + $size = 0; + foreach($this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'segments/', '/\.jsonl$/') as $file){ + $size += (int)@filesize($file); + } + $info = array("name" => $dbname, "createdTime" => $format['createdAt'], "size" => $this->fileSizeConvert($size)); + // Shard files found without a shards.json entry are kept aside (see setOrphanShardsAside) + $orphans = array_map('basename', $this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'segments/', '/\.orphan\.\d+$/')); + if(!empty($orphans)){ + $info['orphanFiles'] = $orphans; + } + return $info; + }); + } + /** + * Convert bytes to human readable format + * @param float $bytes + * @return string + */ + private function fileSizeConvert($bytes){ + $bytes = floatval($bytes); + $arBytes = array( + 0 => array("UNIT" => "TB", "VALUE" => pow(1024, 4)), + 1 => array("UNIT" => "GB", "VALUE" => pow(1024, 3)), + 2 => array("UNIT" => "MB", "VALUE" => pow(1024, 2)), + 3 => array("UNIT" => "KB", "VALUE" => 1024), + 4 => array("UNIT" => "B", "VALUE" => 1), + ); + $result = "0 B"; + foreach($arBytes as $arItem){ + if($bytes >= $arItem["VALUE"]){ + $result = $bytes / $arItem["VALUE"]; + $result = str_replace(".", "," , strval(round($result, 2)))." ".$arItem["UNIT"]; + break; + } + } + return $result; + } + + /** + * The databases of this key: names (false), names with information (true), or one database's information + * (its name; false when it doesn't exist) + */ + public function getDBs($info=false){ + // Handle three cases: false (names only), true (with metadata), string (specific db) + $withMetadata = false; + $specificDb = null; + + if(is_bool($info)){ + $withMetadata = $info; + }else{ + $specificDb = $this->layout->normalizeDbName($info); // reading information never creates the database + } + + // If specific database requested + if($specificDb !== null){ + return $this->getDBInfo($specificDb); + } + + // List all databases of this key: {locator}.ndb4 directories whose FORMAT names a database that + // maps to that locator with this key + $dbs = []; + if(!$this->store->openStore(false)){ + return $dbs; + } + $names = []; + $this->directoryListings++; // @test + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; // a link is no database (STORAGE_FORMAT.md §3.4) + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + $dbname = is_array($format) && is_string($format['name'] ?? null) ? $format['name'] : null; + if($dbname === null || preg_match("/^[A-Za-z0-9' -]+$/", $dbname) !== 1 || $this->store->locator($dbname) !== $m[1]){ + continue; + } + $names[] = $dbname; + } + sort($names, SORT_STRING); + foreach($names as $dbname){ + if($withMetadata){ + $info = $this->getDBInfo($dbname); + if($info !== false){ + $dbs[] = $info; + } + }else{ + $dbs[] = $dbname; + } + } + return $dbs; + } + + /** + * The first $limit records of a list of records (a result of find()) + * @param array $array + * @param int $limit At least 1 + * @return array|false False for invalid arguments + */ + public function limit($array, $limit=0){ + if(!is_array($array) || !is_int($limit) || $limit <= 0){ + return false; + } + if(empty($array)){ + return []; + } + // Must be a list of records + if(count($array) === count($array, COUNT_RECURSIVE)) { + return false; + } + return array_slice($array, 0, $limit); + } +} diff --git a/src/Internal/Core/Kernel.php b/src/Internal/Core/Kernel.php new file mode 100644 index 0000000..94a7a23 --- /dev/null +++ b/src/Internal/Core/Kernel.php @@ -0,0 +1,331 @@ + values linked to the process-wide caches (shared by every instance) */ + public $linked = []; + /** @var \noneDB\Internal\Config\Configuration */ + public $config; + /** @var \noneDB\Internal\Cache\CacheManager */ + public $cache; + /** @var \noneDB\Internal\Storage\StoreRecord */ + public $store; + /** @var \noneDB\Internal\Storage\Layout */ + public $layout; + /** @var \noneDB\Internal\Storage\Segments */ + public $segments; + /** @var \noneDB\Internal\Storage\Checkpoints */ + public $checkpoints; + /** @var \noneDB\Internal\Storage\RecordIO */ + public $records; + /** @var \noneDB\Internal\Storage\Writer */ + public $writer; + /** @var \noneDB\Internal\Storage\Buffer */ + public $buffer; + /** @var \noneDB\Internal\Concurrency\LockManager */ + public $locks; + /** @var \noneDB\Internal\Concurrency\Snapshot */ + public $snapshot; + /** @var \noneDB\Internal\Recovery\RollbackScope */ + public $scope; + /** @var \noneDB\Internal\Recovery\RecoveryLog */ + public $recoveryLog; + /** @var \noneDB\Internal\Recovery\Recovery */ + public $recovery; + /** @var \noneDB\Internal\Durability\FileIO */ + public $io; + /** @var \noneDB\Internal\Durability\Sync */ + public $sync; + /** @var \noneDB\Internal\Sharding\ShardMeta */ + public $shardMeta; + /** @var \noneDB\Internal\Sharding\ShardRouter */ + public $shardRouter; + /** @var \noneDB\Internal\Sharding\LayoutSwitch */ + public $layoutSwitch; + /** @var \noneDB\Internal\Sharding\ShardInfo */ + public $shardInfo; + /** @var \noneDB\Internal\Index\SecondaryFiles */ + public $secondary; + /** @var \noneDB\Internal\Index\FieldIndex */ + public $fieldIndex; + /** @var \noneDB\Internal\Index\FieldIndexApi */ + public $fieldIndexApi; + /** @var \noneDB\Internal\Index\Spatial\GeoJson */ + public $geoJson; + /** @var \noneDB\Internal\Index\Spatial\Geometry */ + public $geometry; + /** @var \noneDB\Internal\Index\Spatial\RTree */ + public $rtree; + /** @var \noneDB\Internal\Index\Spatial\SpatialIndexApi */ + public $spatialIndexApi; + /** @var \noneDB\Internal\Query\FindFilter */ + public $findFilter; + /** @var \noneDB\Internal\Query\Values */ + public $values; + /** @var \noneDB\Internal\Query\Planner */ + public $planner; + /** @var \noneDB\Internal\Query\QueryMethods */ + public $queryMethods; + /** @var \noneDB\Internal\Query\Aggregation */ + public $aggregation; + /** @var \noneDB\Internal\Query\Utility */ + public $utility; + /** @var \noneDB\Internal\Query\SpatialQueries */ + public $spatialQueries; + /** @var \noneDB\Internal\Core\Crud */ + public $crud; + /** @var \noneDB\Internal\Core\DatabaseOps */ + public $db; + /** @var \noneDB\Internal\Diagnostics\Integrity */ + public $integrity; + /** @var \noneDB\Internal\Diagnostics\Metrics */ + public $metrics; + /** @var \noneDB\Internal\Maintenance\Maintenance */ + public $maintenance; + /** @var \noneDB\Internal\Maintenance\Compaction */ + public $compaction; + /** @var \noneDB\Internal\Maintenance\KeyMaintenance */ + public $keyMaintenance; + /** @var \noneDB\Internal\Maintenance\BackupRestore */ + public $backupRestore; + + + public $dbDir = null; // Set via .nonedb config file or constructor + public $autoCreateDB = true; + public $devMode = false; + + // Sharding configuration + public $shardingEnabled=true; // A database switches to the sharded layout once it has assigned shardSize keys + public $shardSize=10000; // Records per shard of databases created from now on (each keeps the size in its FORMAT) + + // File locking configuration + public $lockTimeout=5; // Max seconds to wait for a database lock (0 = wait forever) + public $lockRetryDelay=10000; // Microseconds between lock attempts (10ms) + + // Write buffer configuration (off by default: direct writes are faster and always visible) + public $bufferEnabled=false; // Enable/disable write buffering + public $bufferSizeLimit=1048576; // 1MB buffer size limit per buffer + public $bufferCountLimit=10000; // Max records per buffer (safety limit) + public $bufferFlushInterval=30; // Seconds between auto-flush (0 = disabled) + public $bufferAutoFlushOnShutdown=true; // Register shutdown handler for flush + + // Buffer state tracking (runtime) + public $bufferLastFlush=[]; // Track last flush time per DB/shard + public $shutdownHandlerRegistered=false; // Track if shutdown handler is registered + + // Performance caches (runtime). All keys are full file paths (or key-fingerprint|dbname for hashes), + // so instances with different dbDir/secretKey never see each other's entries. + public $hashCache=[]; // keyFingerprint|dbname -> locator + public $metaCache=[]; // meta path -> meta data + + // Index configuration + public $indexCache=[]; // checkpoint path -> key index of the segment (Checkpoints, KeyIndex) + public $verifiedIndexes=[]; // checkpoint path -> true: the cached index covers the data file as it is + public $shardedCache=[]; // meta path -> bool + + // JSONL Storage Engine + public $autoCompactThreshold=0.3; // Trigger compaction when garbage > 30% of live records + public $generations=[]; // per-instance view of $staticGenerations + + // Durability mode (docs/DURABILITY.md): true = 'durable', every sync of §3 is done and required + public $durable=false; + + // Permissions of what noneDB creates in dbDir, whatever the umask (docs/STORAGE_FORMAT.md §3) + public $filePermissions = 0640; + public $directoryPermissions = 0750; + + // Field indexing configuration + public $fieldIndexEnabled = true; // Enable field-based indexing + public $fieldIndexCache = []; // field index path -> index + + // Spatial indexing configuration + public $spatialIndexEnabled = true; // Enable spatial R-tree indexing + public $spatialIndexCache = []; // spatial index path -> index + public $globalSpatialIndexCache = []; // global spatial index path -> index + public $rtreeNodeSize = 32; // Max entries per R-tree node + public $appendMismatch = false; // appendToFile(): the file didn't have the expected size + public $dataSizes = []; // data path -> size, until another process writes (global index skips) + public $dataEpochs = []; // data path -> segment epoch read from its checkpoint (global index entries) + public $shardFilesUpTo = []; // database file prefix -> highest shard number with files (setOrphanShardsAside) + public $globalSkip = []; // lock path -> global field index path -> shards a lookup checks (globalCheckShards) + public $segWrites = []; // lock path -> ['seq' => n, 'paths' => [data path => seq]]: data files this process wrote + public $writableIndexes = []; // checkpoint path -> lock session id in which it was checked against its data file + public $indexedCandidates = []; // key => true: the last spatialCandidates() found it in an R-tree (valid geometry) + + public $keyFingerprint = null; // HMAC of the secret, used to bind caches to the key + + public $cloneSourceId = null; + // ========================================== + // DATABASE DIRECTORY (docs/STORAGE_FORMAT.md §4, §5, §8) + // ========================================== + // + // {dbDir}{locator}.ndb4/ holds FORMAT (identity), LOCK (lock + state record), CURRENT (active + // generation) and generations/{N}/ (metadata, shards.json, indexes.json, segments/, indexes/, + // spatial/, buffer/). The generation number of a database is read from CURRENT once per lock epoch. + + public $currentGenerations = []; // lock path => active generation number + public $formatRecords = []; // lock path => FORMAT record of a present database + + public $dbHomes = []; // database name => its directory (dbDir and key are fixed per instance) + public $lockPaths = []; // database name => its LOCK file + public $genPaths = []; // database name => paths of its active generation (genDir()) + + /** + * Kernels of the live noneDB instances by the instance's id. The instance holds only that id: a dump of + * it (var_export, json_encode) shows nothing internal, and no reference cycle is reachable from it. + * @var array + */ + public static $instances = []; + + /** The services: kernel property => class (created on first use) */ + public const SERVICES = [ + 'config' => \noneDB\Internal\Config\Configuration::class, + 'cache' => \noneDB\Internal\Cache\CacheManager::class, + 'store' => \noneDB\Internal\Storage\StoreRecord::class, + 'layout' => \noneDB\Internal\Storage\Layout::class, + 'segments' => \noneDB\Internal\Storage\Segments::class, + 'checkpoints' => \noneDB\Internal\Storage\Checkpoints::class, + 'records' => \noneDB\Internal\Storage\RecordIO::class, + 'writer' => \noneDB\Internal\Storage\Writer::class, + 'buffer' => \noneDB\Internal\Storage\Buffer::class, + 'locks' => \noneDB\Internal\Concurrency\LockManager::class, + 'snapshot' => \noneDB\Internal\Concurrency\Snapshot::class, + 'scope' => \noneDB\Internal\Recovery\RollbackScope::class, + 'recoveryLog' => \noneDB\Internal\Recovery\RecoveryLog::class, + 'recovery' => \noneDB\Internal\Recovery\Recovery::class, + 'io' => \noneDB\Internal\Durability\FileIO::class, + 'sync' => \noneDB\Internal\Durability\Sync::class, + 'shardMeta' => \noneDB\Internal\Sharding\ShardMeta::class, + 'shardRouter' => \noneDB\Internal\Sharding\ShardRouter::class, + 'layoutSwitch' => \noneDB\Internal\Sharding\LayoutSwitch::class, + 'shardInfo' => \noneDB\Internal\Sharding\ShardInfo::class, + 'secondary' => \noneDB\Internal\Index\SecondaryFiles::class, + 'fieldIndex' => \noneDB\Internal\Index\FieldIndex::class, + 'fieldIndexApi' => \noneDB\Internal\Index\FieldIndexApi::class, + 'geoJson' => \noneDB\Internal\Index\Spatial\GeoJson::class, + 'geometry' => \noneDB\Internal\Index\Spatial\Geometry::class, + 'rtree' => \noneDB\Internal\Index\Spatial\RTree::class, + 'spatialIndexApi' => \noneDB\Internal\Index\Spatial\SpatialIndexApi::class, + 'findFilter' => \noneDB\Internal\Query\FindFilter::class, + 'values' => \noneDB\Internal\Query\Values::class, + 'planner' => \noneDB\Internal\Query\Planner::class, + 'queryMethods' => \noneDB\Internal\Query\QueryMethods::class, + 'aggregation' => \noneDB\Internal\Query\Aggregation::class, + 'utility' => \noneDB\Internal\Query\Utility::class, + 'spatialQueries' => \noneDB\Internal\Query\SpatialQueries::class, + 'crud' => \noneDB\Internal\Core\Crud::class, + 'db' => \noneDB\Internal\Core\DatabaseOps::class, + 'integrity' => \noneDB\Internal\Diagnostics\Integrity::class, + 'metrics' => \noneDB\Internal\Diagnostics\Metrics::class, + 'maintenance' => \noneDB\Internal\Maintenance\Maintenance::class, + 'compaction' => \noneDB\Internal\Maintenance\Compaction::class, + 'keyMaintenance' => \noneDB\Internal\Maintenance\KeyMaintenance::class, + 'backupRestore' => \noneDB\Internal\Maintenance\BackupRestore::class, + ]; + + public function __construct($config = null){ + $this->rt = Runtime::get(); + unset($this->config, $this->cache, $this->store, $this->layout, $this->segments, $this->checkpoints, $this->records, $this->writer, $this->buffer, $this->locks, $this->snapshot, $this->scope, $this->recoveryLog, $this->recovery, $this->io, $this->sync, $this->shardMeta, $this->shardRouter, $this->layoutSwitch, $this->shardInfo, $this->secondary, $this->fieldIndex, $this->fieldIndexApi, $this->geoJson, $this->geometry, $this->rtree, $this->spatialIndexApi, $this->findFilter, $this->values, $this->planner, $this->queryMethods, $this->aggregation, $this->utility, $this->spatialQueries, $this->crud, $this->db, $this->integrity, $this->metrics, $this->maintenance, $this->compaction, $this->keyMaintenance, $this->backupRestore); + $this->linkCaches(); + $this->config->initialize($config); + } + + /** + * A service, created on first use: it binds the kernel's values by reference (after linkCaches(), so + * its references follow the links) + */ + public function __get(string $name) { + $class = self::SERVICES[$name] ?? null; + if($class === null){ + throw new \LogicException('noneDB: no service ' . $name); + } + $service = new $class($this); + $this->$name = $service; + $service->bind(); + return $service; + } + + /** + * Link the instance's caches to the process-wide ones (static cache sharing): before any service exists + */ + private function linkCaches(): void { + // Link instance caches to static caches for cross-instance sharing + if($this->rt->staticCacheEnabled){ + $this->indexCache = &$this->rt->staticIndexCache; + $this->verifiedIndexes = &$this->rt->staticVerifiedIndexes; + $this->shardedCache = &$this->rt->staticShardedCache; + $this->metaCache = &$this->rt->staticMetaCache; + $this->hashCache = &$this->rt->staticHashCache; + $this->fieldIndexCache = &$this->rt->staticFieldIndexCache; + $this->spatialIndexCache = &$this->rt->staticSpatialIndexCache; + $this->globalSpatialIndexCache = &$this->rt->staticGlobalSpatialIndexCache; + $this->generations = &$this->rt->staticGenerations; + $this->writableIndexes = &$this->rt->staticWritableIndexes; + $this->dataSizes = &$this->rt->staticDataSizes; + $this->globalSkip = &$this->rt->staticGlobalSkip; + $this->segWrites = &$this->rt->staticSegWrites; + $this->dataEpochs = &$this->rt->staticDataEpochs; + $this->shardFilesUpTo = &$this->rt->staticShardFilesUpTo; + $this->currentGenerations = &$this->rt->staticCurrentGenerations; + $this->formatRecords = &$this->rt->staticFormatRecords; + $this->linked = array_fill_keys(['indexCache', 'verifiedIndexes', 'shardedCache', 'metaCache', 'hashCache', 'fieldIndexCache', 'spatialIndexCache', 'globalSpatialIndexCache', 'generations', 'writableIndexes', 'dataSizes', 'globalSkip', 'segWrites', 'dataEpochs', 'shardFilesUpTo', 'currentGenerations', 'formatRecords'], true); + } + } + + private function unsetServices(): void { + unset($this->config, $this->cache, $this->store, $this->layout, $this->segments, $this->checkpoints, $this->records, $this->writer, $this->buffer, $this->locks, $this->snapshot, $this->scope, $this->recoveryLog, $this->recovery, $this->io, $this->sync, $this->shardMeta, $this->shardRouter, $this->layoutSwitch, $this->shardInfo, $this->secondary, $this->fieldIndex, $this->fieldIndexApi, $this->geoJson, $this->geometry, $this->rtree, $this->spatialIndexApi, $this->findFilter, $this->values, $this->planner, $this->queryMethods, $this->aggregation, $this->utility, $this->spatialQueries, $this->crud, $this->db, $this->integrity, $this->metrics, $this->maintenance, $this->compaction, $this->keyMaintenance, $this->backupRestore); + } + + /** + * A copy for a cloned noneDB instance: its own values (settings, per-instance caches), the same links to + * the process-wide caches, its own services + */ + public function cloneKernel(): self { + $copy = clone $this; + // The services hold the kernel's values by reference, so a plain clone would share every value with + // this kernel. Only the caches linked to the process-wide ones are shared; everything else is copied. + foreach (get_object_vars($this) as $name => $value) { + if ($name === 'rt' || is_object($value) || isset($this->linked[$name])) { + continue; + } + unset($copy->$name); + $copy->$name = $value; + } + $copy->unsetServices(); + $copy->config->cloned(); + return $copy; + } + + /** + * Drop the handles between the services and this kernel (the noneDB instance is gone): without the + * cycles the memory is freed at once, not by the cycle collector later. Values bound by reference + * (settings, caches, hooks) are not touched; only handles to engine objects are cleared. + */ + public function release(): void { + foreach (self::SERVICES as $name => $class) { + if (isset($this->$name)) { + $this->$name->unbind(); + $this->$name = null; + } + } + } + + /** + * The kernel of a noneDB instance + */ + public static function of(\noneDB $db): self { + static $id = null; + $id = $id ?? \Closure::bind(function (\noneDB $db) { return $db->id; }, null, \noneDB::class); + return self::$instances[$id($db)]; + } +} diff --git a/src/Internal/Diagnostics/Integrity.php b/src/Internal/Diagnostics/Integrity.php new file mode 100644 index 0000000..35f8409 --- /dev/null +++ b/src/Internal/Diagnostics/Integrity.php @@ -0,0 +1,384 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->geoJson, $this->geometry, $this->io, $this->layout, $this->locks, $this->records, $this->recoveryLog, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->integrityChecks = &$this->rt->integrityChecks; + $this->lastIntegrity = &$this->rt->lastIntegrity; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recoveryLog = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Check a database against its own data, read-only: every line of every data file is + * decoded; the key index, field indexes, spatial indexes and shard meta are compared with what the + * data says. Nothing is written, repaired, flushed or recovered (rebuildIndex() repairs). + * @param string $dbname + * @return array ['ok' => bool, 'segments', 'lines', 'records', 'badLines' => [[segment, offset, length]], + * 'tornTail', 'indexMismatches', 'bufferedRecords', 'pendingRecovery', 'quarantineFiles', + * 'recoveryLog' => last entries] + */ + public function checkIntegrity($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + $report = $this->integrityReport($dbname); + // The result getMetrics() reports (docs/OPERATIONS.md §1), by database instance: a database dropped and + // created again is another one + $this->lastIntegrity = ['database' => $dbname, 'ok' => $report['ok'], 'time' => time(), 'records' => $report['records'], + 'issues' => count($report['badLines']) + count($report['tornTail']) + count($report['indexMismatches'])]; + $format = $this->layout->readFormatRecord($dbname); + if($format !== null){ + $this->integrityChecks[$format['databaseId']] = $this->lastIntegrity; + } + return $report; + } + + /** + * The report of checkIntegrity(), not kept for getMetrics() (restore() verifies a generation with it) + */ + public function integrityReport($dbname){ + $check = function() use ($dbname){ + return $this->checkIntegrityLocked($dbname); + }; + $this->noWrites++; + try { + // A missing database is answered without a lock: nothing is created + return $this->locks->withLock($dbname, false, $check); + } finally { + $this->noWrites--; + } + } + + private function checkIntegrityLocked($dbname){ + $report = ['ok' => true, 'database' => $dbname, 'exists' => false, 'segments' => 0, 'lines' => 0, 'records' => 0, + 'badLines' => [], 'tornTail' => [], 'indexMismatches' => [], 'bufferedRecords' => 0, 'pendingRecovery' => false, + 'quarantineFiles' => [], 'recoveryLog' => []]; + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + if($lock !== null && (!empty($lock['flags']['j']) || !empty($lock['flags']['c']) || !empty($lock['flags']['d']) + || !empty($lock['flags']['g']) || $lock['epoch'] === null)){ + $report['pendingRecovery'] = true; // the next writer or reader recovers (journal, creation, drop, generation) + } + if(!$this->layout->dbPresent($dbname)){ + $report['ok'] = !$report['pendingRecovery']; + return $report; + } + // Generations other than CURRENT's: left by an interrupted switch, never removed without proof + $current = $this->layout->generationNumber($dbname); + foreach(@scandir($this->layout->dbHome($dbname) . 'generations') ?: [] as $generation){ + if(ctype_digit((string)$generation) && (int)$generation !== $current){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'generation', 'detail' => "generation $generation is not the current one ($current)"]; + } + } + $segPrefix = $this->layout->genDir($dbname) . 'segments/s'; + $metaPath = $this->shardMeta->getMetaPath($dbname); + clearstatcache(true, $metaPath); + $sharded = file_exists($metaPath); + $meta = $sharded ? $this->shardMeta->readMeta($dbname) : null; + if($sharded && $meta === null){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'meta file unreadable']; + } + $segments = $sharded ? array_map(function($f) use ($segPrefix){ return (int)substr($f, strlen($segPrefix)); }, + $this->segments->listFilesWithPrefix($segPrefix, '/^\d+\.jsonl$/')) : (is_file($this->segments->getDataPath($dbname)) ? [null] : []); + sort($segments); + $report['exists'] = !empty($segments) || $sharded; + $size = $meta !== null ? (int)$meta['shardSize'] : $this->segments->shardSizeOf($dbname); + if($sharded && $meta === null){ + try { + $size = (int)$this->shardMeta->deriveShardMeta($dbname)['shardSize']; // as recovery would rebuild it + } catch(\RuntimeException $e){ + } + } + $manifest = $this->deriveOrReadManifest($dbname); + $liveTotal = 0; + $maxKey = -1; + $byShard = []; + + $done = false; // a segment was checked: the cache may drop what it loaded before the next one + foreach($segments as $shardId){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $report['segments']++; + $seg = $shardId === null ? 'main' : $shardId; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + // The live records of the data: key => offset, key => length (as a key index holds them) + $livePos = []; + $liveLen = []; + $offset = 0; + $fp = @fopen($dataPath, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $report['tornTail'][] = ['segment' => $seg, 'offset' => $offset, 'length' => strlen($line)]; + break; + } + $report['lines']++; + $entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $shardId === null ? 0 : $shardId * $size, + $shardId === null ? null : ($shardId + 1) * $size); + if($entry === null){ + $report['badLines'][] = [$seg, $offset, strlen($line)]; + } elseif($entry[0] === 't'){ + unset($livePos[$entry[1]], $liveLen[$entry[1]]); + $maxKey = max($maxKey, $entry[1]); + } else { + $livePos[$entry[1]] = $offset; + $liveLen[$entry[1]] = $entry[3]; + $maxKey = max($maxKey, $entry[1]); + } + $offset += strlen($line); + } + if($fp !== false){ + fclose($fp); + } + ksort($livePos, SORT_NUMERIC); + $liveTotal += count($livePos); + $byShard[$shardId === null ? 'main' : $shardId] = count($livePos); + + // Key index: its checkpoint fits the data file, and with the complete lines after it + // gives the same records at the same places. A missing one is rebuilt from the data on the next + // access (normal: nothing lost), a broken one too, but that is reported. + $kixPath = \noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath); + clearstatcache(true, $kixPath); + $raw = is_file($kixPath) && filesize($kixPath) > 0 ? @file_get_contents($kixPath) : null; // empty: missing + $file = $raw === null ? null : \noneDB\Internal\Storage\KeyIndex::decodeFile($raw); + clearstatcache(true, $dataPath); + $fileSize = (int)@filesize($dataPath); + $index = null; + if($raw !== null && $file === null){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'not a valid checkpoint (rebuilt on next access)']; + } elseif($file !== null){ + if(!$this->checkpoints->checkpointFits($dataPath, $file[0]['s'], $fileSize, $file[0]['h'] ?? null)){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => 'checkpoint does not fit the data file (rebuilt on next access)']; + } else { + $index = \noneDB\Internal\Storage\KeyIndex::fromFile($file[0], $file[1]); + } + } + if($index !== null){ + $this->checkpoints->replaySegment($dbname, $shardId, $index, $fileSize, null, false); + [$indexPos, $indexLen] = \noneDB\Internal\Storage\KeyIndex::locate($index, null); + if($indexPos != $livePos || $indexLen != $liveLen){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'key', 'detail' => count(array_diff_key($livePos, $indexPos)) . ' records not indexed, ' + . count(array_diff_key($indexPos, $livePos)) . ' indexed records not in the data or elsewhere']; + } + $indexPos = $indexLen = null; + } + + // Field and spatial indexes against the records: what they must hold, from the records read a batch + // at a time (RecordIO::BATCH, in key order) + $expectedFields = array_fill_keys($manifest['fields'], []); + $expectedSpatial = array_fill_keys($manifest['spatial'], []); + foreach(empty($manifest['fields']) && empty($manifest['spatial']) ? [] : array_chunk($livePos, \noneDB\Internal\Storage\RecordIO::BATCH, true) as $chunk){ + $records = $this->records->readJsonlRecordsBatch($dataPath, $chunk, array_intersect_key($liveLen, $chunk)); + ksort($records, SORT_NUMERIC); + foreach($manifest['fields'] as $field){ + foreach($records as $key => $record){ + if(array_key_exists($field, $record) && $this->fieldIndex->isIndexableValue($record[$field])){ + $bucket = \noneDB\Internal\Query\Values::valueKey($record[$field]); + $held = $expectedFields[$field][$bucket] ?? null; + if($held === null){ + $expectedFields[$field][$bucket] = (int)$key; + } elseif(is_int($held)){ + $expectedFields[$field][$bucket] = [$held, (int)$key]; + } else { + $expectedFields[$field][$bucket][] = (int)$key; + } + } + } + } + foreach($manifest['spatial'] as $field){ + foreach($records as $key => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid'] && $this->geometry->calculateMBR($geometry) !== null){ + $expectedSpatial[$field][] = (int)$key; + } + } + } + $records = null; + } + // Field and spatial indexes: one that fits the segment (its epoch, the bytes it covers), with the + // lines after it replayed, must hold exactly the records; one that doesn't fit is rebuilt when + // used (not damage). + $keyEpoch = is_array($index) && isset($index['e']) && is_string($index['e']) ? $index['e'] : null; + foreach($manifest['fields'] as $field){ + $actual = $this->integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + continue; // built on first use + } + \noneDB\Internal\Index\FieldIndex::canonicalBuckets($expectedFields[$field]); + \noneDB\Internal\Index\FieldIndex::canonicalBuckets($actual); + if($actual !== $expectedFields[$field]){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'field ' . $field, 'detail' => 'buckets differ from the records']; + } + } + foreach($manifest['spatial'] as $field){ + $expected = $expectedSpatial[$field]; + $actual = $this->integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $offset); + if($actual === null){ + continue; + } + sort($actual); + if($actual !== $expected){ + $report['indexMismatches'][] = ['segment' => $seg, 'index' => 'spatial ' . $field, 'detail' => 'entries differ from the records']; + } + } + } + $report['records'] = $liveTotal; + + if($meta !== null){ + $listed = array_map('intval', array_column($meta['shards'], 'id')); + sort($listed); + if($listed !== $segments){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => 'shards listed in the meta differ from the shard files']; + } + if((int)($meta['totalRecords'] ?? -1) !== $liveTotal){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "totalRecords {$meta['totalRecords']}, data has $liveTotal"]; + } + if((int)$meta['nextKey'] <= $maxKey){ + $report['indexMismatches'][] = ['segment' => null, 'index' => 'meta', 'detail' => "nextKey {$meta['nextKey']} would reuse key $maxKey"]; + } + } + + foreach(array_merge([$this->buffer->getBufferPath($dbname)], $this->buffer->flushingFiles($dbname)) as $buffer){ + if(is_file($buffer)){ + $report['bufferedRecords'] += count($this->buffer->readBufferRecords($buffer)); + } + } + $report['quarantineFiles'] = array_map('basename', $this->segments->listFilesWithPrefix($this->layout->dbHome($dbname) . 'recovery/q.', '/\.bin$/')); + $report['recoveryLog'] = array_slice($this->recoveryLog->getRecoveryLogEntries($dbname), -20); + $report['ok'] = empty($report['badLines']) && empty($report['tornTail']) && empty($report['indexMismatches']) && !$report['pendingRecovery']; + return $report; + } + + /** + * The index list without writing anything (checkIntegrity) + */ + private function deriveOrReadManifest($dbname){ + $path = $this->fieldIndex->getIndexManifestPath($dbname); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? [] : json_decode($content, true); + if(!is_array($manifest) || ($content !== false && ($manifest['kind'] ?? null) !== 'indexes')){ + $manifest = $this->fieldIndex->deriveIndexManifest($dbname); + } + return ['fields' => array_values($manifest['fields'] ?? []), 'spatial' => array_values($manifest['spatial'] ?? [])]; + } + + /** + * checkIntegrity(): buckets of a segment's field index that fits the key checkpoint, with the + * data lines up to $complete replayed; null when there is none that fits + */ + private function integrityFieldBuckets($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->secondary->readSecondaryFile($this->fieldIndex->getFieldIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'fx', $field, $keyEpoch, $complete)){ + return null; + } + $buckets = $this->fieldIndex->decodeFieldIndexBody($file[0], $file[1]); + if($buckets === null){ + return []; // pages that can't be read hold none of the records + } + $index = $this->fieldIndex->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $this->fieldIndex->replayFieldIndex($dbname, $shardId, $index, $complete); + return $index['b']; + } + + /** + * checkIntegrity(): keys of a segment's R-tree that fits the key checkpoint, replayed up to + * $complete; null when there is none that fits + */ + private function integritySpatialKeys($dbname, $shardId, $field, $keyEpoch, $complete){ + $file = $this->secondary->readSecondaryFile($this->rtree->getSpatialIndexPath($dbname, $field, $shardId)); + if($file === null || !$this->integrityHeaderFits($dbname, $shardId, $file[0], 'rx', $field, $keyEpoch, $complete)){ + return null; + } + $tree = $this->rtree->treeOf($field, $file[0], $file[1]); + if($tree === null){ + return []; // a body that can't be read holds none of the records + } + $this->rtree->replaySpatialIndex($dbname, $shardId, $tree, $complete); + $this->rtree->rtreeMaps($tree); + return array_keys($tree['leafOf']); + } + + private function integrityHeaderFits($dbname, $shardId, array $header, $kind, $field, $keyEpoch, $complete){ + return ($header['kind'] ?? null) === $kind && ($header['field'] ?? null) === (string)$field && $keyEpoch !== null + && ($header['e'] ?? null) === $keyEpoch && is_int($header['s'] ?? null) && $header['s'] >= 0 && $header['s'] <= $complete + && $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), $header['s']) === ($header['h'] ?? null); + } +} diff --git a/src/Internal/Diagnostics/Metrics.php b/src/Internal/Diagnostics/Metrics.php new file mode 100644 index 0000000..2a32848 --- /dev/null +++ b/src/Internal/Diagnostics/Metrics.php @@ -0,0 +1,182 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->fieldIndex, $this->layout, $this->locks, $this->shardInfo, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->integrityChecks = &$this->rt->integrityChecks; + $this->lastIntegrity = &$this->rt->lastIntegrity; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->fieldIndex = null; + $this->layout = null; + $this->locks = null; + $this->shardInfo = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + /** + * ['process' => ...] and, for a database, ['database' => ...]: nothing is scanned or written + */ + public function getMetrics($dbname){ + $metrics = ['process' => $this->processMetrics()]; + if($dbname !== null){ + $metrics['database'] = $this->databaseMetrics($this->layout->normalizeDbName($dbname)); + } + return $metrics; + } + + private function processMetrics(){ + $m = $this->metrics; + $durability = $this->sync->getDurability(); + return [ + 'indexCache' => ['hits' => $m['hits'], 'misses' => $m['misses'], 'evictions' => $m['evictions'], + 'limit' => \noneDB\Internal\Cache\CacheManager::cacheLimitBytes()], + 'indexReads' => ['pages' => $m['pages'], 'bytes' => $m['bytes']], + 'checkpoints' => ['written' => $m['checkpoints'], 'seconds' => $m['checkpointSeconds']], + 'compactions' => ['count' => $m['compactions'], 'seconds' => $m['compactionSeconds']], + 'locks' => ['acquired' => $m['locks'], 'waited' => $m['lockWaits'], 'waitSeconds' => $m['lockWaitSeconds'], 'timeouts' => $m['lockTimeouts']], + 'recovery' => ['events' => $m['recoveryEvents'], 'actions' => $m['recoveryActions']], + 'durability' => ['mode' => $durability['mode'], 'lastOperation' => $durability['lastOperation'], + 'lastSyncError' => $durability['lastSyncError'], 'syncs' => $durability['syncs']], + 'integrity' => $this->lastIntegrity, + 'memory' => ['usage' => memory_get_usage(), 'peak' => memory_get_peak_usage()], + ]; + } + + /** + * A database from its metadata, under its shared lock, writing nothing (as checkIntegrity() reads) + */ + private function databaseMetrics($dbname){ + $this->noWrites++; + try { + return $this->locks->withLock($dbname, false, function() use ($dbname){ + if(!$this->layout->dbPresent($dbname)){ + return ['name' => $dbname, 'exists' => false]; // a missing database is answered without a lock + } + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + $flags = $this->lockRegistry[$this->locks->getLockPath($dbname)]['flags']; + $buffered = $this->buffer->getBufferRecordCount($this->buffer->getBufferPath($dbname)); + foreach($this->buffer->flushingFiles($dbname) as $file){ + $buffered += count($this->buffer->readBufferRecords($file)); + } + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $databaseId = $this->layout->readFormatRecord($dbname)['databaseId']; + return [ + 'name' => $dbname, + 'exists' => true, + 'databaseId' => $databaseId, + 'generation' => $this->layout->generationNumber($dbname), + 'layout' => $info['sharded'] ? 'sharded' : 'single', + 'shardSize' => $info['shardSize'], + 'segments' => $info['sharded'] ? $info['shards'] : 1, + 'records' => $info['totalRecords'], + 'nextKey' => $info['nextKey'], + 'bufferedRecords' => $buffered, + 'fieldIndexes' => array_values($manifest['fields']), + 'spatialIndexes' => array_values($manifest['spatial']), + 'bytes' => $this->databaseBytes($dbname), + 'state' => ['buffered' => !empty($flags['b']), + 'recoveryPending' => !empty($flags['j']) || !empty($flags['c']) || !empty($flags['d']) || !empty($flags['g'])], + 'integrity' => $this->integrityChecks[$databaseId] ?? null, + ]; + }); + } finally { + $this->noWrites--; + } + } + + /** + * Sizes of the files of a database directory by their class (docs/OPERATIONS.md §1.2) + */ + private function databaseBytes($dbname){ + $home = $this->layout->dbHome($dbname); + $active = 'generations/' . $this->layout->generationNumber($dbname) . '/'; + $bytes = ['data' => 0, 'indexes' => 0, 'buffer' => 0, 'metadata' => 0, 'recovery' => 0, 'other' => 0]; + $walk = function($dir, $prefix) use (&$walk, &$bytes, $active){ + foreach(@scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..' || is_link($dir . $entry)){ + continue; + } + if(is_dir($dir . $entry)){ + $walk($dir . $entry . '/', $prefix . $entry . '/'); + continue; + } + if(!is_file($dir . $entry)){ + continue; + } + $file = $prefix . $entry; + $inActive = strncmp($file, $active, strlen($active)) === 0; + $relative = $inActive ? substr($file, strlen($active)) : null; + if($inActive && preg_match('#^segments/[^/]+\.jsonl$#', $relative) === 1){ + $class = 'data'; + } elseif($inActive && preg_match('#^(?:segments/[^/]+\.kix|indexes/[^/]+\.(?:fx|gfx)|spatial/[^/]+\.(?:rx|grx))$#', $relative) === 1){ + $class = 'indexes'; + } elseif($inActive && strncmp($relative, 'buffer/', 7) === 0){ + $class = 'buffer'; + } elseif(in_array($inActive ? $relative : $file, $inActive ? ['metadata', 'shards.json', 'indexes.json'] : ['FORMAT', 'CURRENT', 'LOCK'], true)){ + $class = 'metadata'; + } elseif(strncmp($file, 'recovery/', 9) === 0){ + $class = 'recovery'; + } else { + $class = 'other'; + } + $bytes[$class] += (int)@filesize($dir . $entry); + } + }; + $walk($home, ''); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + $bytes['total'] = array_sum($bytes); + return $bytes; + } +} diff --git a/src/Internal/Durability/FileIO.php b/src/Internal/Durability/FileIO.php new file mode 100644 index 0000000..85ccc03 --- /dev/null +++ b/src/Internal/Durability/FileIO.php @@ -0,0 +1,512 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->locks, $this->recovery, $this->scope, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->appendMismatch = &$this->k->appendMismatch; + $this->directoryPermissions = &$this->k->directoryPermissions; + $this->durable = &$this->k->durable; + $this->filePermissions = &$this->k->filePermissions; + $this->exactFloats = &$this->rt->exactFloats; + $this->metrics = &$this->rt->metrics; + $this->faultHook = &$this->rt->faultHook; // @test + $this->flushFn = &$this->rt->flushFn; // @test + $this->writeFn = &$this->rt->writeFn; // @test + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->locks = null; + $this->recovery = null; + $this->scope = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // ATOMIC FILE OPERATIONS + // ========================================== + + /** + * A derived file's header line, its crc last + * @return string|false + */ + public function encodeHeader(array $header){ + $json = $this->encodeJson($header); + return $json === false ? false : \noneDB\Internal\Storage\KeyIndex::sealHeader($json); + } + + /** + * Encode JSON, returning false instead of producing invalid output + * @return string|false + */ + public function encodeJson($data, $prettyPrint = false, $extraFlags = 0){ + $flags = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | $extraFlags; + if($prettyPrint){ + $flags |= JSON_PRETTY_PRINT; + } + // json_encode() writes floats with serialize_precision digits: anything but -1 (the default, + // shortest exact form) would store rounded values (0.30000000000000004 as 0.3 at 10). + // Loops encoding many records run inside withExactFloats(): no ini_get()/finally per record. + if($this->exactFloats || ini_get('serialize_precision') === '-1'){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; + } + } + return $this->withExactFloats(function() use ($data, $flags){ + try { + return json_encode($data, $flags | JSON_THROW_ON_ERROR); + } catch(\JsonException $e){ + return false; + } + }); + } + + /** + * Run $fn with serialize_precision set to -1 (checked once instead of once per encoded value) + */ + public function withExactFloats(callable $fn){ + if($this->exactFloats){ + return $fn(); + } + $precision = ini_get('serialize_precision'); + if($precision !== '-1'){ + ini_set('serialize_precision', '-1'); + } + $this->exactFloats = true; + try { + return $fn(); + } finally { + $this->exactFloats = false; + if($precision !== '-1'){ + ini_set('serialize_precision', (string)$precision); + } + } + } + + /** + * Encode one JSONL record line (without the trailing newline) + * @throws \InvalidArgumentException when the record can't be represented as JSON (invalid UTF-8, NAN, INF, ...) + */ + public function encodeRecord(array $record){ + // Keep floats floats: 2.0 / -0.0 are stored as 2.0 / -0.0, not as the ints 2 / 0 + $json = $this->encodeJson($record, false, JSON_PRESERVE_ZERO_FRACTION); + if($json === false){ + throw new \InvalidArgumentException('noneDB: record cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value): ' . json_last_error_msg()); + } + return $json; + } + + /** + * Write a whole file atomically: temp file in the same directory + rename. + * Readers see either the old or the new content, never a partial file. + * @param string|iterable $content The bytes, or a sequence of chunks (a large derived file written as it is + * made); a chunk that can't be made throws, and the temp file is removed + * @param string|bool $mode What the file is (docs/DURABILITY.md §3): + * 'derived' (false): checkpoints and indexes, rebuilt from the data if lost: never synced + * 'durable': records of the database (FORMAT, CURRENT, metadata, shards.json, indexes.json): synced in + * durable mode: temp file, rename (the commit point), directory + * 'replace' (true): a data file replaced (compaction, the switch to shards): synced in both modes; a failed + * directory sync after the rename only counts in durable mode + * @return bool False when the file could not be written (nothing changed) + * @throws \RuntimeException DURABILITY_FAILED (a sync before the rename failed, nothing changed), + * OUTCOME_UNKNOWN (the directory sync after the rename failed) + */ + public function writeFileAtomic($path, $content, $mode = 'derived', $note = true){ + $started = microtime(true); + $mode = $mode === true ? 'replace' : ($mode === false ? 'derived' : $mode); + if($mode === 'replace'){ + $this->scope->refuseUncommittedFile($path, 'replacement'); + } + if($note){ + $this->locks->noteWrite($path); + } + $required = $this->durable && $mode !== 'derived'; + $sync = $required || $mode === 'replace'; + $dir = dirname($path); + if(!is_dir($dir) && !$this->makeDirectory($dir)){ + return false; + } + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + if(!$sync && is_string($content)){ // @prod + if(!$sync && is_string($content) && $this->writeFn === null && $this->flushFn === null){ // @test + $written = @file_put_contents($tmp, $content); + if($written === false || $written !== strlen($content)){ + @unlink($tmp); + return false; + } + $this->setPermissions($tmp); + } else { + $fp = @fopen($tmp, 'xb'); + if($fp === false){ + return false; + } + try { + $ok = true; + foreach(is_string($content) ? [$content] : $content as $chunk){ + if(!$this->writeAll($fp, $chunk, $path)){ + $ok = false; + break; + } + } + } catch(\Throwable $e){ + fclose($fp); + @unlink($tmp); + throw $e; + } + $ok = $ok && fflush($fp); // @prod + $ok = $ok && ($this->flushFn !== null ? ($this->flushFn)($fp, $path) : fflush($fp)); // @test + $synced = $ok && $sync ? $this->sync->syncHandle($fp, $path) : null; + fclose($fp); + if(!$ok){ + @unlink($tmp); + return false; + } + if($required){ + $this->sync->noteRequired('file', $synced === true); + } + if($sync && $synced !== true){ + @unlink($tmp); + throw $this->sync->failure('file', $path, false); + } + $this->setPermissions($tmp); + } + if($this->faultHook !== null){ $this->recovery->fault('atomic:renaming', ['path' => $path, 'mode' => $mode]); } // @test + if(!$this->sync->renameWithRetry($tmp, $path)){ + @unlink($tmp); + return false; + } + clearstatcache(true, $path); + if($sync){ + $dirSynced = $this->sync->syncDirectory($dir) === true; + if($required){ + $this->sync->noteRequired('dir', $dirSynced); + if(!$dirSynced){ + throw $this->sync->failure('dir', $dir, true); + } + } + } + if($mode === 'derived'){ + // A checkpoint or an index file (getMetrics()) + $this->metrics['checkpoints']++; + $this->metrics['checkpointSeconds'] += microtime(true) - $started; + } + return true; + } + + /** + * Write a file derived from the data (a checkpoint) atomically without moving the generation: its + * content is the same whoever writes it, so a reader may write it under the shared lock + */ + public function writeDerivedFile($path, $content){ + return $this->writeFileAtomic($path, $content, 'derived', false); + } + + /** + * fwrite() every byte (a short write is continued; no progress is a failure) + */ + private function writeAll($fp, $bytes, $path){ + for($done = 0, $length = strlen($bytes); $done < $length; $done += $n){ + $chunk = $done === 0 ? $bytes : substr($bytes, $done); + $n = fwrite($fp, $chunk); // @prod + $n = $this->writeFn !== null ? ($this->writeFn)($fp, $chunk, $path) : fwrite($fp, $chunk); // @test + if($n === false || $n <= 0){ + return false; + } + } + return true; + } + + /** + * Create a directory and its missing parents. In durable mode each new directory's parent is synced, + * so the new entries survive a power loss with what is written into them. + * @throws \RuntimeException DURABILITY_FAILED + */ + public function makeDirectory($dir){ + $dir = rtrim($dir, '/'); + $missing = []; + for($at = $dir; $at !== '' && $at !== '.' && $at !== '/' && !is_dir($at); $at = dirname($at)){ + $missing[] = $at; + } + foreach(array_reverse($missing) as $new){ + if(!@mkdir($new, $this->directoryPermissions) && !is_dir($new)){ + return false; + } + $this->setPermissions($new, true); + $this->sync->requireSync(null, dirname($new), 'dir'); + } + return true; + } + + /** + * A file or directory noneDB created in dbDir gets the configured permissions, whatever the umask + * (STORAGE_FORMAT.md §3) + */ + public function setPermissions($path, $directory = false){ + @chmod($path, $directory ? $this->directoryPermissions : $this->filePermissions); + } + + // File system writes: every write to a database file goes through writeFileAtomic(), + // appendToFile() or the helpers below. Each calls noteWrite() first: the generation moves before the + // file changes, so other processes drop their caches even if this one dies half-way, and a write + // without the database's exclusive lock is a LogicException. SourceWriteGuardTest fails on a raw + // write call anywhere else. + + public function fsUnlink($path){ + clearstatcache(true, $path); + if(!file_exists($path) && !is_link($path)){ + return false; // nothing to remove: no write + } + $this->scope->refuseUncommittedFile($path, 'removal'); + $this->locks->noteWrite($path); + $ok = @unlink($path); + clearstatcache(true, $path); + return $ok; + } + + public function fsRename($from, $to){ + $this->scope->refuseUncommittedFile($from, 'rename'); + $this->scope->refuseUncommittedFile($to, 'replacement'); + $this->locks->noteWrite($from); + $this->locks->noteWrite($to); + $ok = $this->sync->renameWithRetry($from, $to); + clearstatcache(true, $from); + clearstatcache(true, $to); + return $ok; + } + + public function fsTouch($path){ + $this->locks->noteWrite($path); + $created = !file_exists($path); + $ok = @touch($path); + if($ok && $created){ + $this->setPermissions($path); + } + clearstatcache(true, $path); + return $ok; + } + + /** Create a directory of a database (and its parents; synced in durable mode) */ + public function fsMkdir($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(is_dir($path)){ + return true; + } + $this->locks->noteWrite($path . '/'); + return $this->makeDirectory($path); + } + + /** Remove an empty directory of a database */ + public function fsRmdir($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(!is_dir($path)){ + return false; + } + $this->locks->noteWrite($path . '/'); + return @rmdir($path); + } + + /** Cut a file to $size bytes */ + public function fsTruncate($path, $size){ + $this->locks->noteWrite($path); + $fp = @fopen($path, 'r+b'); + if($fp === false){ + return false; + } + $ok = ftruncate($fp, $size); + fclose($fp); + clearstatcache(true, $path); + return $ok; + } + + /** + * Append bytes to a file. Returns the offset the bytes start at, or false (nothing written). + * The append commits when its bytes are written and flushed and, in durable mode, the file is synced + * (and its directory when the append created it: an empty file before). A failure before that cuts the + * bytes back, so the file never ends in a torn line (docs/DURABILITY.md §3, §4). + * @param string|null $dbname Database whose rollback scope records the append (data and buffer files) + * @param int|null $expected Size the caller's index says the file has: anything else writes nothing + * and sets $this->appendMismatch + * @return int|false + * @throws \RuntimeException DURABILITY_FAILED (a required sync failed, the bytes were cut back), + * OUTCOME_UNKNOWN (the bytes could not be cut back) + */ + public function appendToFile($path, $bytes, $dbname = null, $expected = null, $segment = null){ + $this->locks->noteWrite($path); + $this->appendMismatch = false; + $fp = @fopen($path, 'c+b'); + if($fp === false){ + // The directory of a file's first append (buffer/) is created then: no check on every append + $dir = dirname($path); + if(is_dir($dir) || !$this->makeDirectory($dir) || ($fp = @fopen($path, 'c+b')) === false){ + return false; + } + } + try { + if(fseek($fp, 0, SEEK_END) !== 0){ + return false; + } + $offset = ftell($fp); + if($offset === false){ + return false; + } + if($offset === 0){ + $this->setPermissions($path); // created by this append (or empty) + } + if($expected !== null && $offset !== (int)$expected){ + // The file changed behind the cached index. An unterminated tail only (an append that + // never completed): quarantined and cut, then this append goes where the index expects it. + // Anything else: nothing is written, the caller drops its index (rebuilt on next access). + if($dbname === null || $offset < (int)$expected || $this->checkpoints->lastNewlineIn($path, (int)$expected, $offset) >= 0){ + $this->appendMismatch = true; + return false; + } + $this->recovery->quarantineAndCut($dbname, $segment, $path, (int)$expected, $offset); + fseek($fp, 0, SEEK_END); + $offset = ftell($fp); + if($offset !== (int)$expected){ + $this->appendMismatch = true; + return false; + } + } + if($dbname !== null){ + $this->scope->markScopeFile($dbname, $path, $offset, $fp); + } + // Written bytes the stream couldn't hand to the OS (disk full, I/O error): the append failed + if(!$this->writeAll($fp, $bytes, $path) || !fflush($fp)){ // @prod + if(!$this->writeAll($fp, $bytes, $path) || !($this->flushFn !== null ? ($this->flushFn)($fp, $path) : fflush($fp))){ // @test + $this->undoAppend($fp, $offset, $path, null); + return false; + } + if($dbname !== null){ + if(!$this->sync->trySync($fp, $path)){ + $this->undoAppend($fp, $offset, $path, $this->sync->failure('file', $path, false)); + } + if($offset === 0 && !$this->sync->trySync(null, dirname($path), 'dir')){ + $this->undoAppend($fp, $offset, $path, $this->sync->failure('dir', dirname($path), false)); + } + if($this->faultHook !== null){ $this->recovery->fault('append:after', ['db' => $dbname, 'path' => $path]); } // @test + } + return $offset; + } finally { + fclose($fp); + clearstatcache(true, $path); + } + } + + /** + * Cut a failed append back (synced in durable mode), then throw $cause if there is one. A cut that + * fails leaves the bytes possibly there: OUTCOME_UNKNOWN, so the write's rollback scope tries again + * and the next session recovers; never a plain failure that would leave them to be read as a record. + * @throws \RuntimeException + */ + private function undoAppend($fp, $offset, $path, ?\RuntimeException $cause){ + try { + if($this->faultHook !== null){ $this->recovery->fault('append:undo', ['path' => $path]); } // @test + $ok = ftruncate($fp, $offset) && $this->sync->syncUndo($fp, $path); + } catch(\Throwable $e){ + $ok = false; + } + if(!$ok){ + throw new \RuntimeException("noneDB: an append to " . basename($path) . " failed and could not be cut back: the outcome is unknown, read the state again", \noneDB::OUTCOME_UNKNOWN, $cause); + } + if($cause !== null){ + throw $cause; + } + } + + /** + * Read and decode a JSON file + * + * @param string $path File path + * @param mixed $default Value returned when the file doesn't exist or is empty + * @return mixed Decoded data, $default, or null when the file is corrupt + */ + public function atomicRead($path, $default = null){ + clearstatcache(true, $path); + if(!is_file($path)){ + return $default; + } + $content = @file_get_contents($path); + if($content === false || $content === ''){ + return $default; + } + $data = json_decode($content, true); + if($data === null && json_last_error() !== JSON_ERROR_NONE){ + return null; // corrupted + } + return $data !== null ? $data : $default; + } + + /** + * Encode and write a JSON file atomically. Nothing is touched when encoding fails. + * + * @param string $path File path + * @param mixed $data Data to write (will be JSON encoded) + * @param bool $prettyPrint Use JSON_PRETTY_PRINT + * @return bool Success + */ + public function atomicWrite($path, $data, $prettyPrint = false, $mode = 'derived'){ + $json = $this->encodeJson($data, $prettyPrint); + if($json === false){ + return false; + } + return $this->writeFileAtomic($path, $json, $mode); + } +} diff --git a/src/Internal/Durability/Sync.php b/src/Internal/Durability/Sync.php new file mode 100644 index 0000000..ddf5210 --- /dev/null +++ b/src/Internal/Durability/Sync.php @@ -0,0 +1,239 @@ +k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->durable = &$this->k->durable; + $this->lockTimeout = &$this->k->lockTimeout; + $this->lastOperation = &$this->rt->lastOperation; + $this->lastSyncError = &$this->rt->lastSyncError; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->renameFn = &$this->rt->renameFn; // @test + $this->syncFn = &$this->rt->syncFn; // @test + $this->syncStats = &$this->rt->syncStats; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + + /** + * fsync() a file handle where the stream supports it (plain files). Never emits a warning: the result + * goes to the durability report. + * @return bool|null Null when it can't be done here + */ + public function syncHandle($fp, $path = null){ + $meta = @stream_get_meta_data($fp); + if(!is_array($meta) || ($meta['wrapper_type'] ?? '') !== 'plainfile'){ + return null; + } + $ok = @fsync($fp); // @prod + $ok = $this->syncFn !== null ? (bool)($this->syncFn)($fp, 'file', $path) : @fsync($fp); // @test + $this->countSync('file', $ok, $path); + return $ok; + } + + /** + * fsync() a directory so a new, renamed or deleted entry in it survives a power loss (not possible on + * Windows) + * @return bool|null Null when it can't be done here + */ + public function syncDirectory($dir){ + if(PHP_OS_FAMILY === 'Windows'){ + return null; + } + $dir = rtrim($dir, '/\\'); + $fp = @fopen($dir . '/', 'r'); + if($fp === false){ + $this->countSync('dir', false, $dir); + return false; + } + try { + $ok = @fsync($fp); // @prod + $ok = $this->syncFn !== null ? (bool)($this->syncFn)($fp, 'dir', $dir) : @fsync($fp); // @test + $this->countSync('dir', $ok, $dir); + return $ok; + } finally { + fclose($fp); + } + } + + private function countSync($kind, $ok, $path){ + $this->syncStats[$kind]++; + if(!$ok){ + $this->syncStats['failed']++; + $this->lastSyncError = ['path' => $path === null ? null : $this->relative((string)$path), 'op' => $kind, 'time' => time()]; + } + } + + /** + * A path relative to dbDir (the report and messages never show where dbDir is) + */ + private function relative($path){ + $dir = (string)$this->dbDir; + if($path === rtrim($dir, '/')){ + return '.'; + } + return strncmp($path, $dir, strlen($dir)) === 0 ? substr($path, strlen($dir)) : basename($path); + } + + /** + * A sync durable mode requires (DURABILITY.md §3), done in durable mode only. A failure throws: + * DURABILITY_FAILED before the write's commit point, OUTCOME_UNKNOWN after it. + * @throws \RuntimeException + */ + public function requireSync($fp, $path, $kind = 'file', $afterCommit = false){ + if($this->durable && !$this->trySync($fp, $path, $kind)){ + throw $this->failure($kind, $path, $afterCommit); + } + } + + /** + * A sync durable mode requires, for a caller that undoes its change before it throws. Performance mode: + * true, nothing synced. + */ + public function trySync($fp, $path, $kind = 'file'): bool { + if(!$this->durable){ + return true; + } + $ok = ($kind === 'dir' ? $this->syncDirectory($path) : $this->syncHandle($fp, $path)) === true; + $this->noteRequired($kind, $ok); + return $ok; + } + + /** + * Record a sync durable mode required for the running write operation and whether it succeeded + * (getDurability()['lastOperation']) + */ + public function noteRequired($kind, bool $ok){ + foreach($this->lockRegistry as $lock => $entry){ + if($entry['mode'] === LOCK_EX && isset($entry['syncs'])){ + $this->lockRegistry[$lock]['syncs'][$kind] = ($entry['syncs'][$kind] ?? true) && $ok; + } + } + } + + /** + * Sync the undo of a change in durable mode (a cut back append): true when it is durable or nothing + * needs to be (performance mode) + */ + public function syncUndo($fp, $path): bool { + return !$this->durable || $this->syncHandle($fp, $path) === true; + } + + /** + * The exception of a required sync that failed + */ + public function failure($kind, $path, $afterCommit, ?\Throwable $previous = null): \RuntimeException { + $what = ($kind === 'dir' ? 'the directory ' : '') . $this->relative((string)$path); + if($afterCommit){ + return new \RuntimeException("noneDB: fsync of {$what} failed after the commit point: the outcome is unknown, read the state again (docs/DURABILITY.md §2)", \noneDB::OUTCOME_UNKNOWN, $previous); + } + return new \RuntimeException("noneDB: fsync of {$what} failed; the write was rolled back (docs/DURABILITY.md §2)", \noneDB::DURABILITY_FAILED, $previous); + } + + /** + * rename() that retries where it can fail for a moment: on Windows while another process has the target + * open (5, 10, 20, 40, 80, 160 ms, within lockTimeout). The target is never deleted first. + */ + public function renameWithRetry($from, $to){ + $fn = $this->renameFn; // @test + $retry = PHP_OS_FAMILY === 'Windows'; // @prod + $retry = PHP_OS_FAMILY === 'Windows' || $fn !== null; // @test + $started = microtime(true); + foreach([0, 5, 10, 20, 40, 80, 160] as $i => $delay){ + if($delay > 0){ + if(!$retry || ($this->lockTimeout > 0 && microtime(true) - $started + $delay / 1000 > $this->lockTimeout)){ + return false; + } + usleep($delay * 1000); + clearstatcache(true, $from); + if(!file_exists($from)){ + return false; + } + } + if(@rename($from, $to)){ // @prod + if($fn !== null ? $fn($from, $to) : @rename($from, $to)){ // @test + return true; + } + } + return false; + } + + /** + * Record how a write operation ended (getDurability()['lastOperation']) + * @param array $syncs kind => true (every required sync of that kind succeeded) | false + */ + public function finishOperation($op, $dbname, array $syncs, $outcome, $reason = null){ + $required = []; + $completed = []; + if($syncs){ // durable mode only: every write of performance mode ends here with none + foreach(['file', 'dir'] as $kind){ + if(isset($syncs[$kind])){ + $required[] = $kind; + if($syncs[$kind]){ + $completed[] = $kind; + } + } + } + } + $this->lastOperation = ['op' => $op, 'db' => (string)$dbname, 'outcome' => $outcome, + 'required' => $required, 'completed' => $completed, 'reason' => $reason]; + } + + /** + * Durability of this instance and of the last write operation of the process (docs/DURABILITY.md §6) + * @return array + */ + public function getDurability(): array { + $os = \noneDB\Internal\Config\Configuration::osFamily(); + $note = \noneDB\Internal\Config\Configuration::platformNote($os); + return [ + 'mode' => $this->durable ? 'durable' : 'performance', + 'platform' => ['os' => $os, 'durableSupported' => \noneDB\Internal\Config\Configuration::durableSupported(), + 'notes' => $note === '' ? [] : ['durable mode is not supported here: ' . $note]], + 'lastOperation' => $this->lastOperation, + 'lastSyncError' => $this->lastSyncError, + 'syncs' => $this->syncStats, + ]; + } +} diff --git a/src/Internal/Index/FieldIndex.php b/src/Internal/Index/FieldIndex.php new file mode 100644 index 0000000..2f2fb00 --- /dev/null +++ b/src/Internal/Index/FieldIndex.php @@ -0,0 +1,1961 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->records, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->dataSizes = &$this->k->dataSizes; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->globalSkip = &$this->k->globalSkip; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->segWrites = &$this->k->segWrites; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->indexStats = &$this->rt->indexStats; // @test + $this->indexTokens = &$this->rt->indexTokens; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->secondaryChecked = &$this->rt->secondaryChecked; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // FIELD INDEXES + // ========================================== + // Which fields are indexed is recorded in indexes.json of the active generation + // ({"kind":"indexes","v":1,"fields","spatial","spatialOptions"}). A segment's field index + // indexes/{seg}.{token}.fx (paged, docs/STORAGE_FORMAT.md §6.5.1) maps the exact identity of a value + // (valueKey(): "n", "t", "f", "i42", "d", "sAnkara") to the keys that hold it; in memory: + // ['b' => ['n' => [2, 4], 'f' => 10, 'i-3' => [1, 9], 'i42' => 7, 'sAnkara' => 3, 'sIstanbul' => [0, 5, 9]], 's', 'e', ...] + // Buckets are exact (1, 1.0, "1" and true are different values, as for ===) and don't depend on the + // precision, serialize_precision or locale settings. Values that are arrays are not indexed. The + // global field index indexes/{token}.gfx of a sharded database holds a filter of each shard's values as + // of its covered bytes (§6.5.2); a shard's newer lines are checked directly. + + public function getIndexManifestPath($dbname){ + return $this->layout->genPath($dbname, 'indexes.json'); + } + + /** + * Indexed fields of a database: ['fields', 'spatial', 'spatialOptions']. No file: no indexes; a file + * that doesn't decode (zero-length after a power loss) is derived from the index files' headers. + */ + public function getIndexManifest($dbname){ + $path = $this->getIndexManifestPath($dbname); + if(isset($this->fieldIndexCache[$path])){ + $this->cacheTouch[$path] = ++$this->cacheTick; + return $this->fieldIndexCache[$path]; + } + clearstatcache(true, $path); + $content = is_file($path) ? @file_get_contents($path) : false; + $manifest = $content === false ? [] : json_decode($content, true); + if(!is_array($manifest) || ($content !== false && ($manifest['kind'] ?? null) !== 'indexes')){ + $manifest = $this->deriveIndexManifest($dbname); + } + $manifest = [ + 'fields' => array_values(array_unique(array_map('strval', $manifest['fields'] ?? []))), + 'spatial' => array_values(array_unique(array_map('strval', $manifest['spatial'] ?? []))), + 'spatialOptions' => is_array($manifest['spatialOptions'] ?? null) ? $manifest['spatialOptions'] : [], + ]; + $this->fieldIndexCache[$path] = $manifest; + $this->cacheGrew = true; + return $manifest; + } + + public function saveIndexManifest($dbname, array $manifest){ + $file = [ + 'kind' => 'indexes', + 'v' => 1, + 'fields' => array_values(array_unique($manifest['fields'] ?? [])), + 'spatial' => array_values(array_unique($manifest['spatial'] ?? [])), + ]; + $file['spatialOptions'] = array_intersect_key($manifest['spatialOptions'] ?? [], array_flip($file['spatial'])); + $path = $this->getIndexManifestPath($dbname); + if(empty($file['fields']) && empty($file['spatial'])){ + $this->io->fsUnlink($path); + } elseif(!$this->io->atomicWrite($path, $file, false, 'durable')){ + throw new \RuntimeException("noneDB: failed to write index list of database '{$dbname}'"); + } + $this->fieldIndexCache[$path] = ['fields' => $file['fields'], 'spatial' => $file['spatial'], + 'spatialOptions' => $file['spatialOptions']]; + } + + /** + * An index list that doesn't decode: the fields named in the headers of the generation's .fx/.rx files + */ + public function deriveIndexManifest($dbname){ + $fields = []; + $spatial = []; + $options = []; + $gen = $this->layout->genDir($dbname); + foreach(array_merge($this->segments->listFilesWithPrefix($gen . 'indexes/', '/\.fx$/'), $this->segments->listFilesWithPrefix($gen . 'spatial/', '/\.rx$/')) as $file){ + $header = $this->secondary->readSecondaryHeader($file); + if(!is_array($header) || !isset($header['field']) || !is_string($header['field'])){ + continue; + } + if(substr($file, -3) === '.fx'){ + $fields[] = $header['field']; + } else { + $spatial[] = $header['field']; + if(isset($header['ns'])){ + $options[$header['field']] = ['nodeSize' => (int)$header['ns']]; + } + } + } + return ['fields' => array_values(array_unique($fields)), 'spatial' => array_values(array_unique($spatial)), 'spatialOptions' => $options]; + } + + /** + * Path of a segment's field index + */ + public function getFieldIndexPath($dbname, $field, $shardId = null){ + return $this->secondary->secondaryPath($dbname, 'fx', $field, $shardId); + } + + /** + * True if a value can be stored in a field index + */ + public function isIndexableValue($value){ + return is_scalar($value) || is_null($value); + } + + /** + * An empty field index. Besides its buckets 'b' (covering the data bytes [0, s)): 'file', the write token of + * the .fx file whose body is exactly these buckets (null: none); 'idsAt', the data bytes since which the set + * of bucket ids is this one (null: unknown) + */ + public function newFieldIndex($field){ + return ['kind' => 'fx', 'field' => (string)$field, 's' => 0, 'ck' => -1, 'e' => null, 'b' => [], 'mk' => -1, + 'rev' => null, 'views' => null, 'bytes' => 0, 'file' => null, 'idsAt' => null]; + } + + /** + * Build a segment's field index from its data (covers the key index's bytes) + * @return array + */ + public function buildSegmentFieldIndex($dbname, $shardId, $field){ + $key = $this->secondary->keyIndexWithEpoch($dbname, $shardId); + $index = $this->newFieldIndex($field); + // fieldIndexApply() inlined: every key occurs once, so a value is only ever added. A value of one key costs + // no array: when every value differs, those would be most of the memory + $field = (string)$field; + $buckets = []; + $lists = []; + foreach($this->records->eachSegmentBatch($dbname, $shardId) as $batch){ + foreach($batch as $k => $record){ + $k = (int)$k; + if($k > $index['mk']){ + $index['mk'] = $k; + } + if(!isset($record[$field]) && !array_key_exists($field, $record)){ + continue; + } + $value = $record[$field]; + if(is_string($value)){ + $bucket = 's' . $value; + } elseif(is_scalar($value) || $value === null){ + $bucket = \noneDB\Internal\Query\Values::valueKey($value); + } else { + continue; + } + if(isset($lists[$bucket])){ + $lists[$bucket][] = $k; + } elseif(isset($buckets[$bucket])){ + $lists[$bucket] = [$buckets[$bucket], $k]; + } else { + $buckets[$bucket] = $k; + } + } + } + foreach($lists as $bucket => $keys){ + $buckets[$bucket] = $keys; + } + $index['b'] = $buckets; + $index['mk'] = $key === null ? -1 : max($index['mk'], \noneDB\Internal\Storage\KeyIndex::maxKey($key)); + $index['s'] = $key === null ? 0 : (int)$key['s']; + $index['e'] = $key === null ? null : $key['e']; + $index['rev'] = null; + return $index; + } + + /** + * Apply one data line to a field index: the key now holds $record (null: deleted) + */ + private function fieldIndexApply(array &$index, $key, $record){ + $field = $index['field']; + $new = $record !== null && array_key_exists($field, $record) && $this->isIndexableValue($record[$field]) + ? \noneDB\Internal\Query\Values::valueKey($record[$field]) : null; + if($key > $index['mk']){ + // A key never seen: nothing to remove + $index['mk'] = $key; + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + if($index['rev'] !== null){ + $index['rev'][$key] = $new; + } + } + return; + } + if($index['rev'] === null){ + $rev = []; + foreach($index['b'] as $bucket => $keys){ + foreach((array)$keys as $k){ + $rev[$k] = (string)$bucket; + } + } + $index['rev'] = $rev; + } + $old = $index['rev'][$key] ?? null; + if($old === $new){ + return; // the value didn't change: nothing to do + } + if($old !== null){ + $index['file'] = null; + $keys = $index['b'][$old]; + if(is_int($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + $index['idsAt'] = null; + } else { + $at = array_search($key, $keys, true); + if($at !== false){ + unset($keys[$at]); + } + $keys = array_values($keys); + if(count($keys) === 1){ + $index['b'][$old] = $keys[0]; + } elseif(empty($keys)){ + unset($index['b'][$old]); + $index['views'] = null; + $index['idsAt'] = null; + } else { + $index['b'][$old] = $keys; + } + } + unset($index['rev'][$key]); + } + if($new !== null){ + $this->fieldIndexAdd($index, $new, $key); + $index['rev'][$key] = $new; + } + } + + private function fieldIndexAdd(array &$index, $bucket, $key){ + $index['file'] = null; + if(!isset($index['b'][$bucket])){ + $index['b'][$bucket] = $key; + $index['views'] = null; + $index['idsAt'] = null; + } elseif(is_int($index['b'][$bucket])){ + $index['b'][$bucket] = [$index['b'][$bucket], $key]; + } else { + $index['b'][$bucket][] = $key; + } + } + + /** + * A segment's field index, current (the data lines after the bytes it covers are replayed). Null when + * there is none that fits the segment (missing, of another epoch, damaged): see ensureSegmentFieldIndex. + * @return array|null + */ + private function readFieldIndex($dbname, $field, $shardId = null, $path = null, $key = null){ + // The segment's key index (currentKeyIndex() inlined: this is the hot path of every lookup) + if($key === null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $key = isset($this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]) ? ($this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)] ?? null) : null; + if($key !== null){ + $this->metrics['hits']++; + } elseif(($key = $this->checkpoints->readJsonlIndex($dbname, $shardId)) === null){ + return null; + } + } + $path = $path ?? $this->getFieldIndexPath($dbname, $field, $shardId); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->fieldIndexCache[$path])){ + $index = $this->fieldIndexCache[$path]; + if($index['s'] === (int)$key['s'] && $index['e'] === $key['e'] && $index['e'] !== null){ + $this->metrics['hits']++; + if(isset($index['due']) && $this->noWrites === 0 && $this->multiLock === 0){ + unset($index['due'], $this->fieldIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + return $this->fieldIndexCache[$path]; + } + return $index; // the usual case: nothing written since + } + if($index['e'] !== null && $index['e'] === $key['e'] && $index['s'] <= (int)$key['s']){ + $this->metrics['hits']++; + if($index['s'] < (int)$key['s']){ + unset($this->fieldIndexCache[$path], $index['due']); // sole owner while it changes + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + } + return $this->fieldIndexCache[$path]; + } + unset($this->fieldIndexCache[$path]); + } + $this->metrics['misses']++; + $file = $this->secondary->readSecondaryFile($path); + if($file === null || ($file[0]['v'] ?? null) !== 1 || !$this->secondary->secondaryFits($dbname, $shardId, $file[0], 'fx', $field)){ + return null; + } + $buckets = $this->decodeFieldIndexBody($file[0], $file[1]); + if($buckets === null){ + return null; + } + unset($this->fieldIndexCache[$path . '#pg']); // the whole index replaces its pages + $index = $this->newFieldIndex($field); + $index['b'] = $buckets; + $index['s'] = $file[0]['s']; + $index['ck'] = $file[0]['s']; + $index['e'] = $file[0]['e']; + $index['mk'] = (int)($file[0]['mk'] ?? -1); + $index['bytes'] = strlen($file[1]); + $index['file'] = $file[0]['w']; + $index['idsAt'] = $file[0]['s']; + $this->replayFieldIndex($dbname, $shardId, $index, (int)$key['s']); + $this->fieldIndexCache[$path] = $index; + $this->cacheGrew = true; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'fx', $field, $index)){ + $this->fieldIndexCache[$path]['due'] = true; + } + return $this->fieldIndexCache[$path]; + } + + public function replayFieldIndex($dbname, $shardId, array &$index, $to){ + foreach($this->secondary->segmentTail($dbname, $shardId, $index['s'], $to) as [$k, $record]){ + $this->fieldIndexApply($index, $k, $record); + } + $index['s'] = $to; + } + + /** + * Write a field index checkpoint (exclusive lock or a reader: the content is the data's) + */ + public function writeFieldIndex($dbname, $field, array &$index, $shardId = null){ + if($index['e'] === null){ + return false; + } + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$index['s']); + if($crc === null){ + return false; + } + // Buckets that are still those of the file: its body (canonical, checked when it was read) is theirs + $old = $index['file'] === null ? null : $this->secondary->readSecondaryFile($path); + if($old !== null && ($old[0]['w'] ?? null) === $index['file']){ + $body = $old[1]; + [$pg, $c, $n] = [$old[0]['pg'], $old[0]['c'], $old[0]['n']]; + } else { + $this->indexStats['bodiesBuilt']++; // @test + self::canonicalBuckets($index['b']); + $c = count($index['b']); + $n = 0; + foreach($index['b'] as $keys){ + $n += is_int($keys) ? 1 : count($keys); + } + [$body, $pg] = PagedFile::build($index['b']); + } + $header = ['kind' => 'fx', 'v' => 1, 'field' => (string)$field, 'w' => bin2hex(random_bytes(8)), 's' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc, + 'mk' => (int)$index['mk'], 'c' => $c, 'n' => $n, 'created' => time(), 'pg' => $pg]; + $this->secondary->writeSecondaryFile($dbname, $path, $header, $body); + unset($this->fieldIndexCache[$path . '#pg']); + $index['file'] = $header['w']; + $index['idsAt'] = $index['idsAt'] ?? (int)$index['s']; + $index['ck'] = (int)$index['s']; + $index['bytes'] = strlen($body); + $index['views'] = null; + $this->fieldIndexCache[$path] = $index; + $this->cacheGrew = true; + return true; + } + + /** + * Buckets in the fixed order of an index file, so the file is the same whoever writes it: bucket ids + * in byte order (ksort, in C), keys of a bucket ascending. Range queries sort their own views + * (fieldIndexViews), so no order by value is needed here. + */ + public static function canonicalBuckets(array &$buckets){ + ksort($buckets, SORT_STRING); + $lists = []; + foreach($buckets as $bucket => $keys){ + if(is_array($keys)){ + $lists[] = $bucket; + } + } + foreach($lists as $bucket){ + sort($buckets[$bucket]); + } + } + + /** + * The number of an "i…" or "d…" bucket + */ + private static function bucketNumber($bucket){ + if($bucket[0] === 'i'){ + return (int)substr($bucket, 1); + } + $unpacked = unpack('E', (string)hex2bin(substr($bucket, 1))); + return $unpacked === false ? 0.0 : $unpacked[1]; + } + + /** + * Delete a field's index file of a segment + */ + public function deleteFieldIndexFile($dbname, $field, $shardId = null){ + $ok = true; + foreach([$this->getFieldIndexPath($dbname, $field, $shardId)] as $path){ + unset($this->fieldIndexCache[$path]); + $this->cache->markFileNotExists($path); + clearstatcache(true, $path); + if(file_exists($path) && !$this->io->fsUnlink($path)){ + $ok = false; + } + } + return $ok; + } + + /** + * Get list of indexed fields for a database + * @return array + */ + public function getIndexedFields($dbname){ + return $this->getIndexManifest($dbname)['fields']; + } + + /** + * Check if a field has an index + * @return bool + */ + public function isFieldIndexed($dbname, $field){ + return in_array((string)$field, $this->getIndexedFields($dbname), true); + } + + /** + * Invalidate field index cache for a database + */ + public function invalidateFieldIndexCache($dbname){ + $prefix = $this->layout->dbHome($dbname); + foreach(array_keys($this->fieldIndexCache) as $key){ + if(strpos($key, $prefix) === 0){ + unset($this->fieldIndexCache[$key]); + } + } + } + + /** + * A segment's field index: the current one, or built from the data (exclusive lock). Null when none + * can be used (the caller scans). + * @return array|null + */ + public function ensureSegmentFieldIndex($dbname, $shardId, $field, $path = null, $key = null){ + $index = $this->readFieldIndex($dbname, $field, $shardId, $path, $key); + if($index !== null){ + return $index; + } + if($this->noWrites > 0 || $this->multiLock > 0){ + return null; // checkIntegrity(), explain(), a snapshot: nothing is built, the segment is scanned + } + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + return $index; + }, 'buildFieldIndex'); + } + + /** + * Keys of a field index with exactly this value + * @return array + */ + public function fieldIndexKeys(array $index, $value){ + $keys = $index['b'][is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value)] ?? []; + return is_int($keys) ? [$keys] : $keys; + } + + /** + * Keys of a segment with this value through a point lookup in its paged file (§6.5.1): the pages on the + * path to the value and the data lines after the bytes the file covers, without loading the index. + * Null when the index is loaded, its tail is long (loading it writes a new checkpoint), or there is no + * file that fits. + * @return array|null + */ + private function pointKeys($dbname, $shardId, $field, $value, $path, $key = null){ + $key = $key ?? $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($key === null){ + return null; + } + $pk = $path . '#pg'; + $paged = $this->fieldIndexCache[$pk] ?? null; + if($paged !== null && $key['e'] !== null && ($paged['fit'] ?? null) === $key['e']){ + $this->cacheTouch[$pk] = ++$this->cacheTick; // kept, and checked against this epoch: openPagedIndex() in short + $this->metrics['hits']++; + } elseif(($paged = $this->openPagedIndex($dbname, $shardId, $path, $field, $key)) === null){ + return null; + } + // A long tail is replayed by loading the index, which writes a new checkpoint of it (§6.5): point + // lookups don't leave the same lines to every new process. An index in repeated use is loaded too + // (decided once per operation). + $end = (int)$key['s']; + if($end < $paged['h']['s'] + || ($end !== $paged['h']['s'] && $this->secondary->secondaryTailIsLong($paged['h']['s'], $end, $paged['size']))){ + return null; + } + if($paged['ses'] !== $this->sessionCounter){ + // One more operation on the pages, counted in place (the entry isn't copied); loadWholeNow() only + // once the operations may have paid for the load + $entries = (int)($paged['h']['c'] ?? intdiv($paged['size'], 16)); + $paged = null; + $this->fieldIndexCache[$pk]['ses'] = $this->sessionCounter; + $uses = ++$this->fieldIndexCache[$pk]['uses']; + if($uses >= 2 && $uses * self::ENTRIES_PER_LOOKUP >= $entries && $this->loadWholeNow($pk)){ + unset($this->fieldIndexCache[$pk]); + return null; + } + } + $bucket = is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value); + $found = $this->pagedLookup($path, $bucket); + if($found === false){ + // Another reader replaced the file since its header was read here: once more from the new one + $paged = $this->openPagedIndex($dbname, $shardId, $path, $field, $key); + $found = $paged === null ? null : $this->pagedLookup($path, $bucket); + } + if(!is_array($found)){ + unset($this->fieldIndexCache[$pk]); // a damaged page: the whole file is read (and rebuilt) + return null; + } + $this->indexStats['pointLookups']++; // @test + $covered = $this->fieldIndexCache[$pk]['h']['s']; // the file the pages came from (a new one after a retry) + if($end < $covered){ + return null; + } + if($end === $covered){ + return $found; + } + // The lines after the file, worked out once per end of the data: the newest bucket of each key in + // them (null: tombstone or no indexable value). A key in them leaves the file's answer; the keys + // whose newest line holds this value join it. + $tail = $this->fieldIndexCache[$pk]['tail'] ?? null; + if($tail === null || $tail['to'] !== $end){ + $last = []; + foreach($this->secondary->segmentTail($dbname, $shardId, $covered, $end) as [$k, $record]){ + $last[$k] = $record !== null && array_key_exists($field, $record) && $this->isIndexableValue($record[$field]) + ? \noneDB\Internal\Query\Values::valueKey($record[$field]) : null; + } + $by = []; + foreach($last as $k => $b){ + if($b !== null){ + $by[$b][] = $k; + } + } + $this->fieldIndexCache[$pk]['bytes'] += 96 * (count($last) - ($tail === null ? 0 : count($tail['last']))); + $tail = ['to' => $end, 'last' => $last, 'by' => $by]; + $this->fieldIndexCache[$pk]['tail'] = $tail; + $this->cacheGrew = true; + } + $keys = []; + foreach($found as $k){ + if(!array_key_exists($k, $tail['last'])){ + $keys[] = $k; + } + } + foreach($tail['by'][$bucket] ?? [] as $k){ + $keys[] = $k; + } + sort($keys); + return $keys; + } + + /** Entries of an .fx whose whole load costs about one paged lookup more than an array lookup (§6.5.1) */ + private const ENTRIES_PER_LOOKUP = 20; + + /** The body of an .fx in use (from its second operation) of at most this many bytes is read whole on a page miss (§6.5.1) */ + private const RAW_BODY = 262144; + + /** An .fx file of at most this many bytes is loaded whole at its first use (§6.5.1) */ + public const WHOLE_AT_FIRST_USE = 65536; + + /** The largest database (records) an exact map is made for: making it reads every shard's `.fx` whole (§6.5.2) */ + private const EXACT_MAP_RECORDS = 1000000; + + /** + * True when an .fx file not loaded whole is small enough to be loaded at this use, and memory allows (its + * decoded form estimated at 16 bytes per byte of the file). Not in the process's first operation (lock + * session): a single cold operation reads the pages it needs, decoding pays off only when more follow. + * The size: the pages kept of it (header and body), else the file's. + */ + private function wholeAtFirstUse($path){ + if($this->sessionCounter < 2){ + return false; + } + $kept = $this->fieldIndexCache[$path . '#pg'] ?? null; + $size = $kept === null ? @filesize($path) : $kept['at'] + $kept['size']; + return $size !== false && $size <= self::WHOLE_AT_FIRST_USE && \noneDB\Internal\Cache\CacheManager::decodeFits(16 * $size); + } + + /** + * True when an .fx in repeated use should be loaded whole now, while memory allows (plan D6: hot segments + * decoded within the budget): once its operations have paid for the load (at least two, one per + * ENTRIES_PER_LOOKUP entries). A query that runs once, an index used now and then and a process near its + * cache limit keep the pages. + */ + private function loadWholeNow($key){ + $entry = &$this->fieldIndexCache[$key]; + if($entry['ses'] !== $this->sessionCounter){ + $entry['ses'] = $this->sessionCounter; + $entry['uses']++; + } + $uses = $entry['uses']; + $size = $entry['size']; + $header = $entry['h']; + unset($entry); + $entries = (int)($header['c'] ?? intdiv($size, 16)); + if($uses < 2 || $uses * self::ENTRIES_PER_LOOKUP < $entries){ + return false; + } + $held = (int)($header['n'] ?? 4 * $size); + return \noneDB\Internal\Cache\CacheManager::decodeFits(128 + 48 * $entries + 24 * $held); + } + + /** + * The header of an .fx file that fits its segment, kept with its pages in fieldIndexCache[path#pg]; null + * when there is none + */ + private function openPagedIndex($dbname, $shardId, $path, $field, $key = null){ + $entry = $this->fieldIndexCache[$path . '#pg'] ?? null; + if($entry !== null){ + $this->metrics['hits']++; + } else { + $this->metrics['misses']++; + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + if(!is_array($header) || ($header['kind'] ?? null) !== 'fx' || ($header['v'] ?? null) !== 1 || ($header['field'] ?? null) !== (string)$field + || !PagedFile::validTree($header['pg'] ?? null) + || !is_string($header['w'] ?? null) || ($w = strpos($line, '"w":"' . $header['w'] . '"')) === false){ + fclose($fp); + return null; + } + // 'id': the header up to the end of its write token, read again before each page (§6.5.1) + // 'size': the body's bytes (the root is its last page) + $entry = ['h' => $header, 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line), 'pages' => [], 'dirs' => [], + 'size' => (int)$header['pg']['root'][0] + (int)$header['pg']['root'][1], 'bytes' => 2 * strlen($line), 'uses' => 0, 'ses' => -1]; + if((fstat($fp)['size'] ?? null) !== $entry['at'] + $entry['size']){ + fclose($fp); + return null; // the root is the file's last page: a body the file doesn't hold is never asked for + } + // Every lookup starts at the root: read with the header, from the same file + [$offset, $length] = $header['pg']['root']; + fseek($fp, $entry['at'] + (int)$offset); + $root = PagedFile::open((string)fread($fp, (int)$length)); + $this->cache->noteIndexRead((int)$length, 1); + $children = $root === null || $header['pg']['levels'] === 0 ? null : PagedFile::directory($root); + if($root !== null && ($header['pg']['levels'] === 0 || $children !== null)){ + $this->indexStats['pagesRead']++; // @test + if($children === null){ + $entry['pages'][(int)$offset] = $root; + } else { + $entry['dirs'][(int)$offset] = $children; + } + $entry['bytes'] += 2 * (int)$length; + } + fclose($fp); + $this->fieldIndexCache[$path . '#pg'] = $entry; + $this->cacheGrew = true; + } + $this->cacheTouch[$path . '#pg'] = ++$this->cacheTick; + // A segment index fits while its key index keeps the epoch it was checked against (the bytes it + // covers can't change within an epoch): checked once, not at every lookup + if($key === null || $key['e'] === null || ($entry['fit'] ?? null) !== $key['e']){ + if(!$this->secondary->secondaryFits($dbname, $shardId, $entry['h'], 'fx', $field)){ + unset($this->fieldIndexCache[$path . '#pg']); + return null; + } + if($key !== null && $key['e'] !== null){ + $this->fieldIndexCache[$path . '#pg']['fit'] = $entry['fit'] = $key['e']; + } + } + return $entry; + } + + /** + * The keys of a bucket in an opened .fx ([] when the value isn't in it); null when a page on the path is + * damaged; false when another process replaced the file since its header was read (a reader's checkpoint): + * its pages are dropped, they don't fit the new file. Pages are kept as they are; a data page is searched + * where it is (§6.5.1 lookup step 3). + */ + private function pagedLookup($path, $bucket){ + // Every page on the path kept: answered from them, no handle and no header read + $kept = $this->fieldIndexCache[$path . '#pg']; + $pg = $kept['h']['pg']; + [$offset, $length] = $pg['root']; + for($level = $pg['levels']; $level > 0 && isset($kept['dirs'][$offset]); $level--){ + $child = PagedFile::child($kept['dirs'][$offset], $bucket); + if($child === []){ + return []; + } + [$offset, $length] = $child; + } + if($level === 0 && isset($kept['pages'][$offset])){ + return PagedFile::fxMember($kept['pages'][$offset], $bucket); + } + if($level === 0 && isset($kept['ok'][$offset])){ + // The page inside the body read whole, its crc checked: searched where it is + return PagedFile::fxMember($kept['raw'], $bucket, $offset, $offset + $length - 4); + } + unset($kept); + $entry = &$this->fieldIndexCache[$path . '#pg']; + [$offset, $length] = $pg['root']; + $fp = null; // one handle for the pages of this lookup, opened when a page isn't kept + try { + for($level = (int)$pg['levels']; $level > 0; $level--){ + // Directory pages are kept decoded (few, reused by every lookup) + $children = $entry['dirs'][$offset] ?? null; + if($children === null){ + $page = $this->readPage($path, $entry, $fp, $offset, $length); + if(!is_string($page)){ + return $page; + } + $children = PagedFile::directory($page); + if($children === null){ + return null; + } + $entry['dirs'][$offset] = $children; + $entry['bytes'] += 3 * $length; + } + $child = PagedFile::child($children, $bucket); + if($child === []){ + return []; + } + [$offset, $length] = $child; + } + // A data page is kept as it is, and searched where it is + $page = $entry['pages'][$offset] ?? null; + if($page === null){ + $page = $this->readPage($path, $entry, $fp, $offset, $length); + if(!is_string($page)){ + return $page; + } + if(isset($entry['raw'])){ + $entry['ok'][$offset] = true; // checked once; searched inside the body from now on + return PagedFile::fxMember($entry['raw'], $bucket, $offset, $offset + $length - 4); + } + $entry['pages'][$offset] = $page; + $entry['bytes'] += $length; + } + return PagedFile::fxMember($page, $bucket); + } finally { + unset($entry); + if($fp !== null && $fp !== false){ + fclose($fp); + } + } + } + + /** + * Read one page of an opened paged file through the lookup's handle (opened on first use, its header + * checked against the one the entry holds): the page without its crc; null when it is damaged or the + * file can't be opened; false when the file was replaced (the entry is dropped) + */ + private function readPage($path, array &$entry, &$fp, $offset, $length){ + if($length < 1 || $offset + $length > $entry['size']){ + return null; // a child outside the body + } + if(isset($entry['raw'])){ + return PagedFile::open(substr($entry['raw'], $offset, $length)); + } + if($fp === null){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $this->cache->noteIndexRead(strlen($entry['id']), 0); + if(fread($fp, strlen($entry['id'])) !== $entry['id']){ + unset($this->fieldIndexCache[$path . '#pg']); + return false; + } + } + if($entry['uses'] >= 2 && $entry['size'] <= self::RAW_BODY + && \noneDB\Internal\Cache\CacheManager::decodeFits(2 * $entry['size'])){ + // An .fx in use in a later operation: its body read whole from this handle (header checked), and + // every page of it answered from those bytes; the pages kept so far are in them + fseek($fp, $entry['at']); + $raw = (string)fread($fp, $entry['size']); + $this->cache->noteIndexRead(strlen($raw)); + if(strlen($raw) === $entry['size']){ + $this->indexStats['bodiesRead']++; // @test + $entry['raw'] = $raw; + $entry['bytes'] += $entry['size'] - array_sum(array_map('strlen', $entry['pages'])); + $entry['pages'] = []; + $entry['ok'] = []; + $this->cacheGrew = true; + return PagedFile::open(substr($raw, $offset, $length)); + } + } + fseek($fp, $entry['at'] + $offset); + $page = PagedFile::open((string)fread($fp, $length)); + $this->cache->noteIndexRead($length, 1); + if($page !== null){ + $this->indexStats['pagesRead']++; // @test + $this->cacheGrew = true; + } + return $page; + } + + /** + * Every bucket of a paged .fx body: [bucket => key | [keys]]; null when the file is damaged + */ + public function decodeFieldIndexBody(array $header, $body){ + $this->indexStats['bodiesDecoded']++; // @test + $pg = $header['pg'] ?? null; + if(!is_array($pg)){ + return null; + } + $pages = PagedFile::dataPages((string)$body, $pg); + if($pages === null){ + return null; + } + $buckets = []; + foreach($pages as [$offset, $length]){ + $data = PagedFile::open(substr((string)$body, $offset, $length)); + if($data === null || !PagedFile::fxMembersInto($data, $buckets)){ + return null; + } + } + return $buckets; + } + + /** + * The equality filters field indexes can answer: [field => value], empty when there are none. A query works + * this out once, not per segment. + * @return array + */ + public function indexedFilters($dbname, array $filters){ + $usable = []; + if(!$this->fieldIndexEnabled || empty($filters)){ + return $usable; + } + $indexed = $this->getIndexedFields($dbname); + foreach($filters as $field => $value){ + if($this->isIndexableValue($value) && in_array((string)$field, $indexed, true)){ + $usable[$field] = $value; + } + } + return $usable; + } + + /** + * Candidate keys of a segment for equality filters, from field indexes. + * Null means "no usable index, scan the segment". + * @return array|null + */ + public function fieldIndexCandidates($dbname, $shardId, array $filters){ + $usable = $this->indexedFilters($dbname, $filters); + return empty($usable) ? null : $this->segmentCandidates($dbname, $shardId, $usable); + } + + /** + * Candidate keys of a segment for filters of indexedFilters() + * @param array|null $key The segment's key index, when the caller just read it + * @return array + */ + public function segmentCandidates($dbname, $shardId, array $usable, $key = null){ + $candidates = null; + foreach($usable as $field => $value){ + // Loaded whole (or small enough to be at its first use): answered from memory, readFieldIndex() + // replays newer lines; else a point lookup + $path = $this->getFieldIndexPath($dbname, (string)$field, $shardId); + $keys = isset($this->fieldIndexCache[$path]) || $this->wholeAtFirstUse($path) + ? null : $this->pointKeys($dbname, $shardId, (string)$field, $value, $path, $key); + if($keys === null){ + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field, $path, $key); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; // never read without the lock + } + $index = null; // can't be built: scan + } + $keys = $index === null ? null : $this->fieldIndexKeys($index, $value); + } + if($keys === null){ + continue; + } + $candidates = $candidates === null ? $keys : array_values(array_intersect($candidates, $keys)); + if(empty($candidates)){ + return []; + } + } + return $candidates; + } + + // ==================== RANGE VIEWS (I3) ==================== + + /** + * Candidate keys of a segment whose field value may satisfy every range condition ($gt, $gte, $lt, + * $lte => operand): a superset, the caller checks every record. Null: no usable index (scan). + * @param array $conditions [operator => operand] + * @return array|null + */ + public function fieldIndexRangeCandidates($dbname, $shardId, $field, array $conditions){ + if(!$this->fieldIndexEnabled || !$this->isFieldIndexed($dbname, $field)){ + return null; + } + try { + $index = $this->ensureSegmentFieldIndex($dbname, $shardId, $field); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + return null; + } + if($index === null){ + return null; + } + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $views = $this->fieldIndexViews($index); + if(isset($this->fieldIndexCache[$path])){ + $this->fieldIndexCache[$path]['views'] = $views; + } + $buckets = null; // bucket => true + foreach($conditions as $op => $operand){ + if(!in_array($op, ['$gt', '$gte', '$lt', '$lte'], true)){ + return null; + } + $found = $this->rangeBuckets($views, $op, $operand); + if($found === null){ + return null; // an operand nothing compares with: no help from the index + } + $buckets = $buckets === null ? $found : array_intersect_key($buckets, $found); + if(empty($buckets)){ + return []; + } + } + if($buckets === null){ + return null; + } + $keys = []; + foreach(array_keys($buckets) as $bucket){ + $bucketKeys = $index['b'][$bucket] ?? []; + if(is_int($bucketKeys)){ + $keys[] = $bucketKeys; + } else { + foreach($bucketKeys as $k){ + $keys[] = $k; + } + } + } + return $keys; + } + + /** + * Sorted views of a field index's buckets: numbers (ints, floats and numeric strings, by value) and + * strings (byte order) + * @return array ['num' => [float values], 'numIds' => [their buckets], 'str' => [strings], 'strIds' => [their buckets]] + */ + private function fieldIndexViews(array $index){ + if($index['views'] !== null){ + return $index['views']; + } + // Sort keys first, then one array_multisort per view (a comparator per pair is far slower) + $numValues = []; + $numIds = []; + $strValues = []; + $strIds = []; + foreach(array_keys($index['b']) as $bucket){ + $bucket = (string)$bucket; + $type = $bucket[0]; + if($type === 'i' || $type === 'd'){ + $numValues[] = (float)self::bucketNumber($bucket); + $numIds[] = $bucket; + } elseif($type === 's'){ + $s = substr($bucket, 1); + $strValues[] = $s; + $strIds[] = $bucket; + if(is_numeric($s)){ + $numValues[] = (float)$s; + $numIds[] = $bucket; + } + } + } + array_multisort($numValues, SORT_NUMERIC, $numIds, SORT_STRING); + array_multisort($strValues, SORT_STRING, $strIds, SORT_STRING); + return ['num' => $numValues, 'numIds' => $numIds, 'str' => $strValues, 'strIds' => $strIds]; + } + + /** + * Buckets whose values may compare as $op with $operand (\noneDBQuery::compareValues semantics: numbers + * and numeric strings numerically, other strings by byte order; null, booleans and arrays compare + * with nothing). Ties of different types are included (rechecked by the caller). + * @return array|null bucket => true; null when the operand can't be compared + */ + private function rangeBuckets(array $views, $op, $operand){ + if(is_float($operand) && is_nan($operand)){ + return []; // NAN matches nothing + } + if(is_int($operand) || is_float($operand) || (is_string($operand) && is_numeric($operand))){ + $x = (float)$operand; + // A wider band at the boundary: values above 2^53 compare as floats here and as ints there + $slack = abs($x) >= 9007199254740992.0 ? max(1.0, abs($x) * 4.0e-16) : 0.0; + [$from, $length] = $this->rangeSlice($views['num'], $op, $x, $slack); + $found = array_fill_keys(array_slice($views['numIds'], $from, $length), true); + if(is_string($operand)){ + // A numeric string operand compares with non-numeric strings by byte order too + [$from, $length] = $this->rangeSliceStrings($views['str'], $op, $operand); + $found += array_fill_keys(array_slice($views['strIds'], $from, $length), true); + } + return $found; + } + if(is_string($operand)){ + [$from, $length] = $this->rangeSliceStrings($views['str'], $op, $operand); + return array_fill_keys(array_slice($views['strIds'], $from, $length), true); + } + return null; + } + + /** + * Where the entries of a sorted numeric view on the operator's side of $x are, [from, length] (binary search; $slack widens it). + * Without slack ($x below 2^53, where these floats are exact) a strict operator leaves out the values + * equal to $x: compareValues() finds them equal too, so they never match + */ + private function rangeSlice(array $list, $op, $x, $slack){ + $n = count($list); + $strict = $slack == 0.0 && ($op === '$gt' || $op === '$lt'); + if($op === '$gt' || $op === '$gte'){ + $bound = $x - $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid] <= $bound : $list[$mid] < $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [$lo, $n - $lo]; + } + $bound = $x + $slack; + $lo = 0; + $hi = $n; + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + if($strict ? $list[$mid] < $bound : $list[$mid] <= $bound){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [0, $lo]; + } + + /** + * Where the entries of the sorted string view on the operator's side of $x are, [from, length] (the string + * $x itself only for $gte and $lte) + */ + private function rangeSliceStrings(array $list, $op, $x){ + $n = count($list); + $lo = 0; + $hi = $n; + $strict = $op === '$gt' || $op === '$lt'; + if($op === '$gt' || $op === '$gte'){ + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid], $x); + if($strict ? $cmp <= 0 : $cmp < 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [$lo, $n - $lo]; + } + while($lo < $hi){ + $mid = ($lo + $hi) >> 1; + $cmp = strcmp($list[$mid], $x); + if($strict ? $cmp < 0 : $cmp <= 0){ + $lo = $mid + 1; + } else { + $hi = $mid; + } + } + return [0, $lo]; + } + + // ==================== GLOBAL FIELD INDEX (shard skip) ==================== + // indexes/{token}.gfx (docs/STORAGE_FORMAT.md §6.5.2): per shard, a filter of the bucket ids of its field + // index at the bytes [0, s) its entry {"s","e","h"} names, bit-sliced: row r holds bit r of every shard's + // filter, one bit per shard, rows in pages with a crc. While the lock epoch is the one recorded (le), no data + // file was replaced, so every entry still describes its shard; otherwise each entry is checked against its + // shard (epoch and checksum). A shard is skipped only when it has an entry that fits, its filter rules the + // value out, and its lines after s don't have it either. No entry: never skipped. A shard's filter is + // replaced when its field index is checkpointed, never per write. In memory: the header ('le', 'sh', 'bits', + // 'width'; 'id' and 'at': the header bytes through w and where the pages start, of the file it was read + // from; 'uses', 'ses': its operations; 'body': the pages, once loaded whole) and 'cols', the filters this + // session changed and hasn't written yet ('grow': one of them needs larger sizes, the file is made again). + + public function getGlobalFieldIndexPath($dbname, $field){ + return $this->secondary->secondaryPath($dbname, 'gfx', $field); + } + + /** + * The global field index of a field: its header, checked (§6.5.2); null when there is no valid file + * @return array|null + */ + private function readGlobalFieldIndex($dbname, $field, $path = null){ + $path = $path ?? $this->getGlobalFieldIndexPath($dbname, $field); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(array_key_exists($path, $this->fieldIndexCache)){ + $this->metrics['hits']++; + return $this->fieldIndexCache[$path]; // the header, or null (no valid file) + } + $this->metrics['misses']++; + clearstatcache(true, $path); + $fp = @fopen($path, 'rb'); + $line = $fp === false ? false : fgets($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + $size = $fp === false ? 0 : (int)(fstat($fp)['size'] ?? 0); + if($fp !== false){ + fclose($fp); + } + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + $global = null; + if(is_array($header) && ($header['kind'] ?? null) === 'gfx' && ($header['v'] ?? null) === 1 && ($header['field'] ?? null) === (string)$field + && is_string($header['w'] ?? null) && ($w = strpos($line, '"w":"' . $header['w'] . '"')) !== false + && self::isSize($header['bits'] ?? null, 64) && self::isSize($header['width'] ?? null, 8) && is_array($header['sh'] ?? null) + && $size === strlen($line) + self::globalBodyBytes($header['bits'], $header['width'])){ + $global = ['le' => $header['le'] ?? null, 'sh' => [], 'bits' => $header['bits'], 'width' => $header['width'], + 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line), 'cols' => [], 'uses' => 0, 'ses' => -1]; + foreach($header['sh'] as $shardId => $entry){ + if(SecondaryFiles::isEntry($entry) && (int)$shardId < $header['width']){ + $global['sh'][(int)$shardId] = $entry; + } + } + } + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$this->locks->getLockPath($dbname)][$path]); + $this->cacheGrew = true; + return $global; + } + + /** A size of a global field index: a power of two of at least $min */ + private static function isSize($n, $min){ + return is_int($n) && $n >= $min && ($n & ($n - 1)) === 0; + } + + /** Bytes of the pages of a global field index of these sizes */ + private static function globalBodyBytes($bits, $width){ + $per = ShardFilters::pageRows($bits, $width); + return intdiv($bits, $per) * ($per * ($width >> 3) + 4); + } + + /** + * The shards whose filter may hold a bucket id (§6.5.2), with the filters this session changed and hasn't + * written yet: the AND of its six probe rows. The rows come from the body when it is loaded; else from the + * pages that hold them, read from the file after its header (read again from the same handle). The body is + * loaded whole at the file's second operation, when memory allows (every page's crc checked). Null when the + * file is damaged or gone; false when another process replaced it since its header was read. + * @return int[]|null|false ascending + */ + private function globalHolders($path, $bucket){ + $global = $this->fieldIndexCache[$path]; + $probes = ShardFilters::probes($bucket, $global['bits']); + $held = []; + if($global['id'] !== null){ + $rb = $global['width'] >> 3; + $per = ShardFilters::pageRows($global['bits'], $global['width']); + $stored = $per * $rb + 4; + $rows = []; + if(!isset($global['body'])){ + $pages = intdiv($global['bits'], $per); + $whole = $global['uses'] >= 2 && \noneDB\Internal\Cache\CacheManager::decodeFits($pages * $stored); + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + try { + $this->cache->noteIndexRead(strlen($global['id']), 0); + if(fread($fp, strlen($global['id'])) !== $global['id']){ + return false; + } + if($whole){ + fseek($fp, $global['at']); + $body = (string)fread($fp, $pages * $stored); + $this->cache->noteIndexRead(strlen($body), $pages); + for($p = 0; $p < $pages; $p++){ + if(ShardFilters::open(substr($body, $p * $stored, $stored), $stored - 4) === null){ + return null; + } + } + $global['body'] = $this->fieldIndexCache[$path]['body'] = $body; + $this->cacheGrew = true; + } else { + $read = []; + foreach($probes as $r){ + $p = intdiv($r, $per); + if(!isset($read[$p])){ + fseek($fp, $global['at'] + $p * $stored); + $read[$p] = ShardFilters::open((string)fread($fp, $stored), $stored - 4); + $this->cache->noteIndexRead($stored, 1); + if($read[$p] === null){ + return null; + } + } + $rows[] = substr($read[$p], ($r % $per) * $rb, $rb); + } + } + } finally { + fclose($fp); + } + } + $this->indexStats['pointLookups']++; // @test + $held = isset($global['body']) ? ShardFilters::holdersIn($global['body'], $global['bits'], $rb, $bucket) : ShardFilters::holders($rows); + } + if(empty($global['cols'])){ + return $held; + } + $held = array_fill_keys($held, true); + foreach($global['cols'] as $shardId => $filter){ + if(ShardFilters::mayHold($filter, $probes)){ + $held[$shardId] = true; + } else { + unset($held[$shardId]); + } + } + $held = array_keys($held); + sort($held); + return $held; + } + + /** + * Write the global field index (exclusive lock, or a reader: the content is a function of the shards' field + * indexes): its pages copied from the file it was read from, with the bits of the filters of 'cols' in place + * of the old ones. Made again from every shard's field index when the sizes must grow, there was no valid + * file, or the old one can't be copied (replaced or gone since, a damaged page). + */ + public function writeGlobalFieldIndex($dbname, $field, array $global){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $source = empty($global['grow']) ? @fopen($path, 'rb') : false; + if($source !== false){ + $this->cache->noteIndexRead(strlen($global['id']), 0); + } + if($source === false || fread($source, strlen($global['id'])) !== $global['id']){ + if($source !== false){ + fclose($source); + } + $this->rebuildGlobalFieldIndex($dbname, $field); + return; + } + fseek($source, $global['at']); + $pages = function() use ($source, $global){ + $rb = $global['width'] >> 3; + $per = ShardFilters::pageRows($global['bits'], $global['width']); + for($p = 0, $n = intdiv($global['bits'], $per); $p < $n; $p++){ + $rows = ShardFilters::open((string)fread($source, $per * $rb + 4), $per * $rb); + $this->cache->noteIndexRead($per * $rb + 4, 1); + if($rows === null){ + throw new \UnexpectedValueException('damaged page'); + } + foreach($global['cols'] as $shardId => $filter){ + $rows = ShardFilters::withColumn($rows, $p * $per, $rb, $shardId, $filter); + } + yield ShardFilters::seal($rows); + } + }; + try { + $this->writeGlobalFile($dbname, $field, $path, $global['sh'], $global['bits'], $global['width'], $pages()); + } catch(\UnexpectedValueException $e){ + $this->rebuildGlobalFieldIndex($dbname, $field); + } finally { + fclose($source); + } + } + + /** + * Write a global field index file ($pages: its sealed pages, in order) and keep its header as the cached one + */ + private function writeGlobalFile($dbname, $field, $path, array $sh, $bits, $width, iterable $pages){ + $lockPath = $this->locks->getLockPath($dbname); + $le = isset($this->lockRegistry[$lockPath]) ? $this->locks->registryEpoch($lockPath) : null; + ksort($sh); + $header = ['kind' => 'gfx', 'v' => 1, 'field' => (string)$field, 'w' => bin2hex(random_bytes(8)), 'le' => $le, + 'bits' => $bits, 'width' => $width, 'sh' => (object)$sh]; + $line = $this->io->encodeHeader($header); + $content = (function() use ($line, $pages){ + yield $line . "\n"; + yield from $pages; + })(); + if($line === false || !$this->io->writeDerivedFile($path, $content)){ + throw new \RuntimeException("noneDB: failed to write index " . basename($path) . " of database '{$dbname}'"); + } + $this->cache->markFileExists($path); + $w = strpos($line, '"w":"' . $header['w'] . '"'); + $this->fieldIndexCache[$path] = ['le' => $le, 'sh' => $sh, 'bits' => $bits, 'width' => $width, + 'id' => substr($line, 0, $w + 6 + strlen($header['w'])), 'at' => strlen($line) + 1, 'cols' => [], 'uses' => 0, 'ses' => -1]; + unset($this->globalSkip[$lockPath][$path], $this->lockRegistry[$lockPath]['globals'][$path]); + } + + /** + * A shard's field index checkpointed: its filter and entry replace the old ones in the global field index + * (written when the lock session ends) + */ + public function updateGlobalFieldShard($dbname, $field, $shardId, array $index){ + $entry = $this->globalEntry($dbname, $shardId, $index); + $hotKey = $this->getGlobalFieldIndexPath($dbname, $field) . '#hot'; + if(isset($this->fieldIndexCache[$hotKey])){ + // This process's exact map follows: the shard's entry replaced, its values added (a value it no longer + // holds stays listed: the shard is read for it) + $hot = &$this->fieldIndexCache[$hotKey]; + if($entry === null){ + unset($hot['sh'][$shardId]); + } else { + if(!self::sameIdsSince($index, $hot['sh'][$shardId] ?? null)){ + foreach(array_keys($index['b']) as $bucket){ + if(!isset($hot['map'][$bucket])){ + $hot['map'][$bucket] = $shardId; + } elseif(is_int($hot['map'][$bucket])){ + if($hot['map'][$bucket] !== $shardId){ + $hot['map'][$bucket] = [min($hot['map'][$bucket], $shardId), max($hot['map'][$bucket], $shardId)]; + } + } elseif(!in_array($shardId, $hot['map'][$bucket], true)){ + $hot['map'][$bucket][] = $shardId; + sort($hot['map'][$bucket]); // holders in shard order: the order of the records read + } + } + } + $hot['sh'][$shardId] = $entry; + } + unset($hot, $this->globalSkip[$this->locks->getLockPath($dbname)][$hotKey]); + } + $global = $this->readGlobalFieldIndex($dbname, $field); + if($global === null){ + // No valid file: made from every shard when the session ends, as a lookup would + $this->deferGlobalWrite($dbname, 'gfx', $field, ['le' => null, 'sh' => [], 'bits' => 64, 'width' => 8, 'id' => null, 'cols' => [], 'grow' => true]); + return; + } + $global = $this->secondary->freshGlobalEntries($dbname, $global); + if($entry === null){ + unset($global['sh'][$shardId], $global['cols'][$shardId]); // never skipped + } elseif(self::sameIdsSince($index, $global['sh'][$shardId] ?? null)){ + $global['sh'][$shardId] = $entry; // the bucket ids its bits were made from: they stay + } elseif(32 * count($index['b']) > $global['bits'] || $shardId >= $global['width']){ + $global['grow'] = true; + $global['sh'][$shardId] = $entry; + } else { + $this->indexStats['filtersBuilt']++; // @test + $global['cols'][$shardId] = ShardFilters::filter(array_keys($index['b']), $global['bits']); + $global['sh'][$shardId] = $entry; + } + $this->deferGlobalWrite($dbname, 'gfx', $field, $global); + } + + /** + * True when a shard's entry in the global index (or in the exact map) is of a checkpoint of its field index + * since which the index's bucket ids haven't changed: the filter and the values listed with it are the index's. + * Of the same epoch: offsets of a replaced data file (this process's compaction keeps its map) say nothing. + */ + private static function sameIdsSince(array $index, ?array $entry){ + return $entry !== null && $index['idsAt'] !== null && $entry['e'] === $index['e'] && $entry['s'] >= $index['idsAt']; + } + + /** + * Keep a changed global field or spatial index in the cache and write it when the lock session ends + * (flushPendingGlobals()); without a lock at once + */ + public function deferGlobalWrite($dbname, $kind, $field, array $global){ + $lock = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$lock])){ + if($kind === 'gfx'){ + $this->writeGlobalFieldIndex($dbname, $field, $global); + } else { + $this->rtree->writeGlobalSpatialIndex($dbname, $field, $global); + } + return; + } + if($kind === 'gfx'){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $this->fieldIndexCache[$path] = $global; + unset($this->globalSkip[$lock][$path]); + } else { + $path = $this->rtree->getGlobalSpatialIndexPath($dbname, $field); + $this->globalSpatialIndexCache[$path] = $global; + } + $this->lockRegistry[$lock]['globals'][$path] = [$kind, $field, $this->locks->registryEpoch($lock)]; + } + + /** + * The global entry of a shard's field index (the bytes it covers, its epoch and their checksum); null when + * it can't have one (no epoch, data file gone) + * @return array|null + */ + private function globalEntry($dbname, $shardId, array $index){ + if($index['e'] === null){ + return null; + } + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$index['s']); + return $crc === null ? null : ['s' => (int)$index['s'], 'e' => $index['e'], 'h' => $crc]; + } + + /** A shard's field index for the global one: as it is, or built from the data (and written) when there is none */ + private function shardFieldIndex($dbname, $shardId, $field){ + $index = $this->readFieldIndex($dbname, $field, $shardId); + if($index === null){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + } + return $index; + } + + /** + * Make the global field index again from every shard's field index (exclusive lock, or a reader): the sizes and + * the shards' entries from a first pass, then the rows in parts of whole pages, each part one more pass over the + * shards that sets the probe bits falling into it. A part holds at most a quarter of the cache limit (all rows + * at once without a limit); the cache evicts between the shards: the process holds a part and one shard's index. + * @return array|null The header, as readGlobalFieldIndex() returns it + */ + public function rebuildGlobalFieldIndex($dbname, $field){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + $lockPath = $this->locks->getLockPath($dbname); + unset($this->fieldIndexCache[$path], $this->fieldIndexCache[$path . '#hot'], $this->globalSkip[$lockPath][$path], + $this->globalSkip[$lockPath][$path . '#hot'], $this->lockRegistry[$lockPath]['globals'][$path]); + $shards = $this->segments->listSegments($dbname); + sort($shards); + $largest = 0; + $sh = []; + foreach($shards as $shardId){ + // The sizes need each shard's count of values and its entry: a file that fits its data gives them by its + // header (its bucket ids are read once, below); a missing or stale one by the index built for it + $header = $this->secondary->readSecondaryHeader($this->getFieldIndexPath($dbname, $field, $shardId)); + if($header !== null && ($header['v'] ?? null) === 1 && $this->secondary->secondaryFits($dbname, $shardId, $header, 'fx', $field)){ + $count = (int)$header['c']; + $entry = $this->globalEntry($dbname, $shardId, ['e' => $header['e'], 's' => (int)$header['s']]); + } else { + $index = $this->shardFieldIndex($dbname, $shardId, $field); + $count = count($index['b']); + $entry = $this->globalEntry($dbname, $shardId, $index); + unset($index); + } + $largest = max($largest, $count); + if($entry !== null){ + $sh[$shardId] = $entry; // a shard without one is never skipped: no bits + } + $this->cache->evictBetweenSegments(); + } + $bits = ShardFilters::size(32 * $largest, 64); + $width = ShardFilters::size(empty($shards) ? 1 : end($shards) + 1, 8); + $rb = $width >> 3; + $per = ShardFilters::pageRows($bits, $width); + $pages = intdiv($bits, $per); + $limit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes(); + $part = $limit > 0 ? max(1, intdiv(intdiv($limit, 4), $per * $rb)) : $pages; // pages per part + // ShardFilters::probes() inline, per part the places of the first bucket ids kept (about 256 bytes each): at + // most 16 384, and within a quarter of the cache limit. A value that many shards hold is hashed once + $keep = $limit > 0 ? min(16384, intdiv($limit, 1024)) : 16384; + $this->writeGlobalFile($dbname, $field, $path, $sh, $bits, $width, (function() use ($dbname, $field, $sh, $bits, $rb, $per, $pages, $part, $keep){ + $mask = ($bits - 1) & 0xFFFFFFFF; + // A byte with one bit set, per bit: an array lookup where chr(ord() | bit) makes two calls per probe + $withBit = []; + for($bit = 0; $bit < 8; $bit++){ + for($byte = 0; $byte < 256; $byte++){ + $withBit[$bit][chr($byte)] = chr($byte | (1 << $bit)); + } + } + for($first = 0; $first < $pages; $first += $part){ + $lo = $first * $per; + $hi = min($pages, $first + $part) * $per; + $rows = str_repeat("\0", ($hi - $lo) * $rb); // rows [lo, hi), one after the other + $seen = []; + $kept = 0; + foreach(array_keys($sh) as $shardId){ + // The shard's filter written straight into its column: bit r of the filter is its bit in row r + $index = $this->shardFieldIndex($dbname, $shardId, $field); + $byte = $shardId >> 3; + $set = $withBit[$shardId & 7]; + foreach(array_keys($index['b']) as $id){ + $id = (string)$id; + $places = $seen[$id] ?? null; + if($places === null){ + $d = unpack('N2', hash('xxh3', $id, true)); + $y = $d[2] | 1; + $places = []; + foreach([$d[1] & $mask, ($d[1] + $y) & $mask, ($d[1] + 2 * $y) & $mask, + ($d[1] + 3 * $y) & $mask, ($d[1] + 4 * $y) & $mask, ($d[1] + 5 * $y) & $mask] as $r){ + if($r >= $lo && $r < $hi){ + $places[] = ($r - $lo) * $rb; + } + } + if($kept < $keep){ + $seen[$id] = $places; + $kept++; + } + } + foreach($places as $at){ + $at += $byte; + $rows[$at] = $set[$rows[$at]]; + } + } + unset($index); + $this->cache->evictBetweenSegments(); + } + for($p = 0, $n = intdiv($hi - $lo, $per); $p < $n; $p++){ + yield ShardFilters::seal(substr($rows, $p * $per * $rb, $per * $rb)); + } + unset($rows); + } + })()); + return $this->fieldIndexCache[$path]; + } + + /** + * True when a global entry still describes its shard's data bytes [0, s) + */ + public function globalEntryFits($dbname, $shardId, array $entry, $lockEpoch){ + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + if($lockEpoch !== null && $lock !== null && $this->locks->registryEpoch($this->locks->getLockPath($dbname)) === $lockEpoch){ + return true; // no data file was replaced since the entry was written + } + if(($this->dataEpochs[$dataPath] ?? null) === null){ + $this->dataEpochs[$dataPath] = $this->segmentEpochOnDisk($dbname, $shardId); + } + if($this->dataEpochs[$dataPath] !== $entry['e']){ + return false; + } + $checked = $entry['e'] . ':' . $entry['s'] . ':' . $entry['h']; // the checksum too: another file may cover the same bytes + if(($this->secondaryChecked[$dataPath] ?? null) === $checked){ + return true; + } + if($this->checkpoints->segmentCrc($dataPath, (int)$entry['s']) !== $entry['h']){ + return false; + } + $this->secondaryChecked[$dataPath] = $checked; + return true; + } + + /** + * A segment's data epoch without loading its key index: the cached index, else the checkpoint header + */ + private function segmentEpochOnDisk($dbname, $shardId){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + if(isset($this->indexCache[$path])){ + return $this->indexCache[$path]['e']; + } + $fp = @fopen($path, 'rb'); + if($fp === false){ + return false; + } + $head = (string)fread($fp, 512); + fclose($fp); + $this->cache->noteIndexRead(strlen($head)); + return preg_match('/"e":"([0-9a-f]{8})"/', $head, $m) === 1 ? $m[1] : false; + } + + /** + * Current data size of a shard: its key index's when this process has it verified (its own writes + * keep it current), else the file's (cached until a write) + */ + private function shardDataSize($dbname, $shardId, $dataPath = null){ + $dataPath = $dataPath ?? $this->segments->segDataPath($dbname, $shardId); + if(isset($this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)], $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)])){ + return (int)$this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]['s']; + } + if(!isset($this->dataSizes[$dataPath])){ + clearstatcache(true, $dataPath); + $this->dataSizes[$dataPath] = (int)@filesize($dataPath); + } + return $this->dataSizes[$dataPath]; + } + + /** + * The shards a lookup in a global field index looks at besides the value's holders: [shard => false] + * has no entry that fits its data (always read), [shard => true] has lines after its entry (read when + * they may hold the value). Worked out over all shards once per state of the data (another process's + * write or a new epoch starts over: invalidateDbCaches(), bumpEpoch()) and kept current with this + * process's own writes (noteWrite()), so a lookup costs O(holders + such shards), not O(shards). + * @return array shard id => bool + */ + private function globalCheckShards($dbname, array $global, $path, $lockPath){ + $seq = $this->segWrites[$lockPath]['seq'] ?? 0; + $state = $this->globalSkip[$lockPath][$path] ?? null; + if($state !== null && $state['seq'] === $seq){ + return $state['check']; + } + if($state === null){ + // While the lock epoch is the recorded one no data file was replaced: every entry fits + $fresh = $global['le'] !== null && isset($this->lockRegistry[$lockPath]) && $global['le'] === $this->locks->registryEpoch($lockPath); + $check = []; + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + foreach($this->segments->listSegments($dbname) as $shardId){ + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || (!$fresh && !$this->globalEntryFits($dbname, $shardId, $entry, null))){ + $check[$shardId] = false; + } elseif($this->shardDataSize($dbname, $shardId, $prefix . $shardId . '.jsonl') > (int)$entry['s']){ + $check[$shardId] = true; + } + } + $state = ['seq' => $seq, 'check' => $check]; + } else { + // This process appended to some shards since: their entries still fit (the bytes they cover + // didn't change), their newer lines are checked; a new shard has no entry + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + $len = strlen($prefix); + foreach($this->segWrites[$lockPath]['paths'] as $dataPath => $at){ + if($at > $state['seq'] && strncmp($dataPath, $prefix, $len) === 0){ + $shardId = (int)substr($dataPath, $len); + if(!isset($state['check'][$shardId])){ + $state['check'][$shardId] = isset($global['sh'][$shardId]); + } + } + } + $state['seq'] = $seq; + } + $this->globalSkip[$lockPath][$path] = $state; + return $state['check']; + } + + /** + * True when a shard's data lines after a global entry (whose bytes are covered) may hold a value + * matching $match (callable(record) => bool) + */ + public function shardTailMayMatch($dbname, $shardId, array $entry, callable $match){ + if($this->shardDataSize($dbname, $shardId) <= (int)$entry['s']){ + return false; + } + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null){ + return true; + } + foreach($this->secondary->segmentTail($dbname, $shardId, (int)$entry['s'], (int)$index['s']) as [$k, $record]){ + if($record !== null && $match($record)){ + return true; + } + } + return false; + } + + /** + * Shards that can contain matches for the equality filters (null = all shards / not sharded) + * @return array|null + */ + public function targetShardsFromGlobalIndex($dbname, array $filters){ + if(!$this->fieldIndexEnabled || !$this->segments->isSharded($dbname) || empty($filters)){ + return null; + } + $manifest = $this->getIndexManifest($dbname); + $targets = null; + $lockPath = null; + foreach($filters as $field => $value){ + if($field === 'key' || !(is_scalar($value) || $value === null) || !in_array((string)$field, $manifest['fields'], true)){ + continue; + } + // Paths built here, not by the helpers: this runs for every lookup of a sharded database + $lockPath = $lockPath ?? $this->locks->getLockPath($dbname); + $path = $this->layout->genDir($dbname) . 'indexes/' . ($this->indexTokens[(string)$field] ?? $this->secondary->indexToken($field)) . '.gfx'; + $bucket = is_string($value) ? 's' . $value : \noneDB\Internal\Query\Values::valueKey($value); + // The value's holders: by the exact map when this process has one, else by the filters + $hotKey = $path . '#hot'; + $hot = $this->fieldIndexCache[$hotKey] ?? null; + $held = null; + if($hot === null){ + $global = $this->fieldIndexCache[$path] ?? null; + if(isset($global['body']) && empty($global['cols']) && array_key_exists($hotKey, $this->fieldIndexCache)){ + // No map will come and the body is held: this is the warm lookup of such a process + $this->cacheTouch[$path] = ++$this->cacheTick; + $this->metrics['hits']++; + $this->indexStats['bodyLookups']++; // @test + $held = ShardFilters::holdersIn($global['body'], $global['bits'], $global['width'] >> 3, $bucket); + } else { + $global = $this->readGlobalFieldIndex($dbname, $field, $path); + if($global !== null && isset($global['ses']) && $global['ses'] !== $this->sessionCounter){ + $this->fieldIndexCache[$path]['ses'] = $this->sessionCounter; // an operation of the file + $global['uses'] = ++$this->fieldIndexCache[$path]['uses']; + } + $hot = $global === null ? null : $this->globalHotMap($dbname, $field, $hotKey, $global); + } + } else { + $this->cacheTouch[$hotKey] = ++$this->cacheTick; + $this->metrics['hits']++; + } + if($hot !== null){ + $held = $hot['map'][$bucket] ?? []; + $held = is_int($held) ? [$held] : $held; + $state = $this->globalSkip[$lockPath][$hotKey] ?? null; + $check = $state !== null && $state['seq'] === ($this->segWrites[$lockPath]['seq'] ?? 0) + ? $state['check'] : $this->globalCheckShards($dbname, $hot, $hotKey, $lockPath); + $shards = empty($check) ? $held : $this->withTailHolders($dbname, $field, $value, $hot, $held, $check); + $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); + continue; + } + $held = $held ?? ($global === null ? null : $this->globalHolders($path, $bucket)); + if($held === false){ + // Another process replaced the file since its header was read here: read again + unset($this->fieldIndexCache[$path]); + $global = $this->readGlobalFieldIndex($dbname, $field, $path); + $held = $global === null ? null : $this->globalHolders($path, $bucket); + } + if(!is_array($held)){ + // None, damaged (or replaced again): made again from the shards + if($this->noWrites > 0 || $this->multiLock > 0){ + continue; // checkIntegrity(), a snapshot: build nothing, skip nothing + } + $global = $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildGlobalFieldIndex($dbname, $field); + }, 'buildFieldIndex'); + $held = $this->globalHolders($path, $bucket); + if(!is_array($held)){ + continue; + } + } + // The shards whose entry can't rule a value out (worked out once per state of the data) + $state = $this->globalSkip[$lockPath][$path] ?? null; + $check = $state !== null && $state['seq'] === ($this->segWrites[$lockPath]['seq'] ?? 0) + ? $state['check'] : $this->globalCheckShards($dbname, $global, $path, $lockPath); + $shards = empty($check) ? $held : $this->withTailHolders($dbname, $field, $value, $global, $held, $check); + $targets = $targets === null ? $shards : array_values(array_intersect($targets, $shards)); + } + return $targets; + } + + /** + * The shards a lookup reads: the holders by the global index, the shards its entries can't rule the value out + * for ($check: shard => false, no fitting entry: read; true: its newer lines may hold it) and of those with + * newer lines the ones whose lines hold it + * @return int[] ascending + */ + private function withTailHolders($dbname, $field, $value, array $global, array $held, array $check){ + $shards = array_fill_keys($held, true); + $match = null; + foreach($check as $shardId => $tail){ + if(isset($shards[$shardId])){ + continue; + } + $match = $match ?? function($record) use ($field, $value){ + return array_key_exists($field, $record) && $record[$field] === $value; + }; + if(!$tail || $this->shardTailMayMatch($dbname, $shardId, $global['sh'][$shardId], $match)){ + $shards[$shardId] = true; + } + } + $shards = array_keys($shards); + sort($shards); + return $shards; + } + + /** + * The exact map of the values of a global field index to the shards that hold them, built from every shard's + * field index file at the global index's second operation (lock session) when memory allows: warm lookups + * without the filters (docs/STORAGE_FORMAT.md §6.5.2 "Lookup"). The files are read, not loaded into the cache; + * a shard whose file doesn't fit its data has no entry (it is read). The map describes each shard at the bytes + * its file covers, as the global file's entries do: newer lines are checked, a shard whose data was replaced is + * read. It stays until the cache drops it. ['le', 'sh' => [shard => entry], 'map' => [bucket => shard | + * [shards]]]; null when there is none: before the second operation, in checkIntegrity() or a snapshot, or + * when it doesn't fit (cached as null, the filters answer) + * @param string $key Its cache key (the global file's path and "#hot"); the caller found nothing under it + * @return array|null + */ + private function globalHotMap($dbname, $field, $key, array $global){ + if(array_key_exists($key, $this->fieldIndexCache)){ + return null; // didn't fit (an exact map would have been found by the caller) + } + if(!isset($global['uses']) || $global['uses'] < 2 || $this->noWrites > 0 || $this->multiLock > 0){ + return null; + } + // Made where it can pay: a web server's request ends with its statics (§6.5.2), and over a million records the + // build (every shard's file read whole) costs seconds. Elsewhere the filters answer + $sapi = PHP_SAPI; // @prod + $sapi = $this->rt->sapi ?? PHP_SAPI; // @test + $records = self::EXACT_MAP_RECORDS; // @prod + $records = $this->rt->exactMapRecords ?? self::EXACT_MAP_RECORDS; // @test + if($sapi !== 'cli' || (int)($this->shardMeta->getCachedMeta($dbname)['totalRecords'] ?? 0) > $records){ + $this->fieldIndexCache[$key] = null; + return null; + } + $limit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes(); + // The file's sizes bound the map: at most bits / 32 values per shard entry (bits ≥ 32 · the largest c), about + // 32 bytes a pair. A map whose bound is over the room left under half of the limit is not built, and nothing + // is read for it (§6.5.2) + if($limit > 0 && memory_get_usage() + count($global['sh']) * $global['bits'] > intdiv($limit, 2)){ + $this->fieldIndexCache[$key] = null; + return null; + } + $lock = $this->locks->getLockPath($dbname); + $hot = ['le' => isset($this->lockRegistry[$lock]) ? $this->locks->registryEpoch($lock) : null, 'sh' => [], 'map' => []]; + foreach($this->segments->listSegments($dbname) as $shardId){ + $path = $this->getFieldIndexPath($dbname, $field, $shardId); + $header = $this->secondary->readSecondaryHeader($path); + $buckets = null; + if($header !== null && ($header['v'] ?? null) === 1 && $this->secondary->secondaryFits($dbname, $shardId, $header, 'fx', $field)){ + $held = $this->fieldIndexCache[$path] ?? null; + if($held !== null && $held['file'] === $header['w']){ + $buckets = $held['b']; // this process holds the file's buckets as they are: no second decode + } else { + $file = $this->secondary->readSecondaryFile($path); + // (the file read whole must be the one whose header was checked) + $buckets = $file === null || ($file[0]['w'] ?? null) !== $header['w'] ? null : $this->decodeFieldIndexBody($file[0], $file[1]); + } + } + if($buckets !== null){ + $map = &$hot['map']; + foreach(array_keys($buckets) as $bucket){ + if(!isset($map[$bucket])){ + $map[$bucket] = $shardId; + } elseif(is_int($map[$bucket])){ + $map[$bucket] = [$map[$bucket], $shardId]; + } else { + $map[$bucket][] = $shardId; + } + } + unset($map); + $hot['sh'][$shardId] = ['s' => (int)$header['s'], 'e' => $header['e'], 'h' => $header['h']]; + } + unset($file, $buckets, $held); + $this->cache->evictBetweenSegments(); + if($limit > 0 && memory_get_usage() > intdiv($limit, 2)){ + $this->fieldIndexCache[$key] = null; // the map would take more than the cache leaves it + return null; + } + } + $this->fieldIndexCache[$key] = $hot; + $this->cacheGrew = true; + return $hot; + } + + /** + * Delete the global field index file + */ + public function deleteGlobalFieldIndex($dbname, $field){ + $path = $this->getGlobalFieldIndexPath($dbname, $field); + clearstatcache(true, $path); + if(file_exists($path)){ + $this->io->fsUnlink($path); + } + $lock = $this->locks->getLockPath($dbname); + unset($this->fieldIndexCache[$path], $this->fieldIndexCache[$path . '#hot'], $this->globalSkip[$lock][$path], $this->globalSkip[$lock][$path . '#hot']); + } + + /** + * After the switch to the sharded layout: the single layout's index files go; every shard's field and + * spatial indexes are built from the data + */ + public function rebuildIndexesAfterSwitch($dbname){ + $manifest = $this->getIndexManifest($dbname); + foreach($manifest['fields'] as $field){ + $this->deleteFieldIndexFile($dbname, $field, null); + $this->deleteGlobalFieldIndex($dbname, $field); + foreach($this->segments->listSegments($dbname) as $shardId){ + $index = $this->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->writeFieldIndex($dbname, $field, $index, $shardId); + unset($index); + $this->cache->evictBetweenSegments(); + } + $this->rebuildGlobalFieldIndex($dbname, $field); + } + foreach($manifest['spatial'] as $field){ + $this->rtree->rebuildSpatialIndexForDb($dbname, $field); + } + } +} diff --git a/src/Internal/Index/FieldIndexApi.php b/src/Internal/Index/FieldIndexApi.php new file mode 100644 index 0000000..13d202f --- /dev/null +++ b/src/Internal/Index/FieldIndexApi.php @@ -0,0 +1,252 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->db, $this->fieldIndex, $this->layout, $this->locks, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->db = null; + $this->fieldIndex = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // FIELD INDEX PUBLIC API + // ========================================== + + /** + * Create an index on a field for faster equality lookups (find / where with field => value) + * @param string $dbname Database name + * @param string $field Field name (any name except "key") + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] (+ 'shards' => int when sharded) + */ + public function createFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->createFieldIndexUnlocked($dbname, $field); + }, 'createFieldIndex'); + } + + private function createFieldIndexUnlocked($dbname, $field){ + if(!is_string($field) || $field === '' || $field === 'key'){ + return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + } + if($this->segments->dbExists($dbname) && $this->fieldIndex->isFieldIndexed($dbname, $field)){ + $this->dropFieldIndexUnlocked($dbname, $field); // creating it again rebuilds it + } + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + } + $this->db->createDBUnlocked($dbname); + } + + $indexed = 0; + $segments = $this->segments->listSegments($dbname); + foreach($segments as $segment){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $segment); // leftovers of an earlier index of this field + $index = $this->fieldIndex->buildSegmentFieldIndex($dbname, $segment, $field); + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $segment); + foreach($index['b'] as $keys){ + $indexed += is_int($keys) ? 1 : count($keys); + } + unset($index); + $this->cache->evictBetweenSegments(); + } + if($this->segments->isSharded($dbname)){ + $this->fieldIndex->deleteGlobalFieldIndex($dbname, $field); + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + } + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['fields'][] = $field; + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + + $result = ['success' => true, 'indexed' => $indexed, 'error' => null]; + if($this->segments->isSharded($dbname)){ + $result['shards'] = count($segments); + } + return $result; + } + + /** + * Drop a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => string|null] + */ + public function dropFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropFieldIndexUnlocked($dbname, $field); + }, 'dropFieldIndex'); + } + + private function dropFieldIndexUnlocked($dbname, $field){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(!in_array((string)$field, $manifest['fields'], true)){ + return ['success' => false, 'error' => 'Index does not exist']; + } + $segments = $this->segments->listSegments($dbname); + foreach($segments as $segment){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $segment); + } + $this->fieldIndex->deleteGlobalFieldIndex($dbname, $field); + $manifest['fields'] = array_values(array_diff($manifest['fields'], [(string)$field])); + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + $this->fieldIndex->invalidateFieldIndexCache($dbname); + return ['success' => true, 'error' => null]; + } + + /** + * Check if a field has an index + * @param string $dbname Database name + * @param string $field Field name + * @return bool + */ + public function hasFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->segments->dbExists($dbname) && $this->fieldIndex->isFieldIndexed($dbname, $field); + }); + } + + /** + * Get list of field indexes for a database + * @param string $dbname Database name + * @return array ['fields' => array, 'sharded' => bool] + */ + public function getFieldIndexes($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getFieldIndexesUnlocked($dbname); + }); + } + + private function getFieldIndexesUnlocked($dbname){ + if(!$this->segments->dbExists($dbname)){ + return ['fields' => [], 'sharded' => false]; + } + return ['fields' => $this->fieldIndex->getIndexedFields($dbname), 'sharded' => $this->segments->isSharded($dbname)]; + } + + /** + * Rebuild a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createFieldIndex + */ + public function rebuildFieldIndex($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildFieldIndexUnlocked($dbname, $field); + }, 'rebuildFieldIndex'); + } + + private function rebuildFieldIndexUnlocked($dbname, $field){ + if($this->fieldIndex->isFieldIndexed($dbname, $field)){ + $this->dropFieldIndexUnlocked($dbname, $field); + } + return $this->createFieldIndexUnlocked($dbname, $field); + } + + /** + * Get keys matching a field value using field index + * Returns null if no index exists (caller should fall back to scan) + * @param string $dbname Database name + * @param string $field Field name + * @param mixed $value Value to match + * @param int|null $shardId Shard ID for sharded databases (null = all shards) + * @return array|null Array of matching keys, or null if no index + */ + public function getKeysFromFieldIndex($dbname, $field, $value, $shardId = null){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $value, $shardId){ + return $this->getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId); + }); + } + + private function getKeysFromFieldIndexUnlocked($dbname, $field, $value, $shardId = null){ + if(!$this->segments->dbExists($dbname) || !$this->fieldIndex->isFieldIndexed($dbname, $field) || !$this->fieldIndex->isIndexableValue($value)){ + return null; + } + $segments = $shardId !== null ? [(int)$shardId] : $this->segments->listSegments($dbname); + $keys = []; + foreach($segments as $segment){ + if(!in_array($segment, $this->segments->listSegments($dbname), true)){ + continue; + } + $index = $this->fieldIndex->ensureSegmentFieldIndex($dbname, $segment, $field); + $found = $index === null ? null : $this->fieldIndex->fieldIndexKeys($index, $value); + if($found === null){ + // No usable index: the records decide, exactly + foreach($this->records->eachSegmentBatch($dbname, $segment) as $batch){ + foreach($batch as $key => $record){ + if(array_key_exists($field, $record) && $record[$field] === $value){ + $keys[] = (int)$key; + } + } + } + $this->cache->evictBetweenSegments(); + continue; + } + foreach($found as $key){ + $keys[] = (int)$key; + } + $this->cache->evictBetweenSegments(); + } + sort($keys); + return $keys; + } +} diff --git a/src/Internal/Index/PagedFile.php b/src/Internal/Index/PagedFile.php new file mode 100644 index 0000000..906c708 --- /dev/null +++ b/src/Internal/Index/PagedFile.php @@ -0,0 +1,289 @@ + $at; $end = strrpos($json, ',"', $end - 1 - $n)){ + if($json[$end + 2] !== ':'){ + return $end; + } + } + for($end = strpos($json, ',"', $at + 1); $end !== false; $end = strpos($json, ',"', $end + 1)){ + if($json[$end + 2] !== ':'){ + return $end; + } + } + return $n; + } + + /** The id of the member starting at $at in the encoded members $json: up to the first quote no backslash escapes */ + private static function idAt(string $json, int $at): string { + $close = $at + 1; + while($json[$close += strcspn($json, '"\\', $close)] === '\\'){ + $close += 2; + } + return json_decode(substr($json, $at, $close + 1 - $at)); + } + + /** + * Build the body from a segment's buckets in bucket id order (id => key | [keys], ascending), sealed as data + * pages, then the directory levels: [body, pg]. A member is `"id":key` for a bucket of one key, else `"id":[keys]`; + * all of them are encoded at once and cut into pages, members not split, each page as full as it gets. + */ + public static function build(array $buckets): array { + $json = substr((string)json_encode($buckets, self::JSON), 1, -1); + $body = ''; + $level = []; + for($at = 0, $n = strlen($json); $at < $n; $at = $end + 1){ + $end = self::pageEnd($json, $at); + $page = '{' . substr($json, $at, $end - $at) . '}'; + $page .= pack('V', crc32($page)); + $level[] = [self::idAt($json, $at), strlen($body), strlen($page)]; + $body .= $page; + } + if(empty($level)){ + $body = '{}' . pack('V', crc32('{}')); + $level[] = ['', 0, strlen($body)]; + } + $data = strlen($body); + $pages = count($level); + $levels = 0; + while(count($level) > 1){ + $levels++; + $next = []; + $current = []; + $bytes = 0; + $first = ''; + foreach($level as [$id, $offset, $length]){ + $entry = pack('V', strlen($id)) . $id . pack('V2', $offset, $length); + // At least two entries per directory page (long ids): each level has fewer pages + if(count($current) >= 2 && !self::fits($bytes, count($current), $entry)){ + $page = self::seal($current); + $next[] = [$first, strlen($body), strlen($page)]; + $body .= $page; + $current = []; + $bytes = 0; + } + if(empty($current)){ + $first = $id; + } + $current[] = $entry; + $bytes += strlen($entry); + } + $page = self::seal($current); + $next[] = [$first, strlen($body), strlen($page)]; + $body .= $page; + $pages += count($next); + $level = $next; + } + return [$body, ['root' => [$level[0][1], $level[0][2]], 'levels' => $levels, 'pages' => $pages, 'data' => $data]]; + } + + /** True when a header's `pg` can name a tree: a root of some bytes, no more levels than two entries a page allow */ + public static function validTree($pg): bool { + return is_array($pg) && is_int($pg['root'][0] ?? null) && is_int($pg['root'][1] ?? null) && $pg['root'][1] > 0 + && is_int($pg['levels'] ?? null) && $pg['levels'] >= 0 && $pg['levels'] < 64; + } + + /** A page without its crc (entries and table), or null when the crc doesn't match */ + public static function open($bytes): ?string { + if(!is_string($bytes) || strlen($bytes) < 6){ + return null; + } + return crc32($bytes) === \noneDB\Internal\Storage\KeyIndex::CRC_RESIDUE ? substr($bytes, 0, -4) : null; + } + + /** Where a page's table starts (= the bytes of its entries), or null when the count doesn't fit the page */ + private static function tableAt(string $page): ?int { + $len = strlen($page); + $at = $len - 2 - 4 * (ord($page[$len - 2]) | (ord($page[$len - 1]) << 8)); + return $at < 0 ? null : $at; + } + + /** The child of a decoded directory page that can hold $id (the last whose first id is <= $id): [offset, length]; [] when none */ + public static function child(array $children, string $id): array { + $lo = 0; + $hi = count($children) - 1; + $found = -1; + while($lo <= $hi){ + $mid = ($lo + $hi) >> 1; + if(strcmp($children[$mid][0], $id) <= 0){ + $found = $mid; + $lo = $mid + 1; + } else { + $hi = $mid - 1; + } + } + return $found < 0 ? [] : [$children[$found][1], $children[$found][2]]; + } + + /** True when a page's table names exactly the starts of the entries a walk found ($starts) */ + private static function checkedStarts(string $page, array $starts, int $table): bool { + if(count($starts) !== intdiv(strlen($page) - 2 - $table, 4)){ + return false; + } + return $starts === [] || array_values(unpack('V' . count($starts), $page, $table)) === $starts; + } + + /** A directory page: [[first id, offset, length], ...]; null when its table doesn't name its entries */ + public static function directory(string $page): ?array { + $table = self::tableAt($page); + if($table === null){ + return null; + } + $children = []; + $starts = []; + for($o = 0; $o < $table;){ + $starts[] = $o; + if($o + 12 > $table || $o + 12 + ($n = unpack('V', $page, $o)[1]) > $table){ + return null; // an entry that leaves the page: its crc fits, its content doesn't + } + $child = unpack('V2', $page, $o + 4 + $n); + $children[] = [substr($page, $o + 4, $n), $child[1], $child[2]]; + $o += 12 + $n; + } + return self::checkedStarts($page, $starts, $table) ? $children : null; + } + + /** + * The keys of the member named $id in an .fx data page (its JSON object, without the crc), ascending; + * [] when the page doesn't hold it; null when its value isn't a key or a list of keys. A byte search in + * canonical JSON (§6.5.1 lookup step 3), for the name without its opening quote: a type letter stops a + * byte search less often than a quote, which every name starts and ends with. A match preceded by `{"` or + * `,"` is a member's name (inside a name a quote is escaped, a value holds none), and it names exactly $id. + */ + public static function fxMember(string $page, string $id, int $start = 0, ?int $end = null): ?array { + // The name as the page writes it (the last one kept: a query looks one value up in segment after segment) + static $lastId = null, $lastTail = null; + if($id !== $lastId){ + $name = json_encode($id, self::JSON); + if($name === false){ + return []; // not a name any member can have + } + $lastId = $id; + $lastTail = substr($name, 1) . ':'; + } + $tail = $lastTail; + // $start / $end: the page's bytes inside a whole body, copied: a search never runs past the page + if($start > 0 || ($end !== null && $end < strlen($page))){ + $page = substr($page, $start, ($end ?? strlen($page)) - $start); + } + for($at = strpos($page, $tail, 2); $at !== false; $at = strpos($page, $tail, $at + 1)){ + if($page[$at - 1] === '"' && ($page[$at - 2] === ',' || $page[$at - 2] === '{')){ + return self::fxKeysAt($page, $at + strlen($tail)); + } + } + return []; + } + + /** The keys of the .fx member whose value starts at $at; null when it isn't a key or a list of keys */ + private static function fxKeysAt(string $page, int $at): ?array { + if(($page[$at] ?? '') === '['){ + $close = strpos($page, ']', $at); + $length = $close === false ? 0 : $close + 1 - $at; + } else { + $length = strcspn($page, ',}', $at); + } + $keys = $length > 0 ? json_decode(substr($page, $at, $length), true) : null; + if(is_int($keys)){ + return [$keys]; + } + return is_array($keys) ? $keys : null; + } + + /** + * Add the members of an .fx data page (its JSON object, without the crc) to $map in the shape of a + * loaded index: id => key (one key) or [keys] (ascending). False when the page isn't a JSON object in + * canonical form (encoding it again doesn't give its bytes): its byte searches couldn't be trusted. + */ + public static function fxMembersInto(string $page, array &$map): bool { + if($page === '{}'){ + return true; + } + $members = json_decode($page, true); + if(!is_array($members) || json_encode($members, self::JSON) !== $page){ + return false; + } + $map += $members; + return true; + } + + /** + * Every data page of a body, in order, from its directory: [[offset, length], ...]; null when a page's + * crc or table doesn't match or the pages don't cover the data bytes + */ + public static function dataPages(string $body, array $pg): ?array { + if(!self::validTree($pg)){ + return null; + } + // Depth first, children in order, with a stack (a recursive closure would hold the body in a cycle + // until the garbage collector runs) + $pages = []; + $stack = [[$pg['root'][0], $pg['root'][1], $pg['levels']]]; + while(!empty($stack)){ + [$offset, $length, $level] = array_pop($stack); + if($level === 0){ + $pages[] = [$offset, $length]; + continue; + } + $data = self::open(substr($body, $offset, $length)); + $children = $data === null ? null : self::directory($data); + if($children === null){ + return null; + } + for($i = count($children) - 1; $i >= 0; $i--){ + $stack[] = [$children[$i][1], $children[$i][2], $level - 1]; + } + } + $expected = 0; + foreach($pages as [$offset, $length]){ + if($offset !== $expected){ + return null; + } + $expected += $length; + } + return $expected === ($pg['data'] ?? -1) ? $pages : null; + } +} diff --git a/src/Internal/Index/SecondaryFiles.php b/src/Internal/Index/SecondaryFiles.php new file mode 100644 index 0000000..1c21d33 --- /dev/null +++ b/src/Internal/Index/SecondaryFiles.php @@ -0,0 +1,437 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->genPaths = &$this->k->genPaths; + $this->indexCache = &$this->k->indexCache; + $this->lockPaths = &$this->k->lockPaths; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->indexTokens = &$this->rt->indexTokens; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + $this->secondaryChecked = &$this->rt->secondaryChecked; + $this->tailCache = &$this->rt->tailCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * File name part of an index on $field (at most 40 bytes) + */ + public function indexToken($field){ + $field = (string)$field; + if(isset($this->indexTokens[$field])){ + return $this->indexTokens[$field]; + } + $token = preg_match('/^[a-z0-9_]{1,40}$/D', $field) === 1 ? $field + : substr(strtolower(preg_replace('/[^A-Za-z0-9_]/', '_', $field)), 0, 24) . '-' . substr(hash('sha256', $field), 0, 12); + if(count($this->indexTokens) > 4096){ + $this->indexTokens = []; + } + return $this->indexTokens[$field] = $token; + } + + /** + * Path of a secondary index file: 'fx' / 'rx' per segment, 'gfx' / 'grx' per database + */ + public function secondaryPath($dbname, $kind, $field, $shardId = null){ + $k = $shardId ?? -1; + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['x'][$kind][$field][$k]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['x'][$kind][$field][$k]; + } + $dir = $this->layout->genDir($dbname); + if(isset($this->genPaths[$dbname]['x'][$kind][$field][$k])){ + return $this->genPaths[$dbname]['x'][$kind][$field][$k]; + } + $dir .= ($kind === 'fx' || $kind === 'gfx' ? 'indexes/' : 'spatial/'); + $path = $kind === 'gfx' || $kind === 'grx' ? $dir . $this->indexToken($field) . '.' . $kind + : $dir . \noneDB\Internal\Storage\Layout::segmentName($shardId) . '.' . $this->indexToken($field) . '.' . $kind; + return $this->genPaths[$dbname]['x'][$kind][$field][$k] = $path; + } + + /** + * The key index of a segment for a secondary index check: the cached one when it is known to cover + * the data file (no function call chain on the hot path), else readJsonlIndex() + */ + public function currentKeyIndex($dbname, $shardId){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + if(isset($this->verifiedIndexes[$path], $this->indexCache[$path])){ + $this->metrics['hits']++; + return $this->indexCache[$path]; + } + return $this->checkpoints->readJsonlIndex($dbname, $shardId); + } + + /** + * [header, body] of a secondary index file; null when it is missing or its header is unreadable + */ + public function readSecondaryFile($path){ + clearstatcache(true, $path); + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + $body = $line === false ? '' : (string)stream_get_contents($fp); + fclose($fp); + $this->cache->noteIndexRead(($line === false ? 0 : strlen($line)) + strlen($body)); + $header = $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + return is_array($header) ? [$header, $body] : null; + } + + /** + * Only the header line of a secondary index file (null: missing or unreadable) + */ + public function readSecondaryHeader($path){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $line = fgets($fp); + fclose($fp); + $this->cache->noteIndexRead($line === false ? 0 : strlen($line)); + return $line === false ? null : \noneDB\Internal\Storage\KeyIndex::openHeader($line); + } + + /** + * Write a secondary index file (header line + body). A checkpoint: no generation bump, its content is + * the same whoever writes it. + * @throws \RuntimeException + */ + public function writeSecondaryFile($dbname, $path, array $header, $body){ + $line = $this->io->encodeHeader($header); + if($line === false || !$this->io->writeDerivedFile($path, $line . "\n" . $body)){ + throw new \RuntimeException("noneDB: failed to write index " . basename($path) . " of database '{$dbname}'"); + } + $this->cache->markFileExists($path); + } + + /** + * The segment's key index, with an epoch: an index without one (rebuilt without a checkpoint) gets its + * checkpoint written first (exclusive lock), since secondary indexes name the epoch they belong to + * @return array|null + */ + public function keyIndexWithEpoch($dbname, $shardId){ + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] !== null){ + return $index; + } + $lock = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_EX){ + return $index; + } + $this->checkpoints->writeCheckpoint($dbname, $shardId, $index); + return $this->checkpoints->readJsonlIndex($dbname, $shardId); + } + + /** + * True when a secondary index header belongs to the segment as it is: same field, the epoch of its + * key index, and the bytes it covers unchanged (checked once per epoch and offset) + */ + public function secondaryFits($dbname, $shardId, array $header, $kind, $field){ + if(($header['kind'] ?? null) !== $kind || ($header['field'] ?? null) !== (string)$field + || !isset($header['s'], $header['e'], $header['h']) || !is_int($header['s']) || $header['s'] < 0 || !is_string($header['e']) || !is_string($header['h'])){ + return false; + } + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index === null || $index['e'] === null || $header['e'] !== $index['e'] || $header['s'] > (int)$index['s']){ + return false; + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $checked = $index['e'] . ':' . $header['s'] . ':' . $header['h']; // the checksum too: another file may cover the same bytes + if(($this->secondaryChecked[$dataPath] ?? null) === $checked){ + return true; + } + // The bytes the key index checkpoint covers had their crc checked when it was loaded or written + $known = $header['s'] === (int)$index['ck'] ? ($index['ckh'] ?? null) : null; + if(($known ?? $this->checkpoints->segmentCrc($dataPath, $header['s'])) !== $header['h']){ + return false; + } + $this->secondaryChecked[$dataPath] = $checked; + return true; + } + + /** + * Records of the complete data lines [$from, $to) of a segment, in file order: [[key, record or null + * for a tombstone], ...]. Lines that are neither are skipped (salvaged ones count, as in the key index). + */ + public function segmentTail($dbname, $shardId, $from, $to){ + if($to <= $from){ + return []; + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $cached = $this->tailCache[$dataPath] ?? null; + if($cached !== null && $cached[0] === $from && $cached[1] === $to){ + return $cached[2]; + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return []; + } + fseek($fp, $from); + $bytes = ''; + while(strlen($bytes) < $to - $from){ + $chunk = fread($fp, min(1048576, $to - $from - strlen($bytes))); + if($chunk === false || $chunk === ''){ + break; + } + $bytes .= $chunk; + } + fclose($fp); + $base = $shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->segments->shardSizeOf($dbname); + $rows = []; + $pos = 0; + $empty = []; + while(($newline = strpos($bytes, "\n", $pos)) !== false){ + $json = substr($bytes, $pos, $newline - $pos); + $pos = $newline + 1; + $record = json_decode($json, true); + if(is_array($record) && count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $rows[] = [$record['__nonedb_deleted'], null]; + continue; + } + if(is_array($record) && isset($record['key']) && is_int($record['key'])){ + $rows[] = [$record['key'], $record]; + continue; + } + $entry = $this->checkpoints->salvageLine($json, $from + $pos - strlen($json) - 1, $empty, $base, $limit); + if($entry !== null){ + $rows[] = $entry[0] === 't' ? [$entry[1], null] : [$entry[1], json_decode(substr($json, $entry[2], $entry[3]), true)]; + } + } + if(count($this->tailCache) > 64){ + $this->tailCache = []; + } + $this->tailCache[$dataPath] = [$from, $to, $rows]; + return $rows; + } + + /** + * A global index (gfx / grx) about to be rewritten, which stamps it with the current lock epoch: when + * it was written under another epoch (a data file may have been replaced since), only the entries that + * still fit their shard are kept, so the new stamp vouches for no stale entry + */ + public function freshGlobalEntries($dbname, array $global){ + $lockPath = $this->locks->getLockPath($dbname); + if($global['le'] !== null && isset($this->lockRegistry[$lockPath]) && $global['le'] === $this->locks->registryEpoch($lockPath)){ + return $global; + } + foreach($global['sh'] as $shardId => $entry){ + if($this->fieldIndex->globalEntryFits($dbname, $shardId, $entry, null)){ + continue; + } + unset($global['sh'][$shardId], $global['cols'][$shardId]); + } + return $global; + } + + /** + * True when the tail of a secondary index (data after the bytes it covers) is long enough to write a + * new checkpoint of it + */ + public function secondaryTailIsLong($covered, $end, $fileBytes){ + return $end - $covered >= max(self::SECONDARY_TAIL_BYTES, intdiv((int)$fileBytes, 4)); + } + + /** + * True for an entry of a global index: it names the bytes of a shard's segment index that it describes (§6.5) + */ + public static function isEntry($entry){ + return is_int($entry['s'] ?? null) && $entry['s'] >= 0 && is_string($entry['e'] ?? null) && is_string($entry['h'] ?? null); + } + + /** + * A reader that loaded a secondary index and replayed a long tail writes a new checkpoint of it (and of + * its shard's global entry), so the next process doesn't replay the same lines again. Under the shared + * lock only: never inside a write session (whose appends a rollback may still cut), not for + * checkIntegrity(), explain() or a snapshot. Best effort, no generation bump: the content is a function + * of the data. + * @return bool True when one is due but was put off (checkIntegrity(), explain(), a join): the caller + * marks the cached index ("due") and the next plain read of this process writes it + */ + public function maybeSecondaryReaderCheckpoint($dbname, $shardId, $kind, $field, array $index){ + $lock = $this->locks->getLockPath($dbname); + if($index['e'] === null + || !$this->secondaryTailIsLong((int)$index['ck'], (int)$index['s'], (int)$index['bytes'])){ + return false; + } + if($this->noWrites > 0 || $this->multiLock > 0){ + return true; // explain(), checkIntegrity(), a join: due, the next plain read of this process writes it + } + if(!isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_SH){ + return false; // a write session: checkpointSegmentSecondaries() at its end + } + try { + if($kind === 'fx'){ + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->fieldIndex->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } else { + $this->rtree->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: the next load replays the lines again + } + return false; + } + + /** + * End of a write session: write new checkpoints of the field and spatial indexes of a changed segment + * that this process has loaded and whose tail is long, and their global shard summaries (exclusive + * lock, best effort). An index nobody here uses isn't loaded for it: a writer that only writes keeps + * no secondary index in memory, and the next reader that loads it writes the checkpoint. + */ + public function checkpointSegmentSecondaries($dbname, $shardId){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; + } + $key = $this->keyIndexWithEpoch($dbname, $shardId); + if($key === null || $key['e'] === null){ + return; + } + foreach(['fx' => $manifest['fields'], 'rx' => $manifest['spatial']] as $kind => $fields){ + foreach($fields as $field){ + $path = $this->secondaryPath($dbname, $kind, $field, $shardId); + $cached = $kind === 'fx' ? ($this->fieldIndexCache[$path] ?? null) : ($this->spatialIndexCache[$path] ?? null); + if($cached === null && $kind === 'fx' && isset($this->fieldIndexCache[$path . '#pg'])){ + // used here through point lookups (pages of its file): in use like a loaded one + $paged = $this->fieldIndexCache[$path . '#pg']; + $cached = ['e' => $paged['h']['e'] ?? null, 'ck' => (int)$paged['h']['s'], 'bytes' => $paged['size']]; + } + if($cached === null || $cached['e'] !== $key['e']){ + continue; // not used here (or of another epoch): the reader that uses it checkpoints it + } + $covered = (int)$cached['ck']; + $bytes = (int)$cached['bytes']; + if(!$this->secondaryTailIsLong($covered, (int)$key['s'], $bytes)){ + continue; + } + try { + if($kind === 'fx'){ + $index = $this->fieldIndex->ensureSegmentFieldIndex($dbname, $shardId, $field); + if($index !== null){ + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->fieldIndex->updateGlobalFieldShard($dbname, $field, $shardId, $index); + } + } + } else { + $index = $this->rtree->ensureSegmentSpatialIndex($dbname, $field, $shardId); + $this->rtree->writeSpatialIndex($dbname, $field, $index, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $index); + } + } + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + // never needed for correctness: the next load replays a longer tail + } + } + } + } + // ========================================== + // SECONDARY INDEX FILES + // ========================================== + // A field index (.fx), an R-tree (.rx) and the global shard summaries (.gfx, .grx) are checkpoints + // of their segment's data, like the key index: line 1 of the file is a header with the field, the + // data bytes "s" it covers, the crc32b "h" of the bytes before s and the epoch "e" of the key index it + // was made with; the body follows. An index is a function of the data alone: loading it replays the + // data lines after s (in memory), so writes never touch it, and a crash, a rollback or another + // process's writes can't leave it stale. Once its tail reached 64 KiB (or a quarter of the index file), + // the process that uses it writes a new checkpoint: a reader that loaded it, or a write session that + // has it loaded (a process that only writes never loads it). An index whose epoch or bytes don't fit + // the segment is rebuilt from the data (exclusive lock). + // Names (in the active generation): indexes/{seg}.{token}.fx, indexes/{token}.gfx, + // spatial/{seg}.{token}.rx, spatial/{token}.grx; the token is the field when it is [a-z0-9_]{1,40}, + // else a lower-case prefix and 12 hex digits of its sha256: at most 40 bytes, and fields that differ + // only in case ("city", "City") never share a file. + + public const SECONDARY_TAIL_BYTES = 65536; +} diff --git a/src/Internal/Index/ShardFilters.php b/src/Internal/Index/ShardFilters.php new file mode 100644 index 0000000..33a7342 --- /dev/null +++ b/src/Internal/Index/ShardFilters.php @@ -0,0 +1,168 @@ + its eight rows, one byte each ($bit or 0) */ + private static $columnBytes = []; + + /** + * The six probe rows of a bucket id in filters of $bits bits: from its xxh3, x the first four bytes and y the + * last four (lowest bit set), big-endian; row (x + i·y) mod 2³² mod $bits + * @return int[] + */ + public static function probes(string $id, int $bits): array { + $d = unpack('N2', hash('xxh3', $id, true)); + $y = $d[2] | 1; + $rows = []; + for($i = 0; $i < 6; $i++){ + $rows[] = (($d[1] + $i * $y) & 0xFFFFFFFF) & ($bits - 1); + } + return $rows; + } + + /** + * The filter of a shard whose field index holds these bucket ids: $bits bits (bit r: byte r >> 3, value 1 << (r & 7)). + * The probes of probes(), set in 64-bit words (bit r: word r >> 6, bit r & 63), packed little-endian. + */ + public static function filter(array $ids, int $bits): string { + $words = array_fill(0, $bits >> 6, 0); + $mask = ($bits - 1) & 0xFFFFFFFF; // mod 2³² mod $bits (a power of two) + foreach($ids as $id){ + $d = unpack('N2', hash('xxh3', (string)$id, true)); + $y = $d[2] | 1; + for($i = 0, $v = $d[1]; $i < 6; $i++, $v += $y){ + $r = $v & $mask; + $words[$r >> 6] |= 1 << ($r & 63); + } + } + $filter = ''; + foreach(array_chunk($words, 4096) as $chunk){ + $filter .= pack('P*', ...$chunk); + } + return $filter; + } + + /** + * Rows [$first, $first + n) of $rowBytes bytes each (a page: n a power of two, $first a multiple of it), with + * $shard's column taken from its filter: the shard's bit of row r (byte $shard >> 3, value 1 << ($shard & 7)) + * becomes bit $first + r of the filter; every other bit stays + */ + public static function withColumn(string $rows, int $first, int $rowBytes, int $shard, string $filter): string { + $n = intdiv(strlen($rows), $rowBytes); + $bit = 1 << ($shard & 7); + if(!isset(self::$columnBytes[$bit])){ + for($b = 0; $b < 256; $b++){ + $eight = ''; + for($k = 0; $k < 8; $k++){ + $eight .= ($b >> $k) & 1 ? chr($bit) : "\0"; + } + self::$columnBytes[$bit][chr($b)] = $eight; + } + } + // The filter's bits of these rows, one byte per row; then each at its place in its row + $column = substr(strtr(substr($filter, $first >> 3, ($n + 7) >> 3), self::$columnBytes[$bit]), $first & 7, $n); + $at = $shard >> 3; + $column = substr(str_repeat("\0", $at) . chunk_split($column, 1, str_repeat("\0", $rowBytes - 1)), 0, $n * $rowBytes); + $keep = str_repeat(str_repeat("\xFF", $at) . chr(0xFF ^ $bit) . str_repeat("\xFF", $rowBytes - $at - 1), $n); + return ($rows & $keep) | $column; + } + + /** True when a filter has every probe bit (the shard may hold the bucket id) */ + public static function mayHold(string $filter, array $probes): bool { + foreach($probes as $r){ + if((ord($filter[$r >> 3]) & (1 << ($r & 7))) === 0){ + return false; + } + } + return true; + } + + /** The smallest power of two of at least max($min, $n): a number of bits or the width */ + public static function size(int $n, int $min): int { + $size = $min; + while($size < $n){ + $size <<= 1; + } + return $size; + } + + /** Rows per page for rows of $width bits (a page holds about 4096 bytes of rows) */ + public static function pageRows(int $bits, int $width): int { + return min($bits, max(1, intdiv(4096, $width >> 3))); + } + + /** A page as it is stored: its rows, then their crc32 (u32 little-endian) */ + public static function seal(string $rows): string { + return $rows . pack('V', crc32($rows)); + } + + /** The rows of a stored page of $length bytes of rows; null when it is short or its crc doesn't match */ + public static function open(string $stored, int $length): ?string { + if(strlen($stored) !== $length + 4){ + return null; + } + $rows = substr($stored, 0, $length); + return unpack('V', $stored, $length)[1] === crc32($rows) ? $rows : null; + } + + /** + * The shards (bit positions) set in every row: the AND of the rows, its set bits + * @param string[] $rows + * @return int[] ascending + */ + public static function holders(array $rows): array { + $and = null; + foreach($rows as $row){ + $and = $and === null ? $row : $and & $row; + } + return self::setBits($and); + } + + /** + * The shards that may hold a bucket id, from a file's body in memory: the rows of probes() (x + i·y, the mod 2³² of + * a power of two is its mask) read in place, the AND given up at the first row that leaves no shard. A warm lookup. + * @param string $body The stored pages + * @param int $rb Bytes of a row + * @return int[] ascending + */ + public static function holdersIn(string $body, int $bits, int $rb, string $id): array { + $d = unpack('N2', hash('xxh3', $id, true)); + $x = $d[1]; + $y = $d[2] | 1; + $mask = $bits - 1; + $per = self::pageRows($bits, $rb << 3); + $stored = $per * $rb + 4; + $and = null; + for($i = 0; $i < 6; $i++, $x += $y){ + $r = $x & $mask; + $row = substr($body, intdiv($r, $per) * $stored + ($r % $per) * $rb, $rb); + $and = $and === null ? $row : $and & $row; + if(strspn($and, "\0") === $rb){ + return []; + } + } + return self::setBits($and); + } + + /** @return int[] The bit positions set in a row, ascending */ + private static function setBits(string $and): array { + $shards = []; + $length = strlen($and); + for($o = strspn($and, "\0"); $o < $length; $o += 1 + strspn($and, "\0", $o + 1)){ + $byte = ord($and[$o]); + for($bit = 0; $bit < 8; $bit++){ + if($byte & (1 << $bit)){ + $shards[] = 8 * $o + $bit; + } + } + } + return $shards; + } +} diff --git a/src/Internal/Index/Spatial/GeoJson.php b/src/Internal/Index/Spatial/GeoJson.php new file mode 100644 index 0000000..efa3f55 --- /dev/null +++ b/src/Internal/Index/Spatial/GeoJson.php @@ -0,0 +1,875 @@ +k = $k; + $this->rt = $k->rt; + unset($this->geometry); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->geometryValidations = &$this->rt->geometryValidations; // @test + $this->pointEdgeTests = &$this->rt->pointEdgeTests; // @test + $this->segmentPairTests = &$this->rt->segmentPairTests; // @test + $this->sweepBudget = &$this->rt->sweepBudget; // @test + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->geometry = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL INDEX - GEOJSON SUPPORT + // ========================================== + + /** + * Validate a GeoJSON geometry object + * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection + * + * @param array $geometry GeoJSON geometry object + * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] + */ + public function validateGeoJSON($geometry): array { + $this->geometryValidations++; // @test + // Never throws: a stored record that fails here is simply not indexed (a malformed geometry must + // never make later operations on the database fail) + if (!is_array($geometry)) { + return ['valid' => false, 'error' => 'Geometry must be an array', 'type' => null]; + } + if (!isset($geometry['type'])) { + return ['valid' => false, 'error' => 'Missing type property', 'type' => null]; + } + + $type = $geometry['type']; + if (!is_string($type)) { + return ['valid' => false, 'error' => 'Geometry type must be a string', 'type' => null]; + } + + // GeometryCollection has different structure + if ($type === 'GeometryCollection') { + return $this->validateGeometryCollection($geometry); + } + + if (!isset($geometry['coordinates'])) { + return ['valid' => false, 'error' => 'Missing coordinates property', 'type' => null]; + } + + $coords = $geometry['coordinates']; + + // Each validator checks the lists it needs (a Point needs none: values at 0 and 1) + if (!is_array($coords)) { + return ['valid' => false, 'error' => 'Coordinates must be an array', 'type' => null]; + } + + // Strict comparison (a switch compared loosely: 0 matched 'Point' before PHP 8) + if ($type === 'Point') { + return $this->validateGeoJSONPoint($coords); + } + if ($type === 'LineString') { + return $this->validateGeoJSONLineString($coords); + } + if ($type === 'Polygon') { + return $this->validateGeoJSONPolygon($coords); + } + if ($type === 'MultiPoint') { + return $this->validateGeoJSONMultiPoint($coords); + } + if ($type === 'MultiLineString') { + return $this->validateGeoJSONMultiLineString($coords); + } + if ($type === 'MultiPolygon') { + return $this->validateGeoJSONMultiPolygon($coords); + } + return ['valid' => false, 'error' => "Unknown geometry type: $type", 'type' => null]; + } + + /** + * An array with keys 0..n-1 in order (a JSON array) + */ + private static function isJsonList($value): bool { + return is_array($value) && array_is_list($value); + } + + /** + * Validate Point coordinates [lon, lat] or [lon, lat, altitude] + */ + private function validateGeoJSONPoint($coords): array { + // A position: values at 0 and 1 (checked below) are enough, no list check per position + if (!is_array($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'Point must have at least 2 coordinates', 'type' => null]; + } + if (!isset($coords[0], $coords[1]) || !is_numeric($coords[0]) || !is_numeric($coords[1]) + || is_string($coords[0]) || is_string($coords[1]) || !is_finite((float)$coords[0]) || !is_finite((float)$coords[1])) { + return ['valid' => false, 'error' => 'Point coordinates must be finite numbers', 'type' => null]; + } + $lon = (float)$coords[0]; + $lat = (float)$coords[1]; + if ($lon < -180 || $lon > 180) { + return ['valid' => false, 'error' => 'Longitude must be between -180 and 180', 'type' => null]; + } + if ($lat < -90 || $lat > 90) { + return ['valid' => false, 'error' => 'Latitude must be between -90 and 90', 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Point']; + } + + /** + * Validate LineString coordinates [[lon,lat], [lon,lat], ...] + */ + private function validateGeoJSONLineString($coords): array { + if (!self::isJsonList($coords) || count($coords) < 2) { + return ['valid' => false, 'error' => 'LineString must have at least 2 positions', 'type' => null]; + } + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid position at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'LineString']; + } + + /** + * Validate Polygon coordinates [[[lon,lat], ...], ...] + * First ring is outer boundary, subsequent rings are holes + */ + private function validateGeoJSONPolygon($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'Polygon must have at least one ring', 'type' => null]; + } + // Rings in order. The holes before the first failing ring are checked against the outer ring + // together (one sweep for all their vertices); the first error is the same as ring by ring. + $failure = null; + $holes = []; + foreach ($coords as $ringIndex => $ring) { + if (!self::isJsonList($ring) || count($ring) < 4) { + $failure = "Ring $ringIndex must have at least 4 positions"; + break; + } + // Validate each position + foreach ($ring as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + $failure = "Invalid position at ring $ringIndex, index $i: " . $result['error']; + break 2; + } + } + // Check if ring is closed (first == last, compared numerically: 29 and 29.0 are the same) + $first = $ring[0]; + $last = $ring[count($ring) - 1]; + if ((float)$first[0] != (float)$last[0] || (float)$first[1] != (float)$last[1]) { + $failure = "Ring $ringIndex must be closed (first and last positions must be identical)"; + break; + } + if ($this->ringSelfIntersects($ring)) { + $failure = "Ring $ringIndex must not intersect itself"; + break; + } + if ($ringIndex > 0) { + $holes[$ringIndex] = $ring; + } + } + if ($holes) { + // Every vertex of a hole inside the outer ring or on it + $points = []; + $owner = []; + foreach ($holes as $ringIndex => $ring) { + foreach ($ring as $position) { + $points[] = $position; + $owner[] = $ringIndex; + } + } + foreach ($this->locatePointsInRing($coords[0], $points) as $k => $where) { + if ($where < 0) { + return ['valid' => false, 'error' => "Hole {$owner[$k]} must lie inside the outer ring", 'type' => null]; + } + } + } + if ($failure !== null) { + return ['valid' => false, 'error' => $failure, 'type' => null]; + } + return ['valid' => true, 'error' => null, 'type' => 'Polygon']; + } + + /** + * True when two non-adjacent edges of a closed ring share a point (they cross, touch or overlap; + * exact arithmetic). First a sort and sweep by x that compares the edges whose x ranges overlap: + * fast for usual rings. When that work + * passes 16 edge pairs per edge (rings built against it, e.g. a comb whose edges all overlap in x), + * a sweep line over a balanced tree finishes the check: O(n log n) for every ring. + */ + private function ringSelfIntersects(array $ring): bool { + $n = count($ring) - 1; // edges; edge i runs from vertex i to vertex i + 1 + if ($n < 4) { + return false; // every pair of a triangle's edges is adjacent + } + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; + } + $budget = 16 * $n + 256; // @prod + $budget = $this->sweepBudget ?? 16 * $n + 256; // @test + if ($budget === 0) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + $minX = []; + $maxX = []; + for ($i = 0; $i < $n; $i++) { + $a = $xs[$i]; + $b = $xs[$i + 1]; + $minX[$i] = $a < $b ? $a : $b; + $maxX[$i] = $a < $b ? $b : $a; + } + asort($minX, SORT_NUMERIC); + $active = []; // edge => max x, edges whose x range may still overlap the next ones + $work = 0; + foreach ($minX as $i => $from) { + foreach ($active as $j => $to) { + if ($to < $from) { + unset($active[$j]); + continue; + } + $d = $i > $j ? $i - $j : $j - $i; + if ($d === 1 || $d === $n - 1) { + continue; // neighbours share a vertex (the first and the last edge share the closing one) + } + if (++$work > $budget) { + return $this->ringSelfIntersectsSweep($xs, $ys, $n); + } + $this->segmentPairTests++; // @test + if (self::segmentsMeet($xs[$i], $ys[$i], $xs[$i + 1], $ys[$i + 1], $xs[$j], $ys[$j], $xs[$j + 1], $ys[$j + 1])) { + return true; + } + } + $active[$i] = $maxX[$i]; + } + return false; + } + + /** + * ringSelfIntersects() as a sweep line (Shamos-Hoey) over an AVL tree of the edges that cross it, + * ordered from bottom to top: O(n log n). Events are the vertices in (x, y) order. A ring that doesn't + * intersect itself has at every event point exactly one vertex, so exactly its two edges end or + * start there and no other edge passes through it; between events two edges can only cross after + * they became neighbours in the tree, and every new pair of neighbours is tested. + */ + private function ringSelfIntersectsSweep(array $xs, array $ys, int $n): bool { + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $tree = self::avlTree(array_fill(0, $n, 0)); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi); // @prod + $cmp = self::sweepOrder($xs, $ys, $lo, $hi, $this->segmentPairTests); // @test + $meet = function (int $a, int $b) use ($xs, $ys, $n): bool { + if ($a < 0 || $b < 0) { + return false; + } + $d = $a > $b ? $a - $b : $b - $a; + if ($d === 1 || $d === $n - 1) { + return false; + } + $this->segmentPairTests++; // @test + return self::segmentsMeet($xs[$a], $ys[$a], $xs[$a + 1], $ys[$a + 1], $xs[$b], $ys[$b], $xs[$b + 1], $ys[$b + 1]); + }; + $count = count($ee); + for ($k = 0; $k < $count;) { + $px = $ex[$k]; + $py = $ey[$k]; + $edges = []; + $starts = []; + $ends = []; + for (; $k < $count && $ex[$k] == $px && $ey[$k] == $py; $k++) { + $edge = $ee[$k] >> 1; + $edges[$edge] = true; + if ($ee[$k] & 1) { + $ends[] = $edge; + } else { + $starts[] = $edge; + } + } + // One vertex: its two edges, adjacent (a repeated vertex or a zero-length edge gives more) + if (count($edges) !== 2) { + return true; + } + [$e1, $e2] = array_keys($edges); + $d = $e1 > $e2 ? $e1 - $e2 : $e2 - $e1; + if ($d !== 1 && $d !== $n - 1) { + return true; + } + foreach ($ends as $edge) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $edge, $cmp); + } + // No other edge through the vertex; the edges below and above it become neighbours + $below = -1; + $above = -1; + for ($node = $tree['root']; $node >= 0;) { + $this->segmentPairTests++; // @test + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $px, $py); + if ($c === 0) { + return true; // the vertex lies on another edge + } + if ($c > 0) { + $below = $node; + $node = $tree['r'][$node]; + } else { + $above = $node; + $node = $tree['l'][$node]; + } + } + if (!$starts) { + if ($meet($below, $above)) { + return true; + } + continue; + } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + if ($meet($pred, $edge) || $meet($edge, $succ)) { + return true; + } + } + } + return false; + } + + /** + * Sweep events of a ring's edges: [lo, hi, x, y, code] with lo/hi the vertex numbers of each edge's + * smaller and larger end in (x, y) order, and the events (both ends of every edge, code edge * 2 + * for the start, edge * 2 + 1 for the end) sorted by x, then y + */ + private static function sweepEvents(array $xs, array $ys, int $n): array { + $lo = []; + $hi = []; + $ex = []; + $ey = []; + $ee = []; + for ($i = 0; $i < $n; $i++) { + $a = $i; + $b = $i + 1; + if ($xs[$b] < $xs[$a] || ($xs[$b] == $xs[$a] && $ys[$b] < $ys[$a])) { + $a = $i + 1; + $b = $i; + } + $lo[$i] = $a; + $hi[$i] = $b; + $ex[] = $xs[$a]; + $ey[] = $ys[$a]; + $ee[] = $i * 2; + $ex[] = $xs[$b]; + $ey[] = $ys[$b]; + $ee[] = $i * 2 + 1; + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + return [$lo, $hi, $ex, $ey, $ee]; + } + + /** + * Order of two edges that cross the sweep line at the same time and don't cross each other (bottom + * to top): compared where the later one starts, then by direction; equal edges by number (the test + * build counts the comparisons) + */ + private static function sweepOrder(array $xs, array $ys, array $lo, array $hi): \Closure { // @prod + private static function sweepOrder(array $xs, array $ys, array $lo, array $hi, int &$counter): \Closure { // @test + return function (int $s, int $t) use ($xs, $ys, $lo, $hi): int { // @prod + return function (int $s, int $t) use ($xs, $ys, $lo, $hi, &$counter): int { // @test + $counter++; // @test + $sx = $xs[$lo[$s]]; + $sy = $ys[$lo[$s]]; + $tx = $xs[$lo[$t]]; + $ty = $ys[$lo[$t]]; + if ($sx > $tx || ($sx == $tx && $sy > $ty)) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $sx, $sy); + if ($c === 0) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } + } elseif ($sx == $tx && $sy == $ty) { + $c = self::orient($tx, $ty, $xs[$hi[$t]], $ys[$hi[$t]], $xs[$hi[$s]], $ys[$hi[$s]]); + } else { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $tx, $ty); + if ($c === 0) { + $c = -self::orient($sx, $sy, $xs[$hi[$s]], $ys[$hi[$s]], $xs[$hi[$t]], $ys[$hi[$t]]); + } + } + return $c !== 0 ? $c : ($s <=> $t); + }; + } + + /** + * Where points are against a ring that doesn't intersect itself: 1 inside, 0 on its boundary, -1 + * outside (exact). Every point against every edge when that is cheap; otherwise one sweep line over + * the edges with the points as events, O((n + m) log n): the edges below a point, each counted +1 or + * -1 by its direction, give its winding number (a ray straight down). + */ + private function locatePointsInRing(array $ring, array $points): array { + $xs = []; + $ys = []; + foreach ($ring as $position) { + $xs[] = (float)$position[0]; + $ys[] = (float)$position[1]; + } + $n = count($xs) - 1; + $m = count($points); + $budget = (int)(16 * ($n + $m) * log($n + $m + 2, 2)); // @prod + $budget = $this->sweepBudget ?? (int)(16 * ($n + $m) * log($n + $m + 2, 2)); // @test + $out = []; + if ($n < 4 || ($budget > 0 && $n * $m <= $budget)) { + foreach ($points as $point) { + $out[] = self::locatePointInRing((float)$point[0], (float)$point[1], $xs, $ys, $n); + } + return $out; + } + + [$lo, $hi, $ex, $ey, $ee] = self::sweepEvents($xs, $ys, $n); + $vertices = []; + for ($i = 0; $i < $n; $i++) { + $vertices[pack('dd', $xs[$i] + 0.0, $ys[$i] + 0.0)] = true; + } + $weights = []; + for ($i = 0; $i < $n; $i++) { + $weights[$i] = $lo[$i] === $i ? 1 : -1; // run from its smaller end to its larger one + } + foreach ($points as $k => $point) { + $x = (float)$point[0]; + $y = (float)$point[1]; + if (isset($vertices[pack('dd', $x + 0.0, $y + 0.0)])) { + $out[$k] = 0; + continue; + } + $out[$k] = -1; + $ex[] = $x; + $ey[] = $y; + $ee[] = -1 - $k; // point events: negative codes + } + array_multisort($ex, SORT_NUMERIC, $ey, SORT_NUMERIC, $ee); + $tree = self::avlTree($weights); + $cmp = self::sweepOrder($xs, $ys, $lo, $hi); // @prod + $cmp = self::sweepOrder($xs, $ys, $lo, $hi, $this->pointEdgeTests); // @test + $count = count($ee); + for ($k = 0; $k < $count;) { + if ($ee[$k] < 0) { + // A point (never at a vertex): the edges below it + $x = $ex[$k]; + $y = $ey[$k]; + $winding = 0; + for ($node = $tree['root']; $node >= 0;) { + $this->pointEdgeTests++; // @test + $c = self::orient($xs[$lo[$node]], $ys[$lo[$node]], $xs[$hi[$node]], $ys[$hi[$node]], $x, $y); + if ($c === 0) { + $winding = null; // on this edge + break; + } + if ($c > 0) { + $winding += $tree['s'][$tree['l'][$node]] + $weights[$node]; + $node = $tree['r'][$node]; + } else { + $node = $tree['l'][$node]; + } + } + $out[-1 - $ee[$k]] = $winding === null ? 0 : ($winding !== 0 ? 1 : -1); + $k++; + continue; + } + // A vertex: its edges that end here leave the tree, then the ones that start here join it + $px = $ex[$k]; + $py = $ey[$k]; + $starts = []; + for (; $k < $count && $ee[$k] >= 0 && $ex[$k] == $px && $ey[$k] == $py; $k++) { + if ($ee[$k] & 1) { + $tree['root'] = self::avlDelete($tree, $tree['root'], $ee[$k] >> 1, $cmp); + } else { + $starts[] = $ee[$k] >> 1; + } + } + foreach ($starts as $edge) { + $pred = -1; + $succ = -1; + $tree['root'] = self::avlInsert($tree, $tree['root'], $edge, $cmp, $pred, $succ); + } + } + ksort($out); + return $out; + } + + /** + * One point against a ring, exact: 1 inside, 0 on its boundary, -1 outside (winding number) + */ + private static function locatePointInRing(float $x, float $y, array $xs, array $ys, int $n): int { + \noneDB\Internal\Cache\Runtime::get()->pointEdgeTests += $n; // @test + $winding = 0; + for ($i = 0; $i < $n; $i++) { + $x1 = $xs[$i]; + $y1 = $ys[$i]; + $x2 = $xs[$i + 1]; + $y2 = $ys[$i + 1]; + $o = null; + if (($x1 < $x2 ? $x1 <= $x && $x <= $x2 : $x2 <= $x && $x <= $x1) + && ($y1 < $y2 ? $y1 <= $y && $y <= $y2 : $y2 <= $y && $y <= $y1)) { + $o = self::orient($x1, $y1, $x2, $y2, $x, $y); + if ($o === 0) { + return 0; + } + } + if ($y1 <= $y) { + if ($y2 > $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) > 0) { + $winding++; + } + } elseif ($y2 <= $y && ($o ?? self::orient($x1, $y1, $x2, $y2, $x, $y)) < 0) { + $winding--; + } + } + return $winding !== 0 ? 1 : -1; + } + + /** + * True when two closed segments share a point (exact) + */ + private static function segmentsMeet(float $ax, float $ay, float $bx, float $by, float $cx, float $cy, float $dx, float $dy): bool { + if (($ax < $bx ? $ax : $bx) > ($cx > $dx ? $cx : $dx) || ($cx < $dx ? $cx : $dx) > ($ax > $bx ? $ax : $bx) + || ($ay < $by ? $ay : $by) > ($cy > $dy ? $cy : $dy) || ($cy < $dy ? $cy : $dy) > ($ay > $by ? $ay : $by)) { + return false; // bounding boxes apart + } + $o1 = self::orient($ax, $ay, $bx, $by, $cx, $cy); + $o2 = self::orient($ax, $ay, $bx, $by, $dx, $dy); + $o3 = self::orient($cx, $cy, $dx, $dy, $ax, $ay); + $o4 = self::orient($cx, $cy, $dx, $dy, $bx, $by); + if ($o1 * $o2 < 0 && $o3 * $o4 < 0) { + return true; + } + // Touching or collinear: an end of one on the other (inside the boxes, which overlap) + return ($o1 === 0 && self::inBox($ax, $ay, $bx, $by, $cx, $cy)) || ($o2 === 0 && self::inBox($ax, $ay, $bx, $by, $dx, $dy)) + || ($o3 === 0 && self::inBox($cx, $cy, $dx, $dy, $ax, $ay)) || ($o4 === 0 && self::inBox($cx, $cy, $dx, $dy, $bx, $by)); + } + + private static function inBox(float $ax, float $ay, float $bx, float $by, float $px, float $py): bool { + return ($ax < $bx ? $ax <= $px && $px <= $bx : $bx <= $px && $px <= $ax) + && ($ay < $by ? $ay <= $py && $py <= $by : $by <= $py && $py <= $ay); + } + + /** + * Sign of the orientation of c against the directed line a -> b: 1 left, -1 right, 0 on the line. + * Exact for all finite doubles: the floating-point determinant decides when it is clearly away from + * zero (Shewchuk's error bound), otherwise it is summed exactly. + */ + private static function orient(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + $detLeft = ($ax - $cx) * ($by - $cy); + $detRight = ($ay - $cy) * ($bx - $cx); + $det = $detLeft - $detRight; + if ($detLeft > 0.0) { + if ($detRight <= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = $detLeft + $detRight; + } elseif ($detLeft < 0.0) { + if ($detRight >= 0.0) { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $detSum = -$detLeft - $detRight; + } else { + return $det > 0.0 ? 1 : ($det < 0.0 ? -1 : 0); + } + $bound = 3.3306690738754716e-16 * $detSum; // (3 + 16 eps) eps, eps = 2^-53 + if ($det >= $bound || -$det >= $bound) { + return $det > 0.0 ? 1 : -1; + } + return self::orientExact($ax, $ay, $bx, $by, $cx, $cy); + } + + /** + * Exact sign of (a - c) x (b - c): the differences and products as exact float sums (Dekker, + * Knuth), added into a nonoverlapping expansion whose largest part has the sign + */ + private static function orientExact(float $ax, float $ay, float $bx, float $by, float $cx, float $cy): int { + [$acx, $acxTail] = self::twoDiff($ax, $cx); + [$bcy, $bcyTail] = self::twoDiff($by, $cy); + [$acy, $acyTail] = self::twoDiff($ay, $cy); + [$bcx, $bcxTail] = self::twoDiff($bx, $cx); + $sum = []; + foreach ([[$acx, $bcy, 1.0], [$acx, $bcyTail, 1.0], [$acxTail, $bcy, 1.0], [$acxTail, $bcyTail, 1.0], + [$acy, $bcx, -1.0], [$acy, $bcxTail, -1.0], [$acyTail, $bcx, -1.0], [$acyTail, $bcxTail, -1.0]] as [$p, $q, $sign]) { + if ($p == 0.0 || $q == 0.0) { + continue; + } + [$product, $error] = self::twoProduct($p, $q); + $sum = self::growExpansion(self::growExpansion($sum, $sign * $product), $sign * $error); + } + for ($i = count($sum) - 1; $i >= 0; $i--) { + if ($sum[$i] != 0.0) { + return $sum[$i] > 0.0 ? 1 : -1; + } + } + return 0; + } + + /** Add $b to a nonoverlapping expansion (smallest part first), zero parts dropped */ + private static function growExpansion(array $parts, float $b): array { + if ($b == 0.0) { + return $parts; + } + $out = []; + $q = $b; + foreach ($parts as $part) { + [$q, $error] = self::twoSum($q, $part); + if ($error != 0.0) { + $out[] = $error; + } + } + $out[] = $q; + return $out; + } + + /** a + b = x + y exactly */ + private static function twoSum(float $a, float $b): array { + $x = $a + $b; + $bVirtual = $x - $a; + $aVirtual = $x - $bVirtual; + return [$x, ($a - $aVirtual) + ($b - $bVirtual)]; + } + + /** a - b = x + y exactly */ + private static function twoDiff(float $a, float $b): array { + $x = $a - $b; + $bVirtual = $a - $x; + $aVirtual = $x + $bVirtual; + return [$x, ($a - $aVirtual) + ($bVirtual - $b)]; + } + + /** a * b = x + y exactly (Dekker's split into 26-bit halves) */ + private static function twoProduct(float $a, float $b): array { + $x = $a * $b; + $c = 134217729.0 * $a; // 2^27 + 1 + $aHigh = $c - ($c - $a); + $aLow = $a - $aHigh; + $c = 134217729.0 * $b; + $bHigh = $c - ($c - $b); + $bLow = $b - $bHigh; + $error = (($x - $aHigh * $bHigh) - $aLow * $bHigh) - $aHigh * $bLow; + return [$x, $aLow * $bLow - $error]; + } + + /** + * An empty AVL tree for the sweeps; $weights[node] is summed per subtree ('s'). Index -1 is the + * empty child (height 0, sum 0). + */ + private static function avlTree(array $weights): array { + return ['l' => [-1 => -1], 'r' => [-1 => -1], 'h' => [-1 => 0], 's' => [-1 => 0], 'w' => $weights, 'root' => -1]; + } + + /** Insert $node below $x, return the new subtree root; $pred/$succ get its neighbours */ + private static function avlInsert(array &$t, int $x, int $node, \Closure $cmp, int &$pred, int &$succ): int { + if ($x < 0) { + $t['l'][$node] = -1; + $t['r'][$node] = -1; + $t['h'][$node] = 1; + $t['s'][$node] = $t['w'][$node]; + return $node; + } + if ($cmp($node, $x) < 0) { + $succ = $x; + $t['l'][$x] = self::avlInsert($t, $t['l'][$x], $node, $cmp, $pred, $succ); + } else { + $pred = $x; + $t['r'][$x] = self::avlInsert($t, $t['r'][$x], $node, $cmp, $pred, $succ); + } + return self::avlBalance($t, $x); + } + + /** Remove $node (it is in the tree) below $x, return the new subtree root */ + private static function avlDelete(array &$t, int $x, int $node, \Closure $cmp): int { + if ($x < 0) { + return -1; + } + if ($x !== $node) { + if ($cmp($node, $x) < 0) { + $t['l'][$x] = self::avlDelete($t, $t['l'][$x], $node, $cmp); + } else { + $t['r'][$x] = self::avlDelete($t, $t['r'][$x], $node, $cmp); + } + return self::avlBalance($t, $x); + } + $left = $t['l'][$x]; + $right = $t['r'][$x]; + if ($left < 0) { + return $right; + } + if ($right < 0) { + return $left; + } + $next = $right; + while ($t['l'][$next] >= 0) { + $next = $t['l'][$next]; + } + $t['r'][$next] = self::avlDeleteMin($t, $right); + $t['l'][$next] = $left; + return self::avlBalance($t, $next); + } + + private static function avlDeleteMin(array &$t, int $x): int { + if ($t['l'][$x] < 0) { + return $t['r'][$x]; + } + $t['l'][$x] = self::avlDeleteMin($t, $t['l'][$x]); + return self::avlBalance($t, $x); + } + + private static function avlBalance(array &$t, int $x): int { + self::avlUpdate($t, $x); + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $diff = $t['h'][$left] - $t['h'][$right]; + if ($diff > 1) { + if ($t['h'][$t['l'][$left]] < $t['h'][$t['r'][$left]]) { + $t['l'][$x] = self::avlRotate($t, $left, true); + } + return self::avlRotate($t, $x, false); + } + if ($diff < -1) { + if ($t['h'][$t['r'][$right]] < $t['h'][$t['l'][$right]]) { + $t['r'][$x] = self::avlRotate($t, $right, false); + } + return self::avlRotate($t, $x, true); + } + return $x; + } + + /** Rotate: $left = the right child goes up, else the left child; returns the new subtree root */ + private static function avlRotate(array &$t, int $x, bool $left): int { + if ($left) { + $y = $t['r'][$x]; + $t['r'][$x] = $t['l'][$y]; + $t['l'][$y] = $x; + } else { + $y = $t['l'][$x]; + $t['l'][$x] = $t['r'][$y]; + $t['r'][$y] = $x; + } + self::avlUpdate($t, $x); + self::avlUpdate($t, $y); + return $y; + } + + private static function avlUpdate(array &$t, int $x): void { + $left = $t['l'][$x]; + $right = $t['r'][$x]; + $t['h'][$x] = ($t['h'][$left] > $t['h'][$right] ? $t['h'][$left] : $t['h'][$right]) + 1; + $t['s'][$x] = $t['s'][$left] + $t['s'][$right] + $t['w'][$x]; + } + + /** + * True when a point lies on a ring's boundary + */ + public function pointOnRing(float $lon, float $lat, array $ring): bool { + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + if ($this->geometry->crossProductDirection($ring[$i], $ring[$i + 1], [$lon, $lat]) == 0 + && $this->geometry->onSegment($ring[$i], $ring[$i + 1], [$lon, $lat])) { + return true; + } + } + return false; + } + + /** + * Validate MultiPoint coordinates [[lon,lat], [lon,lat], ...] + */ + private function validateGeoJSONMultiPoint($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPoint must have at least one point', 'type' => null]; + } + foreach ($coords as $i => $position) { + $result = $this->validateGeoJSONPoint($position); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid point at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiPoint']; + } + + /** + * Validate MultiLineString coordinates [[[lon,lat], ...], ...] + */ + private function validateGeoJSONMultiLineString($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiLineString must have at least one linestring', 'type' => null]; + } + foreach ($coords as $i => $lineString) { + $result = $this->validateGeoJSONLineString($lineString); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid linestring at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiLineString']; + } + + /** + * Validate MultiPolygon coordinates [[[[lon,lat], ...], ...], ...] + */ + private function validateGeoJSONMultiPolygon($coords): array { + if (!self::isJsonList($coords) || count($coords) < 1) { + return ['valid' => false, 'error' => 'MultiPolygon must have at least one polygon', 'type' => null]; + } + foreach ($coords as $i => $polygon) { + $result = $this->validateGeoJSONPolygon($polygon); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid polygon at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'MultiPolygon']; + } + + /** + * Validate GeometryCollection + */ + private function validateGeometryCollection(array $geometry): array { + if (!isset($geometry['geometries']) || !self::isJsonList($geometry['geometries'])) { + return ['valid' => false, 'error' => 'GeometryCollection must have geometries array', 'type' => null]; + } + foreach ($geometry['geometries'] as $i => $geom) { + if (!is_array($geom)) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: must be an object", 'type' => null]; + } + $result = $this->validateGeoJSON($geom); + if (!$result['valid']) { + return ['valid' => false, 'error' => "Invalid geometry at index $i: " . $result['error'], 'type' => null]; + } + } + return ['valid' => true, 'error' => null, 'type' => 'GeometryCollection']; + } +} diff --git a/src/Internal/Index/Spatial/Geometry.php b/src/Internal/Index/Spatial/Geometry.php new file mode 100644 index 0000000..a22d10d --- /dev/null +++ b/src/Internal/Index/Spatial/Geometry.php @@ -0,0 +1,250 @@ +k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL INDEX - GEOMETRY OPERATIONS + // ========================================== + + /** + * Calculate Minimum Bounding Rectangle (MBR) for any GeoJSON geometry + * @param array $geometry GeoJSON geometry object + * @return array|null [minLon, minLat, maxLon, maxLat]; null for a geometry without any position + * (an empty GeometryCollection): it has no location and is never indexed or matched + */ + public function calculateMBR(array $geometry): ?array { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + + switch ($type) { + case 'Point': + return [$coords[0], $coords[1], $coords[0], $coords[1]]; + + case 'LineString': + case 'MultiPoint': + return $this->mbrFromPoints($coords); + + case 'Polygon': + // Use outer ring (first ring) + return $this->mbrFromPoints($coords[0]); + + case 'MultiLineString': + $allPoints = []; + foreach ($coords as $lineString) { + $allPoints = array_merge($allPoints, $lineString); + } + return $this->mbrFromPoints($allPoints); + + case 'MultiPolygon': + $allPoints = []; + foreach ($coords as $polygon) { + $allPoints = array_merge($allPoints, $polygon[0]); // outer ring + } + return $this->mbrFromPoints($allPoints); + + case 'GeometryCollection': + $mbr = null; + foreach ($geometry['geometries'] as $geom) { + $geomMBR = $this->calculateMBR($geom); + if ($geomMBR !== null) { + $mbr = $mbr === null ? $geomMBR : $this->mbrUnion($mbr, $geomMBR); + } + } + return $mbr; + + default: + return null; + } + } + + /** + * Calculate MBR from array of points + * @param array $points Array of [lon, lat] positions + * @return array [minLon, minLat, maxLon, maxLat] + */ + public function mbrFromPoints(array $points): ?array { + if (empty($points)) { + return null; + } + + $minLon = $maxLon = $points[0][0]; + $minLat = $maxLat = $points[0][1]; + + foreach ($points as $point) { + $minLon = min($minLon, $point[0]); + $maxLon = max($maxLon, $point[0]); + $minLat = min($minLat, $point[1]); + $maxLat = max($maxLat, $point[1]); + } + + return [$minLon, $minLat, $maxLon, $maxLat]; + } + + /** + * Union two MBRs + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return array [minLon, minLat, maxLon, maxLat] + */ + private function mbrUnion(array $mbr1, array $mbr2): array { + return [ + min($mbr1[0], $mbr2[0]), // minLon + min($mbr1[1], $mbr2[1]), // minLat + max($mbr1[2], $mbr2[2]), // maxLon + max($mbr1[3], $mbr2[3]) // maxLat + ]; + } + + /** + * Check if two MBRs overlap + * @param array $mbr1 [minLon, minLat, maxLon, maxLat] + * @param array $mbr2 [minLon, minLat, maxLon, maxLat] + * @return bool + */ + public function mbrOverlaps(array $mbr1, array $mbr2): bool { + // Check if one MBR is completely to the left, right, above, or below the other + if ($mbr1[2] < $mbr2[0] || $mbr2[2] < $mbr1[0]) return false; // No horizontal overlap + if ($mbr1[3] < $mbr2[1] || $mbr2[3] < $mbr1[1]) return false; // No vertical overlap + return true; + } + + /** + * Calculate Haversine distance between two points (in meters). Not memoized: a key for two points costs + * more than the formula + * @param float $lon1 Longitude of point 1 + * @param float $lat1 Latitude of point 1 + * @param float $lon2 Longitude of point 2 + * @param float $lat2 Latitude of point 2 + * @return float Distance in meters + */ + public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float { + $earthRadius = 6371000.0; // meters + + $latDelta = deg2rad($lat2 - $lat1); + $lonDelta = deg2rad($lon2 - $lon1); + + $a = sin($latDelta / 2) * sin($latDelta / 2) + + cos(deg2rad($lat1)) * cos(deg2rad($lat2)) * + sin($lonDelta / 2) * sin($lonDelta / 2); + + $c = 2 * atan2(sqrt($a), sqrt(1 - $a)); + + return $earthRadius * $c; + } + + /** + * Point-in-Polygon test using Winding Number algorithm + * More robust than ray casting for edge cases + * + * @param float $lon Point longitude + * @param float $lat Point latitude + * @param array $polygon GeoJSON Polygon coordinates [[[lon,lat], ...], ...] + * @return bool + */ + public function pointInPolygon(float $lon, float $lat, array $polygon): bool { + // Handle GeoJSON Polygon object or raw coordinates + $coords = isset($polygon['coordinates']) ? $polygon['coordinates'] : $polygon; + + // Check outer ring + if (!$this->pointInRing($lon, $lat, $coords[0])) { + return false; + } + + // Check holes (if any) + for ($i = 1; $i < count($coords); $i++) { + if ($this->pointInRing($lon, $lat, $coords[$i])) { + return false; // Inside a hole + } + } + + return true; + } + + /** + * Check if point is inside a ring using Winding Number algorithm + */ + public function pointInRing(float $lon, float $lat, array $ring): bool { + $windingNumber = 0; + $n = count($ring); + + for ($i = 0; $i < $n - 1; $i++) { + $x1 = $ring[$i][0]; + $y1 = $ring[$i][1]; + $x2 = $ring[$i + 1][0]; + $y2 = $ring[$i + 1][1]; + + if ($y1 <= $lat) { + if ($y2 > $lat) { + // Upward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) > 0) { + $windingNumber++; + } + } + } else { + if ($y2 <= $lat) { + // Downward crossing + if ($this->isLeftOfLine($x1, $y1, $x2, $y2, $lon, $lat) < 0) { + $windingNumber--; + } + } + } + } + + return $windingNumber !== 0; + } + + /** + * Determine if point is left of a line (using cross product) + * Returns: >0 if left, <0 if right, =0 if on line + */ + private function isLeftOfLine(float $x1, float $y1, float $x2, float $y2, float $px, float $py): float { + return (($x2 - $x1) * ($py - $y1)) - (($px - $x1) * ($y2 - $y1)); + } + + /** + * Cross product direction calculation + */ + public function crossProductDirection(array $p1, array $p2, array $p3): float { + return ($p3[0] - $p1[0]) * ($p2[1] - $p1[1]) - ($p2[0] - $p1[0]) * ($p3[1] - $p1[1]); + } + + /** + * Check if point p is on segment p1-p2 + */ + public function onSegment(array $p1, array $p2, array $p): bool { + return min($p1[0], $p2[0]) <= $p[0] && $p[0] <= max($p1[0], $p2[0]) && + min($p1[1], $p2[1]) <= $p[1] && $p[1] <= max($p1[1], $p2[1]); + } +} diff --git a/src/Internal/Index/Spatial/RTree.php b/src/Internal/Index/Spatial/RTree.php new file mode 100644 index 0000000..7f894b5 --- /dev/null +++ b/src/Internal/Index/Spatial/RTree.php @@ -0,0 +1,936 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->geoJson, $this->geometry, $this->io, $this->layout, $this->locks, $this->records, $this->secondary, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->rtreeNodeSize = &$this->k->rtreeNodeSize; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->multiLock = &$this->rt->multiLock; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->secondary = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Path of a segment's R-tree + */ + public function getSpatialIndexPath(string $dbname, string $field, ?int $shardId = null): string { + return $this->secondary->secondaryPath($dbname, 'rx', $field, $shardId); + } + + public function getGlobalSpatialIndexPath(string $dbname, string $field): string { + return $this->secondary->secondaryPath($dbname, 'grx', $field); + } + + /** + * Integer box of a geometry (outwards), or null when it has no position + */ + private function quantizedMBR(array $geometry): ?array { + $mbr = $this->geometry->calculateMBR($geometry); + return $mbr === null ? null : $this->quantizeBox($mbr); + } + + /** + * Integer box of [minLon, minLat, maxLon, maxLat] (clamped to the globe, rounded outwards) + */ + private function quantizeBox(array $box): array { + $minX = max(-180.0, min(180.0, (float)$box[0])); + $minY = max(-90.0, min(90.0, (float)$box[1])); + $maxX = max(-180.0, min(180.0, (float)$box[2])); + $maxY = max(-90.0, min(90.0, (float)$box[3])); + return [(int)floor($minX * self::RTREE_SCALE), (int)floor($minY * self::RTREE_SCALE), + (int)ceil($maxX * self::RTREE_SCALE), (int)ceil($maxY * self::RTREE_SCALE)]; + } + + private static function boxUnion(?array $a, array $b): array { + if($a === null){ + return $b; + } + return [$a[0] < $b[0] ? $a[0] : $b[0], $a[1] < $b[1] ? $a[1] : $b[1], $a[2] > $b[2] ? $a[2] : $b[2], $a[3] > $b[3] ? $a[3] : $b[3]]; + } + + private static function boxOverlaps(?array $a, array $b): bool { + return $a !== null && $a[0] <= $b[2] && $b[0] <= $a[2] && $a[1] <= $b[3] && $b[1] <= $a[3]; + } + + private static function isBox($box): bool { + return is_int($box[0] ?? null) && is_int($box[1] ?? null) && is_int($box[2] ?? null) && is_int($box[3] ?? null); + } + + private static function boxArea(array $a): float { + return ((float)$a[2] - $a[0]) * ((float)$a[3] - $a[1]); + } + + public function newRTree(string $field, int $nodeSize): array { + return ['kind' => 'rx', 'field' => $field, 'ns' => $nodeSize, 's' => 0, 'ck' => -1, 'e' => null, 'bytes' => 0, + 'nodes' => [0 => [0, null, []]], 'root' => 0, 'next' => 1, 'mk' => -1, 'cnt' => 0, 'leafOf' => null, 'parent' => null]; + } + + /** + * Bulk load (sort-tile-recursive): leaves 3/4 full, ties broken by key (the same tree on every PHP). The entries + * come as columns, sorted with array_multisort(): no array per entry until the leaves are made + * @param int[] $ids keys (the leaves), then node ids (the levels above) + * @param int[] $minX, $minY, $maxX, $maxY their boxes, in the same order + */ + private function rtreeBulkLoad(array &$tree, array $ids, array $minX, array $minY, array $maxX, array $maxY): void { + $capacity = max(2, intdiv($tree['ns'] * 3, 4)); + $tree['nodes'] = []; + $tree['next'] = 0; + $tree['cnt'] = count($ids); + $tree['leafOf'] = null; + $tree['parent'] = null; + if(empty($ids)){ + $tree['nodes'][0] = [0, null, []]; + $tree['root'] = 0; + $tree['next'] = 1; + return; + } + for($level = 0; ; $level++){ + $count = count($ids); + $perSlice = (int)ceil(sqrt((int)ceil($count / $capacity))) * $capacity; + // By the x center (doubled: min + max), then the id + $center = []; + for($i = 0; $i < $count; $i++){ + $center[] = $minX[$i] + $maxX[$i]; + } + array_multisort($center, SORT_NUMERIC, $ids, SORT_NUMERIC, $minX, $minY, $maxX, $maxY); + $center = null; + $up = [[], [], [], [], []]; // the nodes made: ids and boxes + for($i = 0; $i < $count; $i += $perSlice){ + // A vertical slice, by the y center, then the id + $sIds = array_slice($ids, $i, $perSlice); + $sMinX = array_slice($minX, $i, $perSlice); + $sMinY = array_slice($minY, $i, $perSlice); + $sMaxX = array_slice($maxX, $i, $perSlice); + $sMaxY = array_slice($maxY, $i, $perSlice); + $m = count($sIds); + $center = []; + for($j = 0; $j < $m; $j++){ + $center[] = $sMinY[$j] + $sMaxY[$j]; + } + array_multisort($center, SORT_NUMERIC, $sIds, SORT_NUMERIC, $sMinX, $sMinY, $sMaxX, $sMaxY); + for($j = 0; $j < $m; $j += $capacity){ + $id = $tree['next']++; + $box = [$sMinX[$j], $sMinY[$j], $sMaxX[$j], $sMaxY[$j]]; + $payload = []; + for($k = $j, $end = min($m, $j + $capacity); $k < $end; $k++){ + $box = [min($box[0], $sMinX[$k]), min($box[1], $sMinY[$k]), max($box[2], $sMaxX[$k]), max($box[3], $sMaxY[$k])]; + $payload[] = $level === 0 ? [$sIds[$k], $sMinX[$k], $sMinY[$k], $sMaxX[$k], $sMaxY[$k]] : $sIds[$k]; + } + $tree['nodes'][$id] = [$level, $box, $payload]; + $up[0][] = $id; + $up[1][] = $box[0]; + $up[2][] = $box[1]; + $up[3][] = $box[2]; + $up[4][] = $box[3]; + } + } + if(count($up[0]) === 1){ + $tree['root'] = $up[0][0]; + return; + } + [$ids, $minX, $minY, $maxX, $maxY] = $up; + } + } + + /** + * Parent and leaf maps (built when the first change needs them) + */ + public function rtreeMaps(array &$tree): void { + if($tree['parent'] !== null){ + return; + } + $parent = []; + $leafOf = []; + foreach($tree['nodes'] as $id => $node){ + if($node[0] === 0){ + foreach($node[2] as $entry){ + $leafOf[$entry[0]] = $id; + } + } else { + foreach($node[2] as $child){ + $parent[$child] = $id; + } + } + } + $tree['parent'] = $parent; + $tree['leafOf'] = $leafOf; + } + + /** + * Insert (or move) a key with its box. The way down is remembered for the box updates and splits, + * so a key never seen before needs no parent map (built only when something is removed). + */ + private function rtreeInsert(array &$tree, int $key, array $box): void { + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } else { + $tree['mk'] = $key; + } + $path = []; + $id = $tree['root']; + while($tree['nodes'][$id][0] > 0){ + $best = null; + $bestGrowth = INF; + $bestArea = INF; + foreach($tree['nodes'][$id][2] as $child){ + $childBox = $tree['nodes'][$child][1]; + if($childBox === null){ + $best = $best ?? $child; + continue; + } + $area = self::boxArea($childBox); + $growth = self::boxArea(self::boxUnion($childBox, $box)) - $area; + if($growth < $bestGrowth || ($growth == $bestGrowth && $area < $bestArea)){ + $best = $child; + $bestGrowth = $growth; + $bestArea = $area; + } + } + if($best === null){ + $tree['nodes'][$id] = [0, null, []]; // an internal node without children becomes a leaf + break; + } + $path[] = $id; + $id = $best; + } + $tree['nodes'][$id][2][] = [$key, $box[0], $box[1], $box[2], $box[3]]; + if($tree['leafOf'] !== null){ + $tree['leafOf'][$key] = $id; + } + $tree['cnt']++; + // Widen the boxes on the way up while they change + for($node = $id, $i = count($path); ; $node = $path[--$i]){ + $old = $tree['nodes'][$node][1]; + $new = self::boxUnion($old, $box); + if($old === $new){ + break; + } + $tree['nodes'][$node][1] = $new; + if($i === 0){ + break; + } + } + if(count($tree['nodes'][$id][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $id, $path); + } + } + + private function rtreeBoxOf(array $tree, int $level, $item): ?array { + return $level === 0 ? [$item[1], $item[2], $item[3], $item[4]] : $tree['nodes'][$item][1]; + } + + /** + * Split an overflowing node (linear seeds, least enlargement), up to the root + * @param array $path The node's ancestors, root first + */ + private function rtreeSplit(array &$tree, int $id, array $path): void { + [$level, , $items] = $tree['nodes'][$id]; + $n = count($items); + $boxes = []; + foreach($items as $i => $item){ + $boxes[$i] = $this->rtreeBoxOf($tree, $level, $item) ?? [0, 0, 0, 0]; + } + // Seeds: the pair farthest apart along the axis with the greatest normalized separation + $best = null; + foreach([0, 1] as $axis){ + $lowHigh = 0; + $highLow = 0; + $min = INF; + $max = -INF; + foreach($boxes as $i => $b){ + if($b[$axis] > $boxes[$lowHigh][$axis]){ + $lowHigh = $i; + } + if($b[$axis + 2] < $boxes[$highLow][$axis + 2]){ + $highLow = $i; + } + $min = min($min, $b[$axis]); + $max = max($max, $b[$axis + 2]); + } + $width = $max - $min; + $separation = $width > 0 ? ($boxes[$lowHigh][$axis] - $boxes[$highLow][$axis + 2]) / $width : 0; + if($best === null || $separation > $best[0]){ + $best = [$separation, $lowHigh, $highLow]; + } + } + [, $seed1, $seed2] = $best; + if($seed1 === $seed2){ + $seed2 = ($seed1 + 1) % $n; + } + $groups = [[$items[$seed1]], [$items[$seed2]]]; + $groupBoxes = [$boxes[$seed1], $boxes[$seed2]]; + $minFill = max(1, (int)($tree['ns'] * 0.4)); + $left = $n - 2; + foreach($items as $i => $item){ + if($i === $seed1 || $i === $seed2){ + continue; + } + if(count($groups[0]) + $left <= $minFill){ + $g = 0; + } elseif(count($groups[1]) + $left <= $minFill){ + $g = 1; + } else { + $grow0 = self::boxArea(self::boxUnion($groupBoxes[0], $boxes[$i])) - self::boxArea($groupBoxes[0]); + $grow1 = self::boxArea(self::boxUnion($groupBoxes[1], $boxes[$i])) - self::boxArea($groupBoxes[1]); + $g = $grow0 < $grow1 || ($grow0 == $grow1 && count($groups[0]) <= count($groups[1])) ? 0 : 1; + } + $groups[$g][] = $item; + $groupBoxes[$g] = self::boxUnion($groupBoxes[$g], $boxes[$i]); + $left--; + } + $newId = $tree['next']++; + $tree['nodes'][$id] = [$level, $groupBoxes[0], $groups[0]]; + $tree['nodes'][$newId] = [$level, $groupBoxes[1], $groups[1]]; + if($tree['parent'] !== null){ + foreach($groups[1] as $item){ + if($level === 0){ + $tree['leafOf'][$item[0]] = $newId; + } else { + $tree['parent'][$item] = $newId; + } + } + } + if(empty($path)){ + $rootId = $tree['next']++; + $tree['nodes'][$rootId] = [$level + 1, self::boxUnion($groupBoxes[0], $groupBoxes[1]), [$id, $newId]]; + $tree['root'] = $rootId; + if($tree['parent'] !== null){ + $tree['parent'][$id] = $rootId; + $tree['parent'][$newId] = $rootId; + } + return; + } + $parentId = array_pop($path); + $tree['nodes'][$parentId][2][] = $newId; + if($tree['parent'] !== null){ + $tree['parent'][$newId] = $parentId; + } + if(count($tree['nodes'][$parentId][2]) > $tree['ns']){ + $this->rtreeSplit($tree, $parentId, $path); + } + } + + /** + * Remove a key; emptied nodes go, boxes shrink up the tree, a root with one child is replaced by it + * @return bool True when the key was indexed + */ + private function rtreeRemove(array &$tree, int $key): bool { + $this->rtreeMaps($tree); + if(!isset($tree['leafOf'][$key])){ + return false; + } + $id = $tree['leafOf'][$key]; + unset($tree['leafOf'][$key]); + $entries = $tree['nodes'][$id][2]; + foreach($entries as $i => $entry){ + if($entry[0] === $key){ + unset($entries[$i]); + break; + } + } + $tree['nodes'][$id][2] = array_values($entries); + $tree['cnt']--; + // Remove emptied nodes bottom-up + while($id !== $tree['root'] && empty($tree['nodes'][$id][2])){ + $parentId = $tree['parent'][$id]; + $children = $tree['nodes'][$parentId][2]; + $at = array_search($id, $children, true); + if($at !== false){ + unset($children[$at]); + } + $tree['nodes'][$parentId][2] = array_values($children); + unset($tree['nodes'][$id], $tree['parent'][$id]); + $id = $parentId; + } + // Recompute boxes up the tree while they change + while(true){ + [$level, $old, $items] = $tree['nodes'][$id]; + $box = null; + foreach($items as $item){ + $itemBox = $this->rtreeBoxOf($tree, $level, $item); + if($itemBox !== null){ + $box = self::boxUnion($box, $itemBox); + } + } + if($box === $old){ + break; + } + $tree['nodes'][$id][1] = $box; + if(!isset($tree['parent'][$id])){ + break; + } + $id = $tree['parent'][$id]; + } + // Collapse the root + while(true){ + [$level, , $items] = $tree['nodes'][$tree['root']]; + if($level === 0){ + break; + } + if(empty($items)){ + $tree['nodes'][$tree['root']] = [0, null, []]; + break; + } + if(count($items) > 1){ + break; + } + $child = $items[0]; + unset($tree['nodes'][$tree['root']]); + $tree['root'] = $child; + unset($tree['parent'][$child]); + } + return true; + } + + /** + * Keys whose boxes overlap one of the (integer) boxes + */ + private function rtreeSearch(array $tree, array $boxes): array { + $found = []; + $stack = [$tree['root']]; + while(!empty($stack)){ + $id = array_pop($stack); + [$level, $nodeBox, $items] = $tree['nodes'][$id]; + if($nodeBox === null){ + continue; + } + $hit = false; + foreach($boxes as $box){ + if(self::boxOverlaps($nodeBox, $box)){ + $hit = true; + break; + } + } + if(!$hit){ + continue; + } + foreach($items as $item){ + if($level > 0){ + $stack[] = $item; + continue; + } + foreach($boxes as $box){ + if($item[1] <= $box[2] && $box[0] <= $item[3] && $item[2] <= $box[3] && $box[1] <= $item[4]){ + $found[$item[0]] = $item[0]; + break; + } + } + } + } + return array_values($found); + } + + /** + * Apply one data line to an R-tree: the key now holds $record (null: deleted) + */ + private function rtreeApply(array &$tree, int $key, ?array $record): void { + $geometry = $record === null ? null : ($record[$tree['field']] ?? null); + $box = is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid'] ? $this->quantizedMBR($geometry) : null; + if($box === null){ + if($key <= $tree['mk']){ + $this->rtreeRemove($tree, $key); + } elseif($key > $tree['mk']){ + $tree['mk'] = $key; + } + return; + } + $this->rtreeInsert($tree, $key, $box); + } + + /** + * Build a segment's R-tree from its data (covers the key index's bytes) + */ + public function buildSegmentSpatialIndex(string $dbname, string $field, ?int $shardId, array $options = []): array { + $key = $this->secondary->keyIndexWithEpoch($dbname, $shardId); + $nodeSize = isset($options['nodeSize']) && (int)$options['nodeSize'] >= 4 ? (int)$options['nodeSize'] : $this->rtreeNodeSize; + $tree = $this->newRTree($field, $nodeSize); + $columns = [[], [], [], [], []]; // keys, then their boxes' minX, minY, maxX, maxY + foreach($this->records->eachSegmentBatch($dbname, $shardId) as $batch){ + foreach($batch as $k => $record){ + $geometry = $record[$field] ?? null; + if(is_array($geometry) && $this->geoJson->validateGeoJSON($geometry)['valid']){ + $box = $this->quantizedMBR($geometry); + if($box !== null){ + $columns[0][] = (int)$k; + $columns[1][] = $box[0]; + $columns[2][] = $box[1]; + $columns[3][] = $box[2]; + $columns[4][] = $box[3]; + } + } + } + } + $this->rtreeBulkLoad($tree, ...$columns); + $tree['mk'] = $key === null ? -1 : \noneDB\Internal\Storage\KeyIndex::maxKey($key); + $tree['s'] = $key === null ? 0 : (int)$key['s']; + $tree['e'] = $key === null ? null : $key['e']; + return $tree; + } + + /** + * A segment's R-tree, current (the data lines after the bytes it covers are replayed); null when + * none fits the segment (see ensureSegmentSpatialIndex) + */ + private function readSpatialIndex(string $dbname, string $field, ?int $shardId = null): ?array { + $key = $this->secondary->currentKeyIndex($dbname, $shardId); + if($key === null){ + return null; + } + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->spatialIndexCache[$path])){ + $tree = $this->spatialIndexCache[$path]; + if($tree['e'] !== null && $tree['e'] === $key['e'] && $tree['s'] <= (int)$key['s']){ + $this->metrics['hits']++; + if($tree['s'] < (int)$key['s']){ + unset($this->spatialIndexCache[$path], $tree['due']); + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + } elseif(isset($tree['due']) && $this->noWrites === 0 && $this->multiLock === 0){ + unset($tree['due'], $this->spatialIndexCache[$path]['due']); // a checkpoint a read-only load put off + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + } + return $this->spatialIndexCache[$path]; + } + unset($this->spatialIndexCache[$path]); + } + $this->metrics['misses']++; + $file = $this->secondary->readSecondaryFile($path); + if($file === null || !$this->secondary->secondaryFits($dbname, $shardId, $file[0], 'rx', $field)){ + return null; + } + $tree = $this->treeOf($field, $file[0], $file[1]); + if($tree === null){ + return null; + } + $this->replaySpatialIndex($dbname, $shardId, $tree, (int)$key['s']); + $this->spatialIndexCache[$path] = $tree; + $this->cacheGrew = true; + if($this->secondary->maybeSecondaryReaderCheckpoint($dbname, $shardId, 'rx', $field, $tree)){ + $this->spatialIndexCache[$path]['due'] = true; + } + return $this->spatialIndexCache[$path]; + } + + public function replaySpatialIndex(string $dbname, ?int $shardId, array &$tree, int $to): void { + foreach($this->secondary->segmentTail($dbname, $shardId, $tree['s'], $to) as [$k, $record]){ + $this->rtreeApply($tree, (int)$k, $record); + } + $tree['s'] = $to; + } + + /** + * Write an R-tree checkpoint + */ + public function writeSpatialIndex(string $dbname, string $field, array &$tree, ?int $shardId = null): bool { + if($tree['e'] === null){ + return false; + } + $path = $this->getSpatialIndexPath($dbname, $field, $shardId); + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$tree['s']); + if($crc === null){ + return false; + } + // One line of nodes (a leaf: its number of entries), then the entries of the leaves in that order + $nodes = []; + $entries = ''; + foreach($tree['nodes'] as $id => $node){ + if($node[0] === 0){ + $entries .= pack('P*', ...array_merge(...$node[2])); + $node[2] = count($node[2]); + } + $nodes[(string)$id] = $node; + } + $body = $this->io->encodeJson($nodes); + if($body === false){ + return false; + } + $body .= "\n" . $entries; + $rootBox = $tree['nodes'][$tree['root']][1]; + $this->secondary->writeSecondaryFile($dbname, $path, ['kind' => 'rx', 'v' => 1, 'field' => $field, 'q' => self::RTREE_SCALE, 'ns' => $tree['ns'], + 's' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'mk' => $tree['mk'], 'cnt' => $tree['cnt'], + 'root' => $tree['root'], 'next' => $tree['next'], 'mbr' => $rootBox, 'bc' => crc32($body)], $body); + $tree['ck'] = (int)$tree['s']; + $tree['bytes'] = strlen($body); + $this->spatialIndexCache[$path] = $tree; + $this->cacheGrew = true; + return true; + } + + /** + * The tree of an .rx file, or null when it holds none (STORAGE_FORMAT.md §6.5): the crc of the body doesn't fit, or a + * node isn't [level, box, items], where a box is four integers (null only without items), the items of a leaf a + * number of entries the body holds, the items above it nodes of a lower level, named once. So a walk meets every + * node it is sent to and ends. An entry is five integers by its format: no check per entry + */ + public function treeOf(string $field, array $header, string $body): ?array { + [$root, $next, $size] = [$header['root'] ?? null, $header['next'] ?? null, $header['ns'] ?? null]; + if(($header['v'] ?? null) !== 1 || ($header['bc'] ?? null) !== crc32($body) || !is_int($root) || !is_int($next) || !is_int($size) || $size < 4 + || !is_int($header['mk'] ?? null) || !is_int($header['cnt'] ?? null)){ + return null; + } + $end = (int)strpos($body, "\n"); + $nodes = json_decode(substr($body, 0, $end), true); + if(!is_array($nodes) || !isset($nodes[$root])){ + return null; + } + $entries = 0; + $named = []; + foreach($nodes as $id => $node){ + $level = $node[0] ?? null; + $box = $node[1] ?? null; + $items = $node[2] ?? null; + if($id >= $next || !is_int($level) || $level < 0 || ($box === null ? !empty($items) : !self::isBox($box))){ + return null; + } + if($level === 0){ + if(!is_int($items) || $items < 0){ + return null; + } + $entries += $items; + continue; + } + if(!is_array($items)){ + return null; + } + foreach($items as $item){ + if(!is_int($item) || ($nodes[$item][0] ?? $level) >= $level || isset($named[$item])){ + return null; + } + $named[$item] = true; + } + } + if($entries * 40 !== strlen($body) - $end - 1){ + return null; + } + $at = $end + 1; + foreach($nodes as $id => $node){ + if($node[0] === 0){ + $nodes[$id][2] = $node[2] === 0 ? [] : array_chunk(unpack('P' . 5 * $node[2], $body, $at), 5); + $at += 40 * $node[2]; + } + } + $tree = $this->newRTree($field, $size); + $tree['nodes'] = $nodes; + $tree['root'] = $root; + $tree['next'] = $next; + $tree['mk'] = $header['mk']; + $tree['cnt'] = $header['cnt']; + $tree['s'] = $header['s']; + $tree['ck'] = $header['s']; + $tree['e'] = $header['e']; + $tree['bytes'] = strlen($body); + return $tree; + } + + /** + * Read the global spatial index: ['le', 'sh' => [shard => ['s','e','h','b' => box or null]]] + */ + public function readGlobalSpatialIndex(string $dbname, string $field): ?array { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $this->cacheTouch[$path] = ++$this->cacheTick; + if(array_key_exists($path, $this->globalSpatialIndexCache)){ + $this->metrics['hits']++; + return $this->globalSpatialIndexCache[$path]; + } + $this->metrics['misses']++; + $header = $this->secondary->readSecondaryHeader($path); + $global = null; + if(is_array($header) && ($header['kind'] ?? null) === 'grx' && ($header['v'] ?? null) === 1 && ($header['field'] ?? null) === $field && is_array($header['sh'] ?? null)){ + $global = ['le' => $header['le'] ?? null, 'sh' => []]; + foreach($header['sh'] as $shardId => $entry){ + if(\noneDB\Internal\Index\SecondaryFiles::isEntry($entry) && array_key_exists('b', $entry) && ($entry['b'] === null || self::isBox($entry['b']))){ + $global['sh'][(int)$shardId] = $entry; + } + } + } + $this->globalSpatialIndexCache[$path] = $global; + $this->cacheGrew = true; + return $global; + } + + public function writeGlobalSpatialIndex(string $dbname, string $field, array $global): void { + $path = $this->getGlobalSpatialIndexPath($dbname, $field); + $lock = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $global['le'] = $lock === null ? null : $this->locks->registryEpoch($this->locks->getLockPath($dbname)); + $sh = []; + foreach($global['sh'] as $shardId => $entry){ + $sh[(string)$shardId] = $entry; + } + $this->secondary->writeSecondaryFile($dbname, $path, ['kind' => 'grx', 'v' => 1, 'field' => $field, 'q' => self::RTREE_SCALE, + 'le' => $global['le'], 'sh' => empty($sh) ? new \stdClass() : $sh], ''); + $this->globalSpatialIndexCache[$path] = $global; + unset($this->lockRegistry[$this->locks->getLockPath($dbname)]['globals'][$path]); + } + + /** + * Record a shard's box (at the bytes its R-tree covers) in the global spatial index + */ + public function updateGlobalSpatialShard(string $dbname, string $field, int $shardId, array $tree): void { + $entry = $this->globalSpatialEntry($dbname, $shardId, $tree); + if($entry === null){ + return; + } + $global = $this->secondary->freshGlobalEntries($dbname, $this->readGlobalSpatialIndex($dbname, $field) ?? ['le' => null, 'sh' => []]); + $lockPath = $this->locks->getLockPath($dbname); + if(($global['sh'][$shardId] ?? null) === $entry && $global['le'] === (isset($this->lockRegistry[$lockPath]) ? $this->locks->registryEpoch($lockPath) : null)){ + return; + } + $global['sh'][$shardId] = $entry; + $this->fieldIndex->deferGlobalWrite($dbname, 'grx', $field, $global); + } + + /** + * A shard's entry in the global spatial index: its R-tree's root box at the bytes it covers (null: + * none can be recorded) + */ + private function globalSpatialEntry(string $dbname, int $shardId, array $tree): ?array { + if($tree['e'] === null){ + return null; + } + $crc = $this->checkpoints->segmentCrc($this->segments->segDataPath($dbname, $shardId), (int)$tree['s']); + return $crc === null ? null : ['s' => (int)$tree['s'], 'e' => $tree['e'], 'h' => $crc, 'b' => $tree['nodes'][$tree['root']][1]]; + } + + /** + * A segment's R-tree: the current one, or built from the data (exclusive lock) + */ + public function ensureSegmentSpatialIndex(string $dbname, string $field, ?int $shardId): array { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if($tree !== null){ + return $tree; + } + if($this->noWrites > 0){ + throw new \RuntimeException("noneDB: no spatial index to read in a read-only check"); + } + return $this->locks->withLock($dbname, true, function () use ($dbname, $field, $shardId) { + $tree = $this->readSpatialIndex($dbname, $field, $shardId); + if ($tree === null) { + $nodeSize = $this->fieldIndex->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null; + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $shardId, $nodeSize ? ['nodeSize' => $nodeSize] : []); + $this->writeSpatialIndex($dbname, $field, $tree, $shardId); + if ($shardId !== null) { + $this->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); + } + } + return $tree; + }, 'buildSpatialIndex'); + } + + /** + * Keys of a segment whose geometry box may overlap the query boxes (float [minLon, minLat, maxLon, + * maxLat], or a list of them); every caller checks the exact geometry + */ + public function rtreeSearchMBR(string $dbname, string $field, array $searchMBR, ?int $shardId = null): array { + $tree = $this->ensureSegmentSpatialIndex($dbname, $field, $shardId); + $boxes = is_array($searchMBR[0] ?? null) ? $searchMBR : [$searchMBR]; + return $this->rtreeSearch($tree, array_map([$this, 'quantizeBox'], $boxes)); + } + + /** + * Rebuild a spatial index for the whole database (all segments + global shard boxes) + */ + public function rebuildSpatialIndexForDb($dbname, $field, $nodeSize = null){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $nodeSize = $nodeSize ?? $manifest['spatialOptions'][$field]['nodeSize'] ?? null; + $options = $nodeSize ? ['nodeSize' => $nodeSize] : []; + $segments = $this->segments->listSegments($dbname); + if(!in_array(null, $segments, true)){ + $segments[] = null; // leftovers of the non-sharded database + } + foreach($segments as $segment){ + $path = $this->getSpatialIndexPath($dbname, $field, $segment); + $this->io->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + foreach([$this->getGlobalSpatialIndexPath($dbname, $field)] as $path){ + $this->io->fsUnlink($path); + unset($this->globalSpatialIndexCache[$path]); + } + $indexed = 0; + $sharded = $this->segments->isSharded($dbname); + $global = ['le' => null, 'sh' => []]; // written once at the end (once per shard was O(shards²)) + foreach ($this->segments->listSegments($dbname) as $segment) { + $tree = $this->buildSegmentSpatialIndex($dbname, $field, $segment, $options); + $this->writeSpatialIndex($dbname, $field, $tree, $segment); + $indexed += $tree['cnt']; + if ($sharded && ($entry = $this->globalSpatialEntry($dbname, $segment, $tree)) !== null) { + $global['sh'][$segment] = $entry; + } + unset($tree); + $this->cache->evictBetweenSegments(); + } + if ($sharded) { + ksort($global['sh']); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + return $indexed; + } + + /** + * Delete the spatial index files of one shard + */ + public function removeSpatialShardFiles($dbname, $shardId){ + foreach ($this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'spatial/s' . (int)$shardId . '.', '/\.rx$/') as $file) { + $this->io->fsUnlink($file); + unset($this->spatialIndexCache[$file]); + } + foreach ($this->fieldIndex->getIndexManifest($dbname)['spatial'] as $field) { + $global = $this->readGlobalSpatialIndex($dbname, $field); + if ($global !== null && isset($global['sh'][(int)$shardId])) { + unset($global['sh'][(int)$shardId]); + $this->writeGlobalSpatialIndex($dbname, $field, $global); + } + } + } + + /** + * True when a shard can be skipped for the query boxes: its global entry fits the shard, its box + * misses every box, and the shard's newer lines hold no geometry in them + */ + public function globalSpatialSkips(string $dbname, string $field, int $shardId, ?array $global, array $boxes): bool { + $entry = $global['sh'][$shardId] ?? null; + if($entry === null || !$this->fieldIndex->globalEntryFits($dbname, $shardId, $entry, $global['le'])){ + return false; + } + $quantized = array_map([$this, 'quantizeBox'], $boxes); + foreach($quantized as $box){ + if(self::boxOverlaps($entry['b'], $box)){ + return false; + } + } + return !$this->fieldIndex->shardTailMayMatch($dbname, $shardId, $entry, function($record) use ($field, $quantized){ + $geometry = $record[$field] ?? null; + if(!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid']){ + return false; + } + $box = $this->quantizedMBR($geometry); + foreach($quantized as $q){ + if(self::boxOverlaps($box, $q)){ + return true; + } + } + return false; + }); + } + + /** + * Check if a spatial index exists + * @param string $dbname Database name + * @param string $field Field name + * @param int|null $shardId Shard ID + * @return bool + */ + public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $shardId){ + return $this->hasSpatialIndexUnlocked($dbname, $field, $shardId); + }); + } + + private function hasSpatialIndexUnlocked(string $dbname, string $field, ?int $shardId = null): bool{ + if (!$this->segments->dbExists($dbname) || !in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true)) { + return false; + } + return $shardId === null || in_array($shardId, $this->segments->listSegments($dbname), true); + } + // ========================================== + // SPATIAL INDEX - R-TREE ENGINE + // ========================================== + // One R-tree per segment and indexed field: spatial/{seg}.{token}.rx (see SecondaryFiles), a + // checkpoint like the field indexes: + // {"kind":"rx","v":1,"field":"loc","q":10000000,"ns":32,"s":…,"e":…,"h":…,"mk":…,"cnt":…,"root":3,"next":9} + // {"0":[0,[minX,minY,maxX,maxY],[[key,minX,minY,maxX,maxY],…]],"3":[1,[…],[0,2]],…} + // A node is [level, box, payload]: level 0 holds entries [key, box], a higher level child node ids. + // Boxes are integers: degrees * 1e7 rounded outwards (1e-7° is about 1 cm), so they don't depend on + // serialize_precision, and every query re-checks the exact geometry. Built with sort-tile-recursive + // bulk loading; changes after the checkpoint are replayed from the data when the index is used. + // Sharded databases keep spatial/{token}.grx: each shard's box (null = empty) as of the bytes its + // entry covers; a shard is skipped only when its entry fits, its box misses the query and the shard's + // newer lines hold no geometry in it. + + public const RTREE_SCALE = 10000000; +} diff --git a/src/Internal/Index/Spatial/SpatialIndexApi.php b/src/Internal/Index/Spatial/SpatialIndexApi.php new file mode 100644 index 0000000..339d99b --- /dev/null +++ b/src/Internal/Index/Spatial/SpatialIndexApi.php @@ -0,0 +1,198 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->db, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->globalSpatialIndexCache = &$this->k->globalSpatialIndexCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->db = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL INDEX - PUBLIC API + // ========================================== + + /** + * Create a spatial index on a GeoJSON field + * @param string $dbname Database name + * @param string $field Field name containing GeoJSON + * @param array $options Options ['nodeSize' => 32] + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + */ + public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['nodeSize'], 'option', ' of createSpatialIndex()'); + if(array_key_exists('nodeSize', $options) && (!is_int($options['nodeSize']) || $options['nodeSize'] < 4)){ + throw new \InvalidArgumentException("noneDB: 'nodeSize' must be an integer of at least 4"); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field, $options){ + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + }, 'createSpatialIndex'); + } + + private function createSpatialIndexUnlocked(string $dbname, string $field, array $options = []): array{ + if (!$this->spatialIndexEnabled) { + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + if ($field === '' || $field === 'key') { + return ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + } + $this->buffer->flushPendingBuffers($dbname); + if (!$this->segments->dbExists($dbname)) { + if (!$this->autoCreateDB) { + return ['success' => false, 'indexed' => 0, 'error' => 'Database does not exist']; + } + $this->db->createDBUnlocked($dbname); + } + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if (in_array($field, $manifest['spatial'], true)) { + return ['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field '$field'"]; + } + // Build first, list it last: an index that fails half-way is never listed + $nodeSize = $options['nodeSize'] ?? null; + $indexed = $this->rtree->rebuildSpatialIndexForDb($dbname, $field, $nodeSize); + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['spatial'][] = $field; + if ($nodeSize !== null) { + $manifest['spatialOptions'][$field] = ['nodeSize' => $nodeSize]; + } + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + return ['success' => true, 'indexed' => $indexed, 'error' => null]; + } + + /** + * Drop a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => ?string] + */ + public function dropSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->dropSpatialIndexUnlocked($dbname, $field); + }, 'dropSpatialIndex'); + } + + private function dropSpatialIndexUnlocked(string $dbname, string $field): array{ + foreach (array_merge([null], $this->segments->isSharded($dbname) ? $this->segments->listSegments($dbname) : []) as $segment) { + foreach ([$this->rtree->getSpatialIndexPath($dbname, $field, $segment)] as $path) { + $this->io->fsUnlink($path); + unset($this->spatialIndexCache[$path]); + } + } + foreach ([$this->rtree->getGlobalSpatialIndexPath($dbname, $field)] as $globalPath) { + $this->io->fsUnlink($globalPath); + unset($this->globalSpatialIndexCache[$globalPath]); + } + + if ($this->segments->dbExists($dbname)) { + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $manifest['spatial'] = array_values(array_diff($manifest['spatial'], [$field])); + unset($manifest['spatialOptions'][$field]); + $this->fieldIndex->saveIndexManifest($dbname, $manifest); + } + return ['success' => true, 'error' => null]; + } + + /** + * Get list of spatial indexes for a database + * @param string $dbname Database name + * @return array List of indexed field names + */ + public function getSpatialIndexes(string $dbname): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getSpatialIndexesUnlocked($dbname); + }); + } + + private function getSpatialIndexesUnlocked(string $dbname): array{ + return $this->segments->dbExists($dbname) ? $this->fieldIndex->getIndexManifest($dbname)['spatial'] : []; + } + + /** + * Rebuild a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createSpatialIndex + */ + public function rebuildSpatialIndex(string $dbname, string $field): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname, $field){ + return $this->rebuildSpatialIndexUnlocked($dbname, $field); + }, 'rebuildSpatialIndex'); + } + + private function rebuildSpatialIndexUnlocked(string $dbname, string $field): array{ + if (!$this->spatialIndexEnabled) { + // Checked before dropping anything: a drop must not succeed where the create then fails + return ['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled']; + } + $options = []; + $nodeSize = $this->segments->dbExists($dbname) ? ($this->fieldIndex->getIndexManifest($dbname)['spatialOptions'][$field]['nodeSize'] ?? null) : null; + if ($nodeSize) { + $options['nodeSize'] = $nodeSize; + } + $this->dropSpatialIndexUnlocked($dbname, $field); + return $this->createSpatialIndexUnlocked($dbname, $field, $options); + } +} diff --git a/src/Internal/Maintenance/BackupRestore.php b/src/Internal/Maintenance/BackupRestore.php new file mode 100644 index 0000000..026b96d --- /dev/null +++ b/src/Internal/Maintenance/BackupRestore.php @@ -0,0 +1,627 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->integrity, $this->io, $this->layout, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->shardInfo, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->integrity = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->shardInfo = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ---------------------------------------------------------------- backup (§10.2) + + /** + * Copy a database, as it is at one moment, into $dir (STORAGE_FORMAT.md §10.2) + * @return array ['database', 'dir', 'databaseId', 'generation', 'nextKey', 'records', 'bufferedRecords', 'files', 'bytes'] + * @throws \InvalidArgumentException $dir inside dbDir, or not an empty directory + */ + public function backup($dbname, $dir){ + $dbname = $this->layout->normalizeDbName($dbname); + $target = $this->backupTarget($dir); + if(!$this->store->openStore(false)){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist"); + } + // STORE.LOCK, then the database's LOCK, both shared until the copy is complete: one moment of the + // database and of the store record + return $this->store->withStoreLock(LOCK_SH, function() use ($dbname, $target){ + $this->store->checkMaintenanceJournal(); + $store = $this->store->loadStoreRecord(); + if($store === null || !in_array($this->store->keyCheck(), $store['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + return $this->locks->withLock($dbname, false, function() use ($dbname, $target, $store){ + $this->scope->refuseInUncommittedWrite($dbname, 'backup'); + if(!$this->layout->dbPresent($dbname)){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist"); + } + $this->noWrites++; // nothing of the database is written: no flush, no checkpoint + try { + return $this->copyDatabase($dbname, $target, $store); + } finally { + $this->noWrites--; + } + }); + }); + } + + /** + * The backup directory resolved (§10.2 step 1): outside dbDir once links are resolved, absent or empty + * @throws \InvalidArgumentException + */ + private function backupTarget($dir){ + if(!is_string($dir) || trim($dir) === ''){ + throw new \InvalidArgumentException('noneDB: backup() needs a directory'); + } + $resolved = \noneDB\Internal\Config\Configuration::canonicalPath($dir); + $root = rtrim($this->dbDir, '/\\'); // canonical (Configuration::resolveDbDir()) + if($resolved === $root || strncmp($resolved . DIRECTORY_SEPARATOR, $root . DIRECTORY_SEPARATOR, strlen($root) + 1) === 0){ + throw new \InvalidArgumentException("noneDB: the backup directory {$dir} must lie outside dbDir"); + } + if(file_exists($resolved) && (!is_dir($resolved) || count(scandir($resolved) ?: []) > 2)){ + throw new \InvalidArgumentException("noneDB: the backup directory {$dir} must not exist or be an empty directory"); + } + return $resolved; + } + + /** + * The copy (§10.2 steps 3-5) under both locks; what it created is removed when it fails + */ + private function copyDatabase($dbname, $target, array $store){ + $takenAt = time(); + $home = $this->layout->dbHome($dbname); + $generation = $this->layout->generationNumber($dbname); + $genDir = $this->layout->genDir($dbname); + $format = $this->layout->readFormatRecord($dbname); + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + if(!is_array($info)){ + throw new \RuntimeException("noneDB: the counters of database '{$dbname}' can't be read"); + } + $sources = ['CURRENT' => $home . 'CURRENT', 'FORMAT' => $home . 'FORMAT']; + foreach($this->filesUnder($genDir) as $file){ + if(preg_match(self::GENERATION_FILE, $file) === 1){ + $sources['generations/' . $generation . '/' . $file] = $genDir . $file; + } + } + foreach($this->filesUnder($home . 'recovery/') as $file){ + if(preg_match(self::RECOVERY_FILE, 'recovery/' . $file) === 1 && preg_match('/tmp[0-9a-f]{8}$/', $file) !== 1){ + $sources['recovery/' . $file] = $home . 'recovery/' . $file; + } + } + $buffered = 0; + foreach($sources as $path => $source){ + if(preg_match('#^generations/[0-9]+/buffer/#', $path) === 1){ + $buffered += count($this->buffer->readBufferRecords($source)); + } + } + $created = []; + try { + $this->makeBackupDirectory($target, $created); + $files = []; + foreach($sources as $path => $source){ + // A data or buffer file up to its last "\n": an unterminated tail is never a record (§6.1) + clearstatcache(true, $source); + $length = preg_match(self::COMPLETE_LINES, $path) === 1 ? $this->checkpoints->lastNewlineIn($source, 0, (int)@filesize($source)) + 1 + : (int)@filesize($source); + $files[$path] = $this->writeBackupFile($target . '/' . $path, $this->chunks($source, $length, null, ''), $created); + $this->recovery->fault('backup:copied', ['path' => $path]); // @test + } + $files['store.json'] = $this->writeBackupFile($target . '/store.json', [\noneDB\Internal\Storage\StoreRecord::sealRecord($store)], $created); + $paths = array_keys($files); + sort($paths, SORT_STRING); + $entries = []; + foreach($paths as $path){ + [$size, $sha256] = $files[$path]; + if(!$this->fileHolds($target . '/' . $path, $size, $sha256, '')){ + throw new \RuntimeException("noneDB: the backup copy of {$path} in {$target} does not read back as it was written"); + } + $entries[] = ['path' => $path, 'size' => $size, 'sha256' => $sha256]; + } + $this->recovery->fault('backup:verified', []); // @test + foreach(array_unique(array_map('dirname', array_map(function($path) use ($target){ return $target . '/' . $path; }, $paths))) as $directory){ + $this->sync->requireSync(null, $directory, 'dir'); + } + $manifest = ['magic' => 'NONEDB4', 'kind' => 'backup', 'storageFormat' => \noneDB::STORAGE_FORMAT, + 'databaseId' => $format['databaseId'], 'name' => $dbname, 'generation' => $generation, + 'nextKey' => (int)$info['nextKey'], 'records' => (int)$info['totalRecords'], 'bufferedRecords' => $buffered, + 'takenAt' => $takenAt, 'files' => $entries]; + $tmp = $target . '/manifest.json.tmp' . bin2hex(random_bytes(4)); + $this->writeBackupFile($tmp, [\noneDB\Internal\Storage\StoreRecord::sealRecord($manifest)], $created); + if(!$this->sync->renameWithRetry($tmp, $target . '/manifest.json')){ + throw new \RuntimeException("noneDB: cannot write the manifest of the backup in {$target}"); + } + $created[] = $target . '/manifest.json'; + $this->sync->requireSync(null, $target, 'dir'); + } catch(\Throwable $e){ + $this->removeBackupFiles($created); + throw $e; + } + return ['database' => $dbname, 'dir' => $target, 'databaseId' => $format['databaseId'], 'generation' => $generation, + 'nextKey' => $manifest['nextKey'], 'records' => $manifest['records'], 'bufferedRecords' => $buffered, + 'files' => count($entries), 'bytes' => array_sum(array_column($entries, 'size'))]; + } + + /** + * Regular files under a database directory, relative, sorted; a symbolic link is an error + * @return string[] + */ + private function filesUnder($dir){ + $files = []; + $walk = function($at, $prefix) use (&$walk, &$files){ + foreach(@scandir($at) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(is_link($at . $entry)){ + throw new \RuntimeException("noneDB: {$at}{$entry} is a symbolic link; a database holds none"); + } + if(is_dir($at . $entry)){ + $walk($at . $entry . '/', $prefix . $entry . '/'); + } elseif(is_file($at . $entry)){ + $files[] = $prefix . $entry; + } + } + }; + $walk($dir, ''); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + sort($files, SORT_STRING); + return $files; + } + + // The backup directory belongs to no database: its files are written here (0700 directories, 0600 + // files), synced in durable mode; what a failed backup created is removed + + /** + * Create a directory of a backup and its missing parents (0700), recording the ones created + */ + private function makeBackupDirectory($dir, array &$created){ + $missing = []; + for($at = $dir; !is_dir($at); $at = dirname($at)){ + $missing[] = $at; + } + foreach(array_reverse($missing) as $new){ + if(!@mkdir($new, 0700) && !is_dir($new)){ + throw new \RuntimeException("noneDB: cannot create the backup directory {$new}"); + } + $created[] = $new; + $this->sync->requireSync(null, dirname($new), 'dir'); + } + @chmod($dir, 0700); + } + + /** + * Write a new file of a backup from $chunks + * @return array [size, SHA-256] of the bytes written + */ + private function writeBackupFile($path, iterable $chunks, array &$created){ + if(!is_dir(dirname($path))){ + $this->makeBackupDirectory(dirname($path), $created); + } + $fp = @fopen($path, 'xb'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot create the backup file {$path}"); + } + $created[] = $path; + @chmod($path, 0600); + try { + $hash = hash_init('sha256'); + $size = 0; + foreach($chunks as $chunk){ + if(fwrite($fp, $chunk) !== strlen($chunk)){ + throw new \RuntimeException("noneDB: cannot write the backup file {$path}"); + } + hash_update($hash, $chunk); + $size += strlen($chunk); + } + if(!fflush($fp)){ + throw new \RuntimeException("noneDB: cannot write the backup file {$path}"); + } + $this->sync->requireSync($fp, $path, 'file'); + } finally { + fclose($fp); + } + return [$size, hash_final($hash)]; + } + + private function removeBackupFiles(array $created){ + foreach(array_reverse($created) as $path){ + if(is_dir($path) && !is_link($path)){ + @rmdir($path); + } else { + @unlink($path); + } + } + } + + // ---------------------------------------------------------------- restore (§10.3) + + /** + * Make a backup the content of a database, with one commit (STORAGE_FORMAT.md §10.3) + * @param array $options ['force' => bool] + * @return array ['database', 'generation', 'nextKey', 'records', 'bufferedRecords', 'target' => 'present'|'absent'|'damaged', 'foreign' => bool] + * @throws \RuntimeException RESTORE_REFUSED (nothing changed) + */ + public function restore($dir, $dbname, array $options){ + $dbname = $this->layout->normalizeDbName($dbname); + \noneDB\Internal\Config\Configuration::checkNames($options, ['force'], 'option', ' of restore()'); + $force = array_key_exists('force', $options) && \noneDB\Internal\Config\Configuration::optionBool('force', $options['force']); + $backup = $this->readBackup($dir); + // A refusal is decided before anything is written: first under the read lock (a missing database is + // answered without creating anything), then again under the write lock + $this->locks->withLock($dbname, false, function() use ($dbname, $backup, $force){ + $this->restoreIdentity($dbname, $backup, $force); + }, 'restore'); + return $this->locks->withLock($dbname, true, function() use ($dbname, $backup, $force){ + $this->scope->refuseInUncommittedWrite($dbname, 'restore'); + [$state, $identity, $foreign] = $this->restoreIdentity($dbname, $backup, $force); + if($state === 'present'){ + $info = $this->shardInfo->getShardInfoUnlocked($dbname); + $nextKey = is_array($info) ? (int)$info['nextKey'] : 0; + } else { + $nextKey = $state === 'absent' ? 0 : $this->damagedNextKey($dbname); + } + $prepare = function($dir, $generation) use ($dbname, $backup, $identity, &$nextKey){ + $nextKey = $this->prepareRestored($dbname, $dir, $generation, $backup, $identity, $nextKey); + }; + if($state === 'present'){ + $generation = $this->layout->switchGeneration($dbname, $prepare); + } elseif($state === 'absent'){ + $this->layout->createDatabaseFiles($dbname, $identity, $prepare); + $generation = 1; + } else { + $generation = $this->layout->replaceDamagedGeneration($dbname, $identity, $prepare); + } + $this->cache->invalidateDbCaches($dbname); + $manifest = $backup['manifest']; + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => $foreign ? 'foreign-restore' : 'restore', 'generation' => $generation, + 'databaseId' => $manifest['databaseId'], 'takenAt' => $manifest['takenAt'], 'records' => $manifest['records']]); + return ['database' => $dbname, 'generation' => $generation, 'nextKey' => $nextKey, 'records' => $manifest['records'], + 'bufferedRecords' => $manifest['bufferedRecords'], 'target' => $state, 'foreign' => $foreign]; + }, 'restore'); + } + + /** + * A valid backup (§10.1): ['dir', 'manifest', 'format', 'metadata'] + * @throws \RuntimeException RESTORE_REFUSED + */ + private function readBackup($dir){ + $root = is_string($dir) && $dir !== '' ? realpath($dir) : false; + $manifest = $root === false ? null : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/manifest.json'), 'backup'); + if($manifest === null || !$this->validManifest($manifest)){ + throw $this->refused($dir, 'it has no valid manifest.json (an interrupted backup has none)'); + } + $generation = $manifest['generation']; + $previous = null; + foreach($manifest['files'] as $entry){ + $path = $entry['path']; + if(!$this->backupFile($path, $generation) || ($previous !== null && strcmp($previous, $path) >= 0)){ + throw $this->refused($dir, "its manifest lists {$path}, which is not a file of a backup in its place"); + } + $previous = $path; + $file = $root . DIRECTORY_SEPARATOR . str_replace('/', DIRECTORY_SEPARATOR, $path); + if(realpath($file) !== $file || !$this->fileHolds($file, $entry['size'], $entry['sha256'], '')){ + throw $this->refused($dir, "{$path} does not match its manifest"); + } + } + $listed = array_flip(array_column($manifest['files'], 'path')); + foreach(['FORMAT', 'CURRENT', 'store.json', "generations/{$generation}/metadata"] as $required){ + if(!isset($listed[$required])){ + throw $this->refused($dir, "it has no {$required}"); + } + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/FORMAT'), 'database'); + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($root . '/CURRENT'), 'current'); + $metadata = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents("{$root}/generations/{$generation}/metadata"), 'generation'); + if($format === null || !$this->layout->validIdentity($format, $manifest['name']) || $format['databaseId'] !== $manifest['databaseId'] + || $current === null || ($current['databaseId'] ?? null) !== $format['databaseId'] || ($current['generation'] ?? null) !== $generation + || $metadata === null || ($metadata['generation'] ?? null) !== $generation || !$this->layout->validIdentity($metadata, $manifest['name']) + || json_encode($this->layout->identityOf($metadata)) !== json_encode($this->layout->identityOf($format)) + || !in_array($metadata['layout'] ?? null, ['single', 'sharded'], true)){ + throw $this->refused($dir, 'its FORMAT, CURRENT and metadata do not agree with its manifest'); + } + return ['dir' => $root, 'manifest' => $manifest, 'format' => $format, 'metadata' => $metadata]; + } + + private function validManifest(array $manifest){ + foreach(['generation' => 1, 'nextKey' => 0, 'records' => 0, 'bufferedRecords' => 0, 'takenAt' => 0] as $member => $least){ + if(!is_int($manifest[$member] ?? null) || $manifest[$member] < $least){ + return false; + } + } + if(($manifest['storageFormat'] ?? null) !== \noneDB::STORAGE_FORMAT || !is_string($manifest['databaseId'] ?? null) + || !is_string($manifest['name'] ?? null) || !is_array($manifest['files'] ?? null) || !array_is_list($manifest['files'])){ + return false; + } + foreach($manifest['files'] as $entry){ + if(!is_array($entry) || array_keys($entry) !== ['path', 'size', 'sha256'] || !is_string($entry['path']) + || !is_int($entry['size']) || $entry['size'] < 0 || !is_string($entry['sha256']) || preg_match('/^[0-9a-f]{64}$/', $entry['sha256']) !== 1){ + return false; + } + } + return true; + } + + /** + * A path a backup of generation $generation may hold (§10.1) + */ + private function backupFile($path, $generation){ + $prefix = 'generations/' . $generation . '/'; + return in_array($path, ['FORMAT', 'CURRENT', 'store.json'], true) || preg_match(self::RECOVERY_FILE, $path) === 1 + || (strncmp($path, $prefix, strlen($prefix)) === 0 && preg_match(self::GENERATION_FILE, substr($path, strlen($prefix))) === 1); + } + + private function refused($dir, $why){ + return new \RuntimeException('noneDB: the backup in ' . (is_string($dir) ? $dir : '(no directory)') . " can't be restored: {$why}", \noneDB::RESTORE_REFUSED); + } + + /** + * The identity a restore gives the database (§10.3 step 3), decided before anything is written + * @return array [target state, identity, foreign] + * @throws \RuntimeException RESTORE_REFUSED + */ + private function restoreIdentity($dbname, array $backup, $force){ + $state = $this->layout->restoreTargetState($dbname); + $theirs = $backup['format']; + $format = $state === 'absent' ? null : $this->layout->readFormatRecord($dbname); + if($format !== null){ + $foreign = $format['databaseId'] !== $theirs['databaseId']; + $identity = $this->layout->identityOf($format); + } else { + $foreign = $theirs['name'] !== $dbname; + $identity = $this->layout->identityOf($theirs); + if($foreign){ + $identity['databaseId'] = \noneDB\Internal\Storage\StoreRecord::uuid4(); + $identity['name'] = $dbname; + $identity['createdAt'] = time(); + } + } + if($foreign && !$force){ + throw new \RuntimeException("noneDB: the backup in {$backup['dir']} is of another database ('{$theirs['name']}'); " + . "restore it with ['force' => true] to load it as database '{$dbname}'", \noneDB::RESTORE_REFUSED); + } + if($backup['metadata']['layout'] === 'sharded' && $identity['shardSize'] !== $theirs['shardSize']){ + throw new \RuntimeException("noneDB: the backup in {$backup['dir']} has shards of {$theirs['shardSize']} records, database " + . "'{$dbname}' of {$identity['shardSize']}; restore it into a database of its own", \noneDB::RESTORE_REFUSED); + } + return [$state, $identity, $foreign]; + } + + /** + * Next key of a damaged database: 1 + the highest key of any complete line in any of its segment files + * (the keys it handed out are not known otherwise, §10.3 step 4.3) + */ + private function damagedNextKey($dbname){ + $next = 0; + $root = $this->layout->dbHome($dbname) . 'generations/'; + foreach(@scandir($root) ?: [] as $generation){ + if(!ctype_digit((string)$generation)){ + continue; + } + foreach(@scandir($root . $generation . '/segments') ?: [] as $file){ + if(preg_match('/^' . self::SEGMENT . '\.jsonl$/', $file) !== 1){ + continue; + } + $fp = @fopen($root . $generation . '/segments/' . $file, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), 0, null)) !== null){ + $next = max($next, $entry[1] + 1); + } + } + if($fp !== false){ + fclose($fp); + } + } + } + return $next; + } + + /** + * PREPARE of a restore (§10.3 step 4): generation $generation in $dir as the backup's copy with its + * metadata, the next key kept, then verified with checkIntegrity() over it + * @return int the next key of the restored database + * @throws \RuntimeException RESTORE_REFUSED when the generation does not verify + */ + private function prepareRestored($dbname, $dir, $generation, array $backup, array $identity, $nextKey){ + $manifest = $backup['manifest']; + $layout = $backup['metadata']['layout']; + $prefix = 'generations/' . $manifest['generation'] . '/'; + $top = max($nextKey, $manifest['nextKey']); + // Keys the database handed out after the backup stay used: a top tombstone of the highest one + $tombstone = ''; + $tombstoned = null; + if($nextKey > $manifest['nextKey']){ + $tombstone = '{"__nonedb_deleted":' . ($nextKey - 1) . "}\n"; + $tombstoned = 'segments/' . ($layout === 'single' ? 'main' : 's' . intdiv($nextKey - 1, $identity['shardSize'])) . '.jsonl'; + } + $copied = []; + foreach($manifest['files'] as $entry){ + if(strncmp($entry['path'], $prefix, strlen($prefix)) !== 0 || $entry['path'] === $prefix . 'metadata'){ + continue; + } + $file = substr($entry['path'], strlen($prefix)); + $this->copyIntoGeneration($backup['dir'] . '/' . $entry['path'], $entry['size'], $entry['sha256'], $dir . $file, $file === $tombstoned ? $tombstone : ''); + $copied[$file] = true; + $this->recovery->fault('restore:copied', ['path' => $file]); // @test + } + if($tombstoned !== null && !isset($copied[$tombstoned])){ + $this->copyIntoGeneration(null, 0, hash('sha256', ''), $dir . $tombstoned, $tombstone); // a new shard holding it + } + $this->layout->writeGenerationMetadata($dbname, $generation, $identity, $layout); + if($layout === 'sharded' && $tombstoned !== null){ + $meta = json_decode((string)@file_get_contents($dir . 'shards.json'), true); + if(!is_array($meta) || !is_array($meta['shards'] ?? null)){ + throw new \RuntimeException("noneDB: the shard list of the backup in {$backup['dir']} is unreadable", \noneDB::RESTORE_REFUSED); + } + $shard = intdiv($nextKey - 1, $identity['shardSize']); + if(!in_array($shard, array_map('intval', array_column($meta['shards'], 'id')), true)){ + $meta['shards'][] = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shard); + usort($meta['shards'], function($a, $b){ return (int)$a['id'] <=> (int)$b['id']; }); + } + $meta['nextKey'] = $nextKey; + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($dir . 'shards.json', $json . "\n", 'durable')){ + throw new \RuntimeException("noneDB: cannot write the shard list of the restored generation of database '{$dbname}'"); + } + } + [$report, $info] = $this->layout->withGenerationView($dbname, $generation, $identity, function() use ($dbname){ + return [$this->integrity->integrityReport($dbname), $this->shardInfo->getShardInfoUnlocked($dbname)]; + }); + $meta = array_filter($report['indexMismatches'], function($mismatch){ return $mismatch['index'] === 'meta'; }); + $restoredKey = is_array($info) ? (int)$info['nextKey'] : -1; + if($report['records'] !== $manifest['records'] || $report['bufferedRecords'] !== $manifest['bufferedRecords'] || !empty($meta) || $restoredKey !== $top){ + throw new \RuntimeException("noneDB: the restored generation of database '{$dbname}' does not verify: {$report['records']} records of " + . "{$manifest['records']}, {$report['bufferedRecords']} buffered of {$manifest['bufferedRecords']}, next key {$restoredKey} of {$top}" + . (empty($meta) ? '' : ', the shard list does not fit the data') . '; nothing was restored', \noneDB::RESTORE_REFUSED); + } + $this->recovery->fault('restore:verified', ['db' => $dbname]); // @test + return $top; + } + + /** + * Write a file of the restored generation from the backup's bytes (and $suffix) with the durability of + * its kind (DURABILITY.md §3), then read it back + */ + private function copyIntoGeneration($source, $size, $sha256, $target, $suffix){ + $file = basename(dirname($target)) . '/' . basename($target); + $mode = preg_match('#(?:^|/)(?:segments|buffer)/[^/]+\.jsonl$#', $file) === 1 ? 'replace' + : (in_array(basename($target), ['shards.json', 'indexes.json'], true) ? 'durable' : 'derived'); + if(!$this->io->writeFileAtomic($target, $this->chunks($source, $size, $sha256, $suffix), $mode)){ + throw new \RuntimeException("noneDB: cannot write {$file} of the restored generation"); + } + if(!$this->fileHolds($target, $size, $sha256, $suffix)){ + throw new \RuntimeException("noneDB: {$file} of the restored generation does not read back as it was copied"); + } + } + + // ---------------------------------------------------------------- bytes + + /** + * The first $length bytes of $path in chunks (checked against $sha256 when one is given), then $suffix + */ + private function chunks($path, $length, $sha256, $suffix){ + if($path !== null){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot read {$path}"); + } + try { + $hash = hash_init('sha256'); + for($left = $length; $left > 0; $left -= strlen($chunk)){ + $chunk = (string)fread($fp, min(self::CHUNK, $left)); + if($chunk === ''){ + throw new \RuntimeException("noneDB: {$path} changed while it was copied"); + } + hash_update($hash, $chunk); + yield $chunk; + } + if($sha256 !== null && hash_final($hash) !== $sha256){ + throw new \RuntimeException("noneDB: {$path} changed while it was copied", \noneDB::RESTORE_REFUSED); + } + } finally { + fclose($fp); + } + } + if($suffix !== ''){ + yield $suffix; + } + } + + /** + * A regular file of $size bytes with SHA-256 $sha256, followed by $suffix exactly + */ + private function fileHolds($path, $size, $sha256, $suffix){ + clearstatcache(true, $path); + if(!is_file($path) || is_link($path) || filesize($path) !== $size + strlen($suffix)){ + return false; + } + $fp = @fopen($path, 'rb'); + if($fp === false){ + return false; + } + $hash = hash_init('sha256'); + $read = $size > 0 ? hash_update_stream($hash, $fp, $size) : 0; + $rest = (string)stream_get_contents($fp); + fclose($fp); + return $read === $size && hash_final($hash) === $sha256 && $rest === $suffix; + } +} diff --git a/src/Internal/Maintenance/Compaction.php b/src/Internal/Maintenance/Compaction.php new file mode 100644 index 0000000..f0b39d8 --- /dev/null +++ b/src/Internal/Maintenance/Compaction.php @@ -0,0 +1,250 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->indexCache = &$this->k->indexCache; + $this->autoCompactThreshold = &$this->k->autoCompactThreshold; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->metrics = &$this->rt->metrics; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * After a write that left old lines in a segment ($index: the segment's index as the write left it). + * Compaction replaces the data file, so it runs after the write committed (see the rollback scope); whether it + * is due is decided here, with the index in hand: a write over many segments doesn't load each evicted index + * again only to find nothing due + */ + public function deferCompaction($dbname, $shardId, array $index){ + if(!$this->compactionDue($index)){ + return; + } + $this->scope->runIndependent($dbname, 'compact:' . ($shardId === null ? 'main' : $shardId), function() use ($dbname, $shardId){ + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($index !== null && $this->compactionDue($index)){ + $this->compactJsonl($dbname, $shardId); + } + }); + } + + /** True once a segment's garbage (old versions, tombstones) exceeds the threshold share of its lines */ + private function compactionDue(array $index){ + $garbage = (int)$index['d']; + return $garbage > 0 && $garbage > (\noneDB\Internal\Storage\KeyIndex::count($index) + $garbage) * $this->autoCompactThreshold; + } + + /** + * Compact a segment: rewrite live records (keys and order preserved), drop old versions and + * tombstones. Crash-safe: the data file is replaced first; if the index rename doesn't happen, + * the shorter data file is detected and the index rebuilt from it. + * @param string $dbname + * @param int|null $shardId + * @return array ['compacted' => int, 'freed' => int] + */ + public function compactJsonl($dbname, $shardId = null){ + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId){ + $started = microtime(true); + $this->scope->refuseUncommittedFile($this->segments->segDataPath($dbname, $shardId), 'compaction'); + $this->recovery->removeStaleTemp($this->segments->segDataPath($dbname, $shardId)); // {seg}.jsonl[.kix].tmp{8 hex} + $plan = null; + for($attempt = 0; $plan === null; $attempt++){ + if($attempt > 1){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + // A record the index points at must verify; if not, rebuild the index once and plan again + $index = $attempt === 0 ? $this->checkpoints->writableSegmentIndex($dbname, $shardId) : $this->checkpoints->repairSegmentIndex($dbname, $shardId, true); + if($index === null){ + return ['compacted' => 0, 'freed' => 0]; + } + $plan = $this->planCompaction($dbname, $shardId, $index); + } + $path = $this->segments->segDataPath($dbname, $shardId); + $live = $plan['live']; + unset($plan['live']); // held once: each record's bytes are released once they are in the buffer + ksort($live, SORT_NUMERIC); + + $newIndex = $this->checkpoints->newJsonlIndex(); + $newIndex['created'] = $index['created'] ?? time(); + $newIndex['n'] = $index['n']; + $buffer = ''; + $offset = 0; + $maxKey = -1; + // A fresh index of distinct keys: its arrays filled directly (KeyIndex::set() per key checks a body) + foreach($live as $key => &$json){ + $newIndex['p'][$key] = $offset; + $newIndex['l'][$key] = strlen($json); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + $maxKey = $key; + $json = null; // in the buffer now: the record's string is freed, the peak stays near one copy + } + unset($json); + $newIndex['live'] = count($live); + $live = null; + // Keep the highest key ever used: an index rebuilt from this file must never hand it out again + $highest = ($shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname)) + (int)$index['n'] - 1; + if($maxKey < $highest){ + $buffer .= '{"__nonedb_deleted":' . $highest . "}\n"; + $newIndex['t'] = 1; + } + // Lines that don't decode are dropped from the data file: their bytes go to quarantine first + $events = []; + foreach($plan['bad'] as $region){ + $quarantine = $this->recovery->quarantineBytes($dbname, $shardId, $path, $region[0], $region[0] + $region[1]); + $events[] = ['action' => 'drop-line', 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]; + } + + // The checkpoint goes first: it must never be taken for the new data file + $this->locks->bumpEpoch($dbname); + $this->io->fsUnlink(\noneDB\Internal\Storage\Checkpoints::ckptPath($path)); + $this->recovery->fault('compact:replace', ['db' => $dbname, 'shard' => $shardId]); // @test + try { + $replaced = $this->io->writeFileAtomic($path, $buffer, true); + } catch(\RuntimeException $e){ + throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data: " . $e->getMessage(), $e->getCode(), $e); + } + if(!$replaced){ + throw new \RuntimeException("noneDB: compaction of database '{$dbname}' failed while writing data"); + } + $this->recovery->fault('compact:replaced', ['db' => $dbname, 'shard' => $shardId]); // @test + $newIndex['s'] = strlen($buffer); + $newIndex['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = $newIndex; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = true; + $this->checkpoints->writeCheckpoint($dbname, $shardId, $newIndex); + foreach($events as $event){ + $this->recoveryLog->logRecoveryEvent($dbname, $event); + } + $this->metrics['compactions']++; + $this->metrics['compactionSeconds'] += microtime(true) - $started; + return ['compacted' => $newIndex['live'], 'freed' => (int)$index['d']]; + }, 'compact'); + } + + /** + * Read a segment line by line for compaction: the JSON of every live record (verified against its + * key) and the byte ranges of lines that are neither records nor tombstones + * @return array|null ['live' => [key => json], 'bad' => [[offset, length], ...]]; null if the index + * doesn't match the data + */ + private function planCompaction($dbname, $shardId, array $index){ + // Every live key's offset and length (a decoded index: its arrays as they are; no array per record) + [$offsets, $lengths] = \noneDB\Internal\Storage\KeyIndex::locate($index, null); + asort($offsets, SORT_NUMERIC); // offsets are distinct: file order (a sort without a callback) + // File order as two lists, no array per record: where each live line starts, and its key + $starts = array_values($offsets); + $keys = array_keys($offsets); + unset($offsets); + $count = count($keys); + // Keys of this segment: a line with another shard's key is damaged, never a superseded record + $size = $shardId === null ? null : $this->segments->shardSizeOf($dbname); + $base = $shardId === null ? 0 : $shardId * $size; + $limit = $shardId === null ? null : $base + $size; + $fp = @fopen($this->segments->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return $count === 0 ? ['live' => [], 'bad' => []] : null; + } + $live = []; + $bad = []; + $next = 0; + $pos = 0; + try { + while(($line = fgets($fp)) !== false){ + $length = strlen($line); + $end = $pos + $length; + $owned = []; // positions in the lists of the records this line holds + while($next < $count && $starts[$next] < $end){ + $owned[] = $next++; + } + if(substr($line, -1) !== "\n"){ + if(!empty($owned)){ + return null; + } + $bad[] = [$pos, $length]; // unterminated tail + break; + } + $json = substr($line, 0, -1); + $whole = count($owned) === 1 && (int)$starts[$owned[0]] === $pos && (int)$lengths[$keys[$owned[0]]] === strlen($json); + foreach($owned as $i){ + $key = (int)$keys[$i]; + $sub = $whole ? $json : substr($line, (int)$starts[$i] - $pos, (int)$lengths[$key]); + $record = json_decode($sub, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== $key){ + return null; + } + $live[$key] = $sub; + } + if(!$whole){ + if(!empty($owned)){ + $bad[] = [$pos, $length]; // a salvaged record inside other bytes: keep the line + } elseif($this->checkpoints->classifyLine($json, $base, $limit) === null){ + $bad[] = [$pos, $length]; // neither a record nor a tombstone of this segment + } + } + $pos = $end; + } + } finally { + fclose($fp); + } + return $next < $count ? null : ['live' => $live, 'bad' => $bad]; + } +} diff --git a/src/Internal/Maintenance/KeyMaintenance.php b/src/Internal/Maintenance/KeyMaintenance.php new file mode 100644 index 0000000..732607e --- /dev/null +++ b/src/Internal/Maintenance/KeyMaintenance.php @@ -0,0 +1,570 @@ +k = $k; + $this->rt = $k->rt; + unset($this->io, $this->locks, $this->recovery, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->storeState = &$this->rt->storeState; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // KEY MAINTENANCE (docs/STORAGE_FORMAT.md §9) + // ========================================== + // + // Every operation follows the maintenance order: STORE.LOCK (exclusive, non-blocking), then every + // target database's LOCK (non-blocking; one busy lock: nothing written, MAINTENANCE_BUSY), then the + // journal .nonedb-maintenance/journal.json, then "m" and a new epoch in every target's state record. + // The logical commit point is the journal in state COMMITTED with the complete target STORE record; + // the STORE copies, clearing "m" and closing the journal follow and are repeated by a re-run. + + /** + * Rotate the secret key of every database in $dbDir that belongs to $oldKey: their directories are + * renamed to the new key's locators (contents unchanged: FORMAT and metadata hold no key) and STORE + * registers $newKey instead of $oldKey. Application processes must be stopped. Re-running after an + * interruption finishes the rotation; ['abort' => true] undoes one that did not commit yet. + * @return array ['success' => bool, 'databases' => string[], 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function migrateKey(string $dbDir, string $oldKey, string $newKey, array $options = []): array { + self::checkOptions($options, ['abort', 'durability'], 'migrateKey()'); + $result = ['success' => false, 'databases' => [], 'storeManifest' => null, 'error' => null, 'code' => null]; + if(trim($oldKey) === '' || trim($newKey) === ''){ + $result['error'] = 'Both keys must be non-empty strings'; + return $result; + } + if(in_array($newKey, \noneDB\Internal\Cache\Runtime::get()->placeholderKeys, true)){ + $result['error'] = 'The new key is a placeholder; generate a random one'; + return $result; + } + if($oldKey === $newKey){ + $result['error'] = 'The new key is the same as the old key'; + return $result; + } + return self::maintenanceCall($dbDir, $oldKey, $result, $options, function(self $db) use ($oldKey, $newKey, $options){ + return $db->rotateKey($oldKey, $newKey, $options['abort'] ?? false); + }); + } + + /** + * Register a further key for the store in $dbDir; $registeredKey must already be registered + * @return array ['success' => bool, 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function registerKey(string $dbDir, string $registeredKey, string $newKey, array $options = []): array { + self::checkOptions($options, ['durability'], 'registerKey()'); + $result = ['success' => false, 'storeManifest' => null, 'error' => null, 'code' => null]; + if(trim($newKey) === '' || in_array($newKey, \noneDB\Internal\Cache\Runtime::get()->placeholderKeys, true)){ + $result['error'] = 'The new key must be a non-empty string and not a placeholder'; + return $result; + } + return self::maintenanceCall($dbDir, $registeredKey, $result, $options, function(self $db) use ($newKey){ + return $db->addStoreKey(\noneDB\Internal\Storage\StoreRecord::keyCheckOf($newKey)); + }); + } + + /** + * Rebuild a store record whose two copies are both invalid (STORAGE_FORMAT.md §9.4). Every database + * directory must belong to one of $keys. ['manifest' => path or array]: a verified STORE record (a + * backup's store.json, a maintenance result's storeManifest) is restored (complete). Without one, only + * ['confirmManualRebuild' => true] rebuilds the record from $keys (not complete: keys without + * databases that were registered before are lost). + * @return array ['success' => bool, 'mode' => 'manifest'|'manual'|null, 'complete' => bool, 'storeManifest' => array|null, + * 'error' => ?string (lists databases no key owns), 'code' => ?int] + */ + public static function repairStore(string $dbDir, array $keys, array $options = []): array { + self::checkOptions($options, ['manifest', 'confirmManualRebuild', 'durability'], 'repairStore()'); + $result = ['success' => false, 'mode' => null, 'complete' => false, 'storeManifest' => null, 'error' => null, 'code' => null]; + $keys = array_values(array_filter($keys, 'is_string')); + if(empty($keys)){ + $result['error'] = 'At least one key is needed'; + return $result; + } + return self::maintenanceCall($dbDir, $keys[0], $result, $options, function(self $db) use ($keys, $options){ + return $db->rebuildStoreRecord($keys, $options); + }); + } + + /** + * The options of a maintenance method are the ones it names, of their types (API.md "Key maintenance"), checked + * before anything is done + * @throws \InvalidArgumentException + */ + private static function checkOptions(array $options, array $known, string $method): void { + \noneDB\Internal\Config\Configuration::checkNames($options, $known, 'option', " of {$method}"); + foreach(['abort', 'confirmManualRebuild'] as $name){ + if(array_key_exists($name, $options)){ + \noneDB\Internal\Config\Configuration::optionBool($name, $options[$name]); + } + } + if(array_key_exists('manifest', $options) && !is_string($options['manifest']) && !is_array($options['manifest'])){ + throw new \InvalidArgumentException("noneDB: 'manifest' must be a file path or a store record (array)"); + } + if(array_key_exists('durability', $options)){ + \noneDB\Internal\Config\Configuration::durabilityValue($options['durability']); // the platform: as the setting's (UNSUPPORTED_DURABILITY) + } + } + + /** + * Run a maintenance operation with an instance for $dbDir and $key (no config file involved). + * $options['durability']: 'performance' (default) or 'durable', as the setting (docs/DURABILITY.md). + * @throws \InvalidArgumentException an invalid durability mode + */ + private static function maintenanceCall(string $dbDir, string $key, array $result, array $options, callable $fn): array { + if(!empty(\noneDB\Internal\Cache\Runtime::get()->lockRegistry)){ + $result['error'] = 'Maintenance can\'t run while this process holds a database lock (inside a database operation)'; + return $result; + } + $dbDir = rtrim($dbDir, '/\\') . '/'; + if(!is_dir($dbDir)){ + $result['error'] = "Directory not found: {$dbDir}"; + return $result; + } + try { + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + // The instance stays referenced while its kernel works: its destructor forgets the key + $instance = new \noneDB(['secretKey' => $key, 'dbDir' => $dbDir, 'autoCreateDB' => false, + 'durability' => $options['durability'] ?? 'performance']); + $out = $fn(\noneDB\Internal\Core\Kernel::of($instance)->keyMaintenance); + unset($instance); + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + if(is_array($out['storeManifest'] ?? null)){ + // The complete STORE record with its checksum: usable as repairStore()'s manifest as it is + $out['storeManifest'] = json_decode(\noneDB\Internal\Storage\StoreRecord::sealRecord($out['storeManifest']), true); + } + return array_merge($result, $out, ['success' => true]); + } catch(\RuntimeException $e){ + \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + return array_merge($result, ['error' => $e->getMessage(), 'code' => $e->getCode()]); + } + } + + // ---------------------------------------------------------------- the maintenance order + + private function journalPath(){ + return $this->dbDir . '.nonedb-maintenance/journal.json'; + } + + private function readJournal(){ + $path = $this->journalPath(); + clearstatcache(true, $path); + if(!is_file($path)){ + return null; + } + $journal = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($path), 'journal'); + if($journal === null){ + throw new \RuntimeException("noneDB: the maintenance journal in {$this->dbDir} is unreadable; nothing is changed", \noneDB::FORMAT_DAMAGED); + } + return $journal; + } + + private function writeJournal(array $journal){ + $dir = $this->dbDir . '.nonedb-maintenance'; + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + throw new \RuntimeException("noneDB: cannot create {$dir}"); + } + $this->recovery->fault('maintenance:journal', ['state' => $journal['state']]); // @test + if(!$this->store->writeStoreFile($this->journalPath(), \noneDB\Internal\Storage\StoreRecord::sealRecord(['magic' => 'NONEDB4', 'kind' => 'journal'] + $journal))){ + throw new \RuntimeException("noneDB: cannot write the maintenance journal in {$this->dbDir}"); + } + } + + /** + * Remove the journal: the maintenance is complete. In durable mode a removal that may not survive a power + * loss is an unknown outcome (the journal could come back and close the store until a re-run). + * @throws \RuntimeException OUTCOME_UNKNOWN + */ + private function closeJournal(){ + $this->recovery->fault('maintenance:close', []); // @test + @unlink($this->journalPath()); + $this->recovery->removeStaleTemp($this->journalPath()); // a crashed journal write's temp (an abort writes none) + $this->sync->requireSync(null, $this->dbDir . '.nonedb-maintenance', 'dir', true); + } + + /** + * Lock every target database (locator order, non-blocking). All or nothing: a busy one releases the + * others and fails with MAINTENANCE_BUSY before anything was written. + * @param string[] $locators + * @return array locator => handle + */ + /** + * Lock every target database exclusively without waiting, in locator order (§9.2 step 2); one busy: + * all released, MAINTENANCE_BUSY, nothing written + * @param array $names locator => database name (for the message) + */ + private function lockMaintenanceTargets(array $names){ + $locators = array_keys($names); + sort($locators, SORT_STRING); + $handles = []; + foreach($locators as $locator){ + $path = $this->dbDir . $locator . '.ndb4/LOCK'; + $fp = is_dir(dirname($path)) ? @fopen($path, 'c+be') : false; + if($fp === false || !flock($fp, LOCK_EX | LOCK_NB)){ + if($fp !== false){ + fclose($fp); + } + foreach($handles as $held){ + flock($held, LOCK_UN); + fclose($held); + } + throw new \RuntimeException("noneDB: database '{$names[$locator]}' ({$locator}.ndb4) is in use; stop the application processes and run the maintenance again", \noneDB::MAINTENANCE_BUSY); + } + $handles[$locator] = $fp; + } + return $handles; + } + + private static function releaseMaintenanceTargets(array $handles){ + foreach($handles as $fp){ + flock($fp, LOCK_UN); + fclose($fp); + } + } + + /** + * Set or clear "m" in a held database lock's state record, always with a new epoch (synced) + */ + private function markMaintenance($fp, $on, $path){ + fseek($fp, 0); + $raw = trim((string)fread($fp, \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE), " \0"); + $tokens = $raw === '' ? [] : explode(' ', $raw); + $state = (string)array_shift($tokens); + $gen = preg_match('/^([0-9]{1,18}):[0-9a-f]{8}$/', $state, $m) === 1 ? (int)$m[1] : 0; + $tokens = array_values(array_filter($tokens, function($t){ return $t !== '' && $t !== 'm'; })); + if($on){ + $tokens[] = 'm'; + } + $record = str_pad(trim(($gen + 1) . ':' . bin2hex(random_bytes(4)) . ' ' . implode(' ', $tokens)), \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE); + fseek($fp, 0); + if(fwrite($fp, $record) !== \noneDB\Internal\Concurrency\LockManager::LOCK_STATE_SIZE || !fflush($fp)){ + throw new \RuntimeException("noneDB: cannot write a database state record in {$this->dbDir}"); + } + // Durable mode: "m" and its epoch are synced before the work, the cleared record before the journal goes + $this->sync->requireSync($fp, $path, 'file'); + } + + /** + * Clear "m" of databases after the commit (their locks are taken again, waiting up to lockTimeout) + */ + private function clearMaintenanceMarks(array $locators){ + foreach($locators as $locator){ + $path = $this->dbDir . $locator . '.ndb4/LOCK'; + if(!is_file($path)){ + continue; + } + $fp = @fopen($path, 'c+be'); + if($fp === false){ + throw new \RuntimeException("noneDB: cannot open the lock of database {$locator}"); + } + try { + $this->locks->acquireFlock($fp, LOCK_EX, $locator); + $this->markMaintenance($fp, false, $path); + flock($fp, LOCK_UN); + } finally { + fclose($fp); + } + } + } + + /** + * Apply a committed journal: STORE copies, "m" cleared, journal closed + */ + private function applyCommittedJournal(array $journal){ + try { + $this->store->writeStoreCopies($journal['store']); + $this->recovery->fault('maintenance:store-written', []); // @test + $this->clearMaintenanceMarks($journal['clear'] ?? []); + $this->closeJournal(); + } catch(\RuntimeException $e){ + if($e->getCode() === \noneDB::OUTCOME_UNKNOWN){ + throw $e; // the journal is gone in this process; whether that is durable is unknown + } + // Committed but not completed: never reported as success; the journal stays open (the store stays + // closed) until a re-run applies it (STORAGE_FORMAT.md §9.2) + throw new \RuntimeException("noneDB: the maintenance of the store in {$this->dbDir} is committed but not completed (" + . $e->getMessage() . "); run it again", \noneDB::MAINTENANCE_IN_PROGRESS, $e); + } + unset($this->storeState[$this->dbDir]); + return $journal['store']; + } + + // ---------------------------------------------------------------- operations + + private function rotateKey($oldKey, $newKey, $abort){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($oldKey, $newKey, $abort){ + $journal = $this->readJournal(); + if($journal !== null){ + return $this->resumeRotation($journal, $abort); + } + $store = $this->store->loadStoreRecord(); + $oldCheck = \noneDB\Internal\Storage\StoreRecord::keyCheckOf($oldKey); + if($store === null || !in_array($oldCheck, $store['keys'], true)){ + throw new \RuntimeException("noneDB: the old key is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + // The databases of the old key and where they go + $targets = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + if(!is_array($format) || !is_string($format['name'] ?? null) || \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $oldKey) !== $m[1]){ + continue; + } + $to = \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $newKey); + if(file_exists($this->dbDir . $to . '.ndb4')){ + throw new \RuntimeException("noneDB: the new key's directory for database '{$format['name']}' already exists", \noneDB::MAINTENANCE_BUSY); + } + $targets[] = ['name' => $format['name'], 'from' => $m[1], 'to' => $to, 'databaseId' => $format['databaseId'], 'done' => false]; + } + $handles = $this->lockMaintenanceTargets(array_column($targets, 'name', 'from')); + $target = $store; + $target['keys'] = array_values(array_unique(array_merge(array_diff($store['keys'], [$oldCheck]), [\noneDB\Internal\Storage\StoreRecord::keyCheckOf($newKey)]))); + sort($target['keys'], SORT_STRING); + $target['seq'] = $store['seq'] + 1; + $journal = ['op' => 'rotate', 'id' => bin2hex(random_bytes(8)), 'state' => 'STARTED', 'targets' => $targets, + 'store' => $target, 'clear' => array_column($targets, 'to')]; + try { + $this->writeJournal($journal); + foreach($targets as $t){ + $this->markMaintenance($handles[$t['from']], true, $this->dbDir . $t['from'] . '.ndb4/LOCK'); + } + $this->recovery->fault('rotate:marked', []); // @test + } finally { + self::releaseMaintenanceTargets($handles); // "m" keeps every process out from here on + } + return $this->resumeRotation($journal, $abort); + }, true); + } finally { + $this->maintenanceDepth--; + } + } + + /** + * Continue a rotation from its journal: rename what is left, verify, commit, apply + */ + private function resumeRotation(array $journal, $abort){ + if($journal['op'] !== 'rotate'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished; run it again", \noneDB::MAINTENANCE_IN_PROGRESS); + } + if($journal['state'] === 'COMMITTED'){ + $store = $this->applyCommittedJournal($journal); + return ['databases' => array_column($journal['targets'], 'name'), 'storeManifest' => $store]; + } + if($abort){ + $this->undoRotationRenames($journal); + return ['databases' => [], 'storeManifest' => $this->store->loadStoreRecord(), 'aborted' => true]; + } + foreach($journal['targets'] as $i => $t){ + $from = $this->dbDir . $t['from'] . '.ndb4'; + $to = $this->dbDir . $t['to'] . '.ndb4'; + if(is_dir($from) && !file_exists($to)){ + if(!$this->sync->renameWithRetry($from, $to)){ + throw new \RuntimeException("noneDB: could not rename the directory of database '{$t['name']}'; run migrateKey() again"); + } + $this->recovery->fault('rotate:renamed', ['db' => $t['name']]); // @test + } + $journal['targets'][$i]['done'] = true; + // Verified before the commit: FORMAT, CURRENT and its generation belong to this database + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/FORMAT'), 'database'); + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/CURRENT'), 'current'); + $metadata = $current === null || !is_int($current['generation'] ?? null) ? null + : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($to . '/generations/' . $current['generation'] . '/metadata'), 'generation'); + if($format === null || ($format['databaseId'] ?? null) !== $t['databaseId'] || ($current['databaseId'] ?? null) !== $t['databaseId'] + || ($metadata['databaseId'] ?? null) !== $t['databaseId']){ + $this->undoRotationRenames($journal); + throw new \RuntimeException("noneDB: database '{$t['name']}' failed verification during the key rotation; nothing was changed", \noneDB::FORMAT_DAMAGED); + } + } + // The renames are durable before the commit (DURABILITY.md §3) + $this->sync->requireSync(null, $this->dbDir, 'dir'); + $journal['state'] = 'COMMITTED'; + $this->writeJournal($journal); + $this->recovery->fault('rotate:committed', []); // @test + $store = $this->applyCommittedJournal($journal); + return ['databases' => array_column($journal['targets'], 'name'), 'storeManifest' => $store]; + } + + /** + * Move every renamed directory of an uncommitted rotation back (abort, failed verification), then clear + * "m" and close the journal. A rename that fails keeps the journal open: the store stays closed until + * a re-run with ['abort' => true] finishes it (§9.3). + */ + private function undoRotationRenames(array $journal){ + foreach(array_reverse($journal['targets']) as $t){ + $from = $this->dbDir . $t['from'] . '.ndb4'; + $to = $this->dbDir . $t['to'] . '.ndb4'; + clearstatcache(true, $from); + clearstatcache(true, $to); + if(is_dir($to) && !file_exists($from) && !$this->sync->renameWithRetry($to, $from)){ + throw new \RuntimeException("noneDB: could not move database '{$t['name']}' back; the key rotation stays open: run migrateKey() with ['abort' => true] again", \noneDB::MAINTENANCE_IN_PROGRESS); + } + } + $this->sync->requireSync(null, $this->dbDir, 'dir'); + $this->clearMaintenanceMarks(array_column($journal['targets'], 'from')); + $this->closeJournal(); + } + + private function addStoreKey($keyCheck){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($keyCheck){ + $journal = $this->readJournal(); + if($journal !== null){ + if($journal['op'] !== 'register' || $journal['state'] !== 'COMMITTED'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished", \noneDB::MAINTENANCE_IN_PROGRESS); + } + return ['storeManifest' => $this->applyCommittedJournal($journal)]; + } + $store = $this->store->loadStoreRecord(); + if($store === null || !in_array($this->store->keyCheck(), $store['keys'], true)){ + throw new \RuntimeException("noneDB: the registered key is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + if(in_array($keyCheck, $store['keys'], true)){ + return ['storeManifest' => $store]; + } + $target = $store; + $target['keys'][] = $keyCheck; + sort($target['keys'], SORT_STRING); + $target['seq'] = $store['seq'] + 1; + $journal = ['op' => 'register', 'id' => bin2hex(random_bytes(8)), 'state' => 'COMMITTED', 'store' => $target, 'clear' => []]; + $this->writeJournal($journal); + $this->recovery->fault('register:committed', []); // @test + return ['storeManifest' => $this->applyCommittedJournal($journal)]; + }, true); + } finally { + $this->maintenanceDepth--; + } + } + + private function rebuildStoreRecord(array $keys, array $options){ + $this->maintenanceDepth++; + try { + return $this->store->withStoreLock(LOCK_EX, function() use ($keys, $options){ + $journal = $this->readJournal(); + if($journal !== null){ + if($journal['op'] !== 'repair-store' || $journal['state'] !== 'COMMITTED'){ + throw new \RuntimeException("noneDB: another maintenance ({$journal['op']}) of the store in {$this->dbDir} is unfinished", \noneDB::MAINTENANCE_IN_PROGRESS); + } + $store = $this->applyCommittedJournal($journal); + return ['mode' => $journal['mode'], 'complete' => $journal['mode'] === 'manifest', 'storeManifest' => $store]; + } + [, $copy1] = $this->store->readStoreCopy('STORE'); + [, $copy2] = $this->store->readStoreCopy('STORE.2'); + if($copy1 !== null || $copy2 !== null){ + throw new \RuntimeException("noneDB: a valid store record exists in {$this->dbDir}; it is repaired automatically, repairStore() is only for two invalid copies", \noneDB::FORMAT_DAMAGED); + } + // Ownership: every database directory must belong to one of the keys + $unattributed = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^([0-9a-f]{32})\.ndb4$/', $entry, $m) !== 1 || is_link($this->dbDir . $entry)){ + continue; + } + $format = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbDir . $entry . '/FORMAT'), 'database'); + $owned = false; + foreach($keys as $key){ + if(is_array($format) && is_string($format['name'] ?? null) && \noneDB\Internal\Storage\StoreRecord::locatorOf($format['name'], $key) === $m[1]){ + $owned = true; + break; + } + } + if(!$owned){ + $unattributed[] = $entry; + } + } + if(!empty($unattributed)){ + throw new \RuntimeException("noneDB: these databases belong to none of the given keys: " . implode(', ', $unattributed) . "; give every key of the store", \noneDB::FORMAT_DAMAGED); + } + $keyChecks = array_map([\noneDB\Internal\Storage\StoreRecord::class, 'keyCheckOf'], $keys); + if(isset($options['manifest'])){ + // A path (store.json, a saved STORE copy) or the storeManifest array of a maintenance result; + // either way its checksum must verify + $source = $options['manifest']; + $raw = is_array($source) ? json_encode($source, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION) . "\n" + : (is_string($source) ? @file_get_contents($source) : false); + $manifest = \noneDB\Internal\Storage\StoreRecord::openRecord($raw, 'store'); + if($manifest === null || ($manifest['storageFormat'] ?? null) !== \noneDB::STORAGE_FORMAT || !is_array($manifest['keys'] ?? null) + || !is_int($manifest['seq'] ?? null) || !is_string($manifest['storeId'] ?? null)){ + throw new \RuntimeException('noneDB: the store manifest' . (is_string($source) ? " {$source}" : '') . ' is not a valid store record', \noneDB::FORMAT_DAMAGED); + } + $mode = 'manifest'; + $store = $manifest; + $store['keys'] = array_values(array_unique(array_merge($manifest['keys'], $keyChecks))); + $store['seq'] = (int)$manifest['seq'] + 1; + } elseif(!empty($options['confirmManualRebuild'])){ + $mode = 'manual'; + $store = ['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT, 'storeId' => \noneDB\Internal\Storage\StoreRecord::uuid4(), + 'seq' => 1, 'keys' => array_values(array_unique($keyChecks))]; + } else { + throw new \RuntimeException("noneDB: without a verified store manifest the record can only be rebuilt by hand: pass ['confirmManualRebuild' => true] (keys without databases that were registered are not restored)", \noneDB::FORMAT_DAMAGED); + } + sort($store['keys'], SORT_STRING); + unset($store['checksum']); + $journal = ['op' => 'repair-store', 'id' => bin2hex(random_bytes(8)), 'state' => 'COMMITTED', 'mode' => $mode, 'store' => $store, 'clear' => []]; + $this->writeJournal($journal); + $this->recovery->fault('repair:committed', []); // @test + $store = $this->applyCommittedJournal($journal); + $this->store->logStoreEvent(['action' => $mode === 'manual' ? 'manual-store-rebuild' : 'store-restored-from-manifest', 'keys' => count($store['keys'])]); + return ['mode' => $mode, 'complete' => $mode === 'manifest', 'storeManifest' => $store]; + }, true); + } finally { + $this->maintenanceDepth--; + } + } +} diff --git a/src/Internal/Maintenance/Maintenance.php b/src/Internal/Maintenance/Maintenance.php new file mode 100644 index 0000000..a9de025 --- /dev/null +++ b/src/Internal/Maintenance/Maintenance.php @@ -0,0 +1,366 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->compaction, $this->db, $this->fieldIndex, $this->io, $this->layout, $this->layoutSwitch, $this->locks, $this->rtree, $this->scope, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->bufferCountLimit = &$this->k->bufferCountLimit; + $this->bufferEnabled = &$this->k->bufferEnabled; + $this->bufferFlushInterval = &$this->k->bufferFlushInterval; + $this->bufferSizeLimit = &$this->k->bufferSizeLimit; + $this->indexCache = &$this->k->indexCache; + $this->shardSize = &$this->k->shardSize; + $this->shardingEnabled = &$this->k->shardingEnabled; + $this->lockRegistry = &$this->rt->lockRegistry; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->compaction = null; + $this->db = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->layoutSwitch = null; + $this->locks = null; + $this->rtree = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE BUFFER PUBLIC API + // ========================================== + + /** + * Manually flush buffer for a database + * @param string $dbname + * @return array ['success' => bool, 'flushed' => int, 'error' => string|null] + */ + public function flush($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + return $this->flushUnlocked($dbname); + }, 'flush'); + } + + private function flushUnlocked($dbname){ + $flushed = $this->buffer->flushPendingBuffers($dbname); + return ['success' => true, 'flushed' => $flushed, 'error' => null]; + } + + /** + * Flush the write buffers of every database (of this secretKey) in dbDir + * @return array ['databases' => int, 'flushed' => int] + */ + public function flushAllBuffers(){ + $names = []; + if(!empty($this->lockRegistry)){ + return ['databases' => 0, 'flushed' => 0]; // inside an interrupted operation (see releaseInterruptedSessions) + } + foreach($this->db->getDBs(false) as $dbname){ + $buffered = $this->locks->withLock($dbname, false, function() use ($dbname){ + return $this->buffer->hasBuffer($this->buffer->getBufferPath($dbname)) || !empty($this->buffer->flushingFiles($dbname)); + }); + if($buffered){ + $names[$dbname] = true; + } + } + $total = 0; + foreach(array_keys($names) as $dbname){ + $result = $this->flush((string)$dbname); + $total += $result['flushed']; + } + return ['databases' => count($names), 'flushed' => $total]; + } + + /** + * Get buffer information for a database + * @param string $dbname + * @return array + */ + public function getBufferInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + $fn = function() use ($dbname){ + return $this->getBufferInfoUnlocked($dbname); + }; + return $this->locks->withLock($dbname, false, $fn); + } + + private function getBufferInfoUnlocked($dbname){ + $info = [ + 'enabled' => $this->bufferEnabled, + 'sizeLimit' => $this->bufferSizeLimit, + 'countLimit' => $this->bufferCountLimit, + 'flushInterval' => $this->bufferFlushInterval, + 'buffers' => [] + ]; + $describe = function($path){ + clearstatcache(true, $path); + $exists = file_exists($path) && filesize($path) > 0; + return [ + 'exists' => $exists, + 'size' => $exists ? (int)filesize($path) : 0, + 'records' => $exists ? $this->buffer->getBufferRecordCount($path) : 0 + ]; + }; + $info['buffers']['main'] = $describe($this->buffer->getBufferPath($dbname)); + return $info; + } + + /** + * Enable or disable write buffering + * @param bool $enable + */ + public function enableBuffering($enable = true){ + $this->bufferEnabled = (bool)$enable; + } + + /** + * Check if buffering is enabled + * @return bool + */ + public function isBufferingEnabled(){ + return $this->bufferEnabled; + } + + /** + * Set buffer size limit (in bytes) + * @param int $bytes + */ + public function setBufferSizeLimit($bytes){ + $this->bufferSizeLimit = max(1024, (int)$bytes); // Minimum 1KB + } + + /** + * Set buffer flush interval (in seconds) + * @param int $seconds 0 to disable time-based flush + */ + public function setBufferFlushInterval($seconds){ + $this->bufferFlushInterval = max(0, (int)$seconds); + } + + /** + * Set buffer count limit + * @param int $count + */ + public function setBufferCountLimit($count){ + $this->bufferCountLimit = max(10, (int)$count); // Minimum 10 records + } + + /** + * Compact a database: drop old record versions and deleted records from the data files. + * Keys are preserved (sharded and non-sharded alike); indexes stay valid. + * @param string $dbname + * @return array ['success' => bool, 'freedSlots' => int, 'totalRecords' => int, 'sharded' => bool] + */ + public function compact($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $this->scope->refuseInUncommittedWrite($dbname, 'compact'); + return $this->compactUnlocked($dbname); + }, 'compact'); + } + + private function compactUnlocked($dbname){ + $result = array("success" => false, "freedSlots" => 0); + $this->buffer->flushPendingBuffers($dbname); + if(!$this->segments->dbExists($dbname)){ + $result['status'] = 'database_not_found'; + return $result; + } + + $sharded = $this->segments->isSharded($dbname); + $freed = 0; + if($sharded){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + $freed = (int)($meta['deletedCount'] ?? 0); + } + $total = 0; + foreach($this->segments->listSegments($dbname) as $segment){ + $compacted = $this->compaction->compactJsonl($dbname, $segment); + if(!$sharded){ + $freed += $compacted['freed']; + } + $total += $compacted['compacted']; + $this->cache->evictBetweenSegments(); + } + if($sharded){ + $this->removeEmptyShards($dbname); + $meta = $this->shardMeta->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + $meta['shards'][$i]['deleted'] = 0; + } + $meta['deletedCount'] = 0; + $meta['totalRecords'] = $total; + $this->shardMeta->writeMeta($dbname, $meta); + } + + $result['success'] = true; + $result['freedSlots'] = $freed; + $result['totalRecords'] = $total; + $result['sharded'] = $sharded; + if($sharded){ + $result['newShardCount'] = count($this->segments->listSegments($dbname)); + } + return $result; + } + + /** + * Remove shards without records (keys never move, so nothing else changes). The shard that holds + * the newest key is kept: new records go there. + */ + private function removeEmptyShards($dbname){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + if($meta === null || count($meta['shards']) < 2){ + return; + } + $lastShard = intdiv(max(0, (int)$meta['nextKey'] - 1), (int)$meta['shardSize']); + $kept = []; + $removed = []; + $done = false; // a shard was read: the cache may drop what it loaded before the next one + foreach($meta['shards'] as $shard){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $shardId = (int)$shard['id']; + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + if($shardId !== $lastShard && ($index === null || \noneDB\Internal\Storage\KeyIndex::count($index) === 0)){ + $removed[] = $shardId; + } else { + $kept[] = $shard; + } + } + if(empty($removed)){ + return; + } + $meta['shards'] = $kept; + // The meta must not lose this change in a power loss after the files are gone (where PHP can sync) + $this->shardMeta->writeMeta($dbname, $meta, true); + + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $this->locks->bumpEpoch($dbname); // shard files go away + foreach($removed as $shardId){ + $dataPath = $this->segments->getShardPath($dbname, $shardId); + foreach($manifest['fields'] as $field){ + $this->fieldIndex->deleteFieldIndexFile($dbname, $field, $shardId); + } + $this->rtree->removeSpatialShardFiles($dbname, $shardId); + $this->io->fsUnlink(\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)); + $this->io->fsUnlink($dataPath); + $this->cache->markFileNotExists($dataPath); + unset($this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($dataPath)]); + } + // The shard list no longer names them: their removal only frees space. A power loss may bring them + // back as files no shard entry names (set aside when a shard with that number is created again). + foreach($manifest['fields'] as $field){ + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + } + } + /** + * Switch a database of the single layout to the sharded layout (STORAGE_FORMAT.md §5.2) + * @param string $dbname + * @return array ["success" => bool, "status" => string] + */ + public function convertToSharded($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $this->scope->refuseInUncommittedWrite($dbname, 'convertToSharded'); + return $this->convertToShardedUnlocked($dbname); + }, 'convertToSharded'); + } + + private function convertToShardedUnlocked($dbname){ + if($this->segments->isSharded($dbname)){ + return array("success" => true, "status" => "already_sharded"); + } + $this->buffer->flushPendingBuffers($dbname); + if($this->segments->isSharded($dbname)){ + return array("success" => true, "status" => "converted"); + } + if(!$this->cache->cachedFileExists($this->segments->getDataPath($dbname))){ + return array("success" => false, "status" => "database_not_found"); + } + if($this->layoutSwitch->convertToSharded($dbname)){ + return array("success" => true, "status" => "converted"); + } + return array("success" => false, "status" => "conversion_failed"); + } + + /** + * Check if sharding is enabled + * @return bool + */ + public function isShardingEnabled(){ + return $this->shardingEnabled; + } + + /** + * Get current shard size setting + * @return int + */ + public function getShardSize(){ + return $this->shardSize; + } +} diff --git a/src/Internal/Query/Aggregation.php b/src/Internal/Query/Aggregation.php new file mode 100644 index 0000000..bc09f54 --- /dev/null +++ b/src/Internal/Query/Aggregation.php @@ -0,0 +1,161 @@ +k = $k; + $this->rt = $k->rt; + unset($this->layout, $this->locks, $this->planner); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Sum numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function sum($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->sumUnlocked($dbname, $field, $filter); + }); + } + + private function sumUnlocked($dbname, $field, $filter = 0){ + $sum = 0; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$sum){ + foreach($records as $record){ + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + } + } + }, true); + return $sum; + } + + /** + * Average of numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function avg($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->avgUnlocked($dbname, $field, $filter); + }); + } + + private function avgUnlocked($dbname, $field, $filter = 0){ + $sum = 0; + $count = 0; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$sum, &$count){ + foreach($records as $record){ + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if($number !== null){ + $sum += $number; + $count++; + } + } + }, true); + return $count > 0 ? $sum / $count : 0; + } + + /** + * Get minimum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function min($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->minUnlocked($dbname, $field, $filter); + }); + } + + private function minUnlocked($dbname, $field, $filter = 0){ + // The order of sort() (and of the query builder's min()); loose < depended on the insertion order + $min = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$min){ + foreach($records as $record){ + if(isset($record[$field]) && ($min === null || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $min) < 0)){ + $min = $record[$field]; + } + } + }, true); + return $min; + } + + /** + * Get maximum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function max($dbname, $field, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $filter){ + return $this->maxUnlocked($dbname, $field, $filter); + }); + } + + private function maxUnlocked($dbname, $field, $filter = 0){ + $max = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use ($field, &$max){ + foreach($records as $record){ + if(isset($record[$field]) && ($max === null || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $max) > 0)){ + $max = $record[$field]; + } + } + }, true); + return $max; + } +} diff --git a/src/Internal/Query/FindFilter.php b/src/Internal/Query/FindFilter.php new file mode 100644 index 0000000..f8aba45 --- /dev/null +++ b/src/Internal/Query/FindFilter.php @@ -0,0 +1,225 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * The filter parameter of find(), count(), sum(), ...: 0, null (all records) and arrays are filters, + * anything else is invalid (null) + * @return array|null + */ + public function normalizeFilter($filter){ + if($filter === null || is_int($filter)){ + return []; + } + return is_array($filter) ? $filter : null; + } + + // ========================================== + // FILTER MATCHING (find / update / delete) + // ========================================== + + /** + * Normalize a "key" filter value to a list of int keys (invalid values match nothing) + * @return array + */ + public function normalizeKeyFilter($value){ + $values = is_array($value) ? $value : [$value]; + $keys = []; + foreach($values as $v){ + if(is_int($v)){ + $k = $v; + } elseif(is_string($v) && preg_match('/\A(0|[1-9][0-9]*)\z/', $v)){ // \z: "5\n" is not 5 + $k = (int)$v; + } else { + continue; + } + if($k >= 0 && !isset($keys[$k])){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + + /** + * Strict equality filter used by find(), update() and delete(): + * every field must exist in the record and be identical (===) to the filter value. + * "key" filters compare against the record key. + */ + public function matchesFilters(array $record, array $filters){ + foreach($filters as $field => $value){ + if($field === 'key'){ + if(!isset($record['key']) || !in_array($record['key'], $this->normalizeKeyFilter($value), true)){ + return false; + } + continue; + } + if(!array_key_exists($field, $record) || $record[$field] !== $value){ + return false; + } + } + return true; + } + + /** + * Records matching $filters, in key order (or in the order of a "key" filter) + * @return array [key => record] + */ + public function selectRecords($dbname, array $filters){ + $result = []; + foreach($this->segmentMatches($dbname, $filters) as $records){ + $result += $records; + } + if(count($result) > 1 && array_key_exists('key', $filters)){ + $ordered = []; + foreach($this->normalizeKeyFilter($filters['key']) as $key){ + if(isset($result[$key])){ + $ordered[$key] = $result[$key]; + } + } + return $ordered; + } + return $result; + } + + /** + * The records matching $filters one segment at a time: yields segment id (null: a single-layout database) + * => [key => record], in key order (a "key" filter: in its order within the segment). Once the consumer is + * done with a segment the cache may drop what it loaded (evictBetweenSegments()), so a write by filter + * over a large database holds one segment's records. Caller holds the lock. + * @return \Generator + */ + public function segmentMatches($dbname, array $filters){ + if(array_key_exists('key', $filters)){ + $bySegment = []; + foreach($this->normalizeKeyFilter($filters['key']) as $key){ + $segment = $this->segments->segmentForKey($dbname, $key); + if($segment !== false){ + $bySegment[$segment === null ? '' : $segment][] = $key; + } + } + unset($filters['key']); + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($bySegment as $segment => $keys){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $segment = $segment === '' ? null : (int)$segment; + foreach(array_chunk($keys, \noneDB\Internal\Storage\RecordIO::BATCH) as $chunk){ + // In the order of the filter's keys + $found = $this->records->readSegmentRecords($dbname, $segment, $chunk); + $matches = []; + foreach($chunk as $key){ + if(isset($found[$key]) && $this->matchesFilters($found[$key], $filters)){ + $matches[$key] = $found[$key]; + } + } + unset($found); + if(!empty($matches)){ + yield $segment => $matches; + } + } + } + return; + } + + // Shards the global field indexes allow (null: every segment); none: nothing to read at all + $targets = $this->fieldIndex->targetShardsFromGlobalIndex($dbname, $filters); + if($targets === []){ + return; + } + $segments = $this->segments->listSegments($dbname); + if($targets !== null){ + // The targets that are segments (a global index may still name a removed shard), in order + $present = array_flip($segments); + $segments = []; + foreach($targets as $target){ + if(isset($present[$target])){ + $segments[] = $target; + } + } + } + $usable = $this->fieldIndex->indexedFilters($dbname, $filters); + $done = false; + foreach($segments as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + $done = false; + } + $keyIndex = null; + $candidates = null; + if(!empty($usable)){ + $keyIndex = $this->checkpoints->readJsonlIndex($dbname, $segment); + $candidates = $this->fieldIndex->segmentCandidates($dbname, $segment, $usable, $keyIndex); + if($candidates === []){ + continue; + } + } + foreach($this->records->eachSegmentBatch($dbname, $segment, $candidates, $keyIndex) as &$batch){ + $matches = []; + foreach($batch as $key => $record){ + if(empty($filters) || $this->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + $batch = null; // the records that don't match go before the caller writes the matches + if(!empty($matches)){ + yield $segment => $matches; + } + } + unset($batch); + $done = true; + } + } +} diff --git a/src/Internal/Query/Planner.php b/src/Internal/Query/Planner.php new file mode 100644 index 0000000..c045481 --- /dev/null +++ b/src/Internal/Query/Planner.php @@ -0,0 +1,384 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->fieldIndex, $this->findFilter, $this->locks, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->findFilter = null; + $this->locks = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // QUERY PLANS AND CURSORS + // ========================================== + // A plan is a tree of what a query's filters ask for that indexes can answer: + // ['eq', field, value] ['in', field, values] ['range', field, operator, operand] + // ['keys', [key, ...]] ['any'] ['and', [nodes]] ['or', [nodes]] + // For each segment it gives candidate keys (a superset of the matches: the caller checks every record + // with the whole filter) or null (scan the segment); shards it can rule out (global field indexes, + // key ranges) are skipped. Negations (whereNot, $ne, $nin, notLike, ...) never use indexes. + // scanRecords() reads the candidates of every segment in key order, in batches (32, growing to 4096), + // and stops as soon as the visitor says so: first(), exists() and limit() read what they need. + + /** + * Candidate keys of a segment for a plan node; null when the node can't narrow the segment down + * @return int[]|null sorted + */ + private function planCandidates($dbname, $shardId, array $node){ + switch($node[0]){ + case 'eq': + if(!$this->fieldIndexEnabled || !$this->fieldIndex->isIndexableValue($node[2]) || !$this->fieldIndex->isFieldIndexed($dbname, $node[1])){ + return null; + } + return $this->fieldIndex->fieldIndexCandidates($dbname, $shardId, [$node[1] => $node[2]]); + case 'in': + if(!$this->fieldIndexEnabled || !$this->fieldIndex->isFieldIndexed($dbname, $node[1])){ + return null; + } + $keys = []; + foreach($node[2] as $value){ + if(!$this->fieldIndex->isIndexableValue($value)){ + return null; + } + $found = $this->fieldIndex->fieldIndexCandidates($dbname, $shardId, [$node[1] => $value]); + if($found === null){ + return null; + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'range': + return $this->fieldIndex->fieldIndexRangeCandidates($dbname, $shardId, $node[1], [$node[2] => $node[3]]); + case 'keys': + $index = $this->checkpoints->readJsonlIndex($dbname, $shardId); + $keys = []; + foreach($node[1] as $k){ + if($index !== null && \noneDB\Internal\Storage\KeyIndex::has($index, (int)$k)){ + $keys[$k] = $k; + } + } + return array_values($keys); + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + continue; + } + $result = $result === null ? $found : array_keys(array_intersect_key(array_flip($result), array_flip($found))); + if(empty($result)){ + return []; + } + } + return $result; + case 'or': + $keys = []; + foreach($node[1] as $child){ + $found = $this->planCandidates($dbname, $shardId, $child); + if($found === null){ + return null; // one branch needs a scan: the whole segment + } + foreach($found as $k){ + $keys[$k] = $k; + } + } + return array_values($keys); + } + return null; + } + + /** + * Shards a plan node can match in (null: all). Only skips a shard when it can't hold a match. + * @return array|null shard id => true + */ + private function planShards($dbname, array $node){ + if(!$this->segments->isSharded($dbname)){ + return null; + } + switch($node[0]){ + case 'eq': + if(!$this->fieldIndex->isIndexableValue($node[2])){ + return null; + } + $shards = $this->fieldIndex->targetShardsFromGlobalIndex($dbname, [$node[1] => $node[2]]); + return $shards === null ? null : array_fill_keys($shards, true); + case 'in': + $all = []; + foreach($node[2] as $value){ + $found = $this->planShards($dbname, ['eq', $node[1], $value]); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + case 'keys': + $size = $this->segments->shardSizeOf($dbname); + $shards = []; + foreach($node[1] as $k){ + $shards[intdiv($k, $size)] = true; + } + return $shards; + case 'and': + $result = null; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found !== null){ + $result = $result === null ? $found : array_intersect_key($result, $found); + } + } + return $result; + case 'or': + $all = []; + foreach($node[1] as $child){ + $found = $this->planShards($dbname, $child); + if($found === null){ + return null; + } + $all += $found; + } + return $all; + } + return null; + } + + /** + * Read the records a plan allows, segment by segment in key order, in batches: $visit([key => record]) + * gets each batch (never an empty one) in key order and returns false to stop. Caller holds the lock. + * @param bool $whole The visitor never stops (count, top-k): each segment is read in one go (RecordIO::BATCH records at a time when it is larger) + * @return array ['segments', 'skipped', 'indexed', 'scanned', 'recordsRead', 'plan'] + */ + public function scanRecords($dbname, array $plan, callable $visit, $whole = false){ + $stats = ['segments' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0, 'recordsRead' => 0, 'plan' => []]; + $segments = $this->segments->listSegments($dbname); + if(empty($segments)){ + return $stats; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($segments as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + $done = false; + } + $stats['segments']++; + if($shards !== null && !isset($shards[$segment])){ + $stats['skipped']++; + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } + continue; + } + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + $access = $keys === null ? 'scan' : 'index'; + if($keys === null){ + $stats['scanned']++; + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + $count = $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index); + // A scan that can't stop early reads the segment as a whole (RecordIO::eachSegmentBatch()): no key list + $keys = $whole || $count === 0 ? null : \noneDB\Internal\Storage\KeyIndex::keys($index); + } else { + $stats['indexed']++; + $count = count($keys); + } + if(count($stats['plan']) < 50){ + $stats['plan'][] = ['segment' => $segment, 'access' => $access, 'candidates' => $count]; + } + if($count === 0){ + continue; + } + if($keys === null){ + foreach($this->records->eachSegmentBatch($dbname, $segment) as $records){ + $stats['recordsRead'] += count($records); + if($visit($records) === false){ + return $stats; + } + } + $done = true; + continue; + } + // Keys come in ascending order almost always (they are assigned in order, an update keeps a + // key's place in the index): sorted only when they aren't + $previous = PHP_INT_MIN; + foreach($keys as $k){ + if($k < $previous){ + sort($keys, SORT_NUMERIC); + break; + } + $previous = $k; + } + if($whole){ + // Index candidates for a visitor that never stops: a batch at a time (RecordIO::BATCH) + foreach($this->records->eachSegmentBatch($dbname, $segment, $keys) as $records){ + $stats['recordsRead'] += count($records); + if($visit($records) === false){ + return $stats; + } + } + $done = true; + continue; + } + // Batches that grow from 32: a visitor that stops early reads little + $n = count($keys); + for($i = 0, $batch = 32; $i < $n; $i += $batch, $batch = min(4096, $batch * 4)){ + $records = $this->records->readSegmentRecords($dbname, $segment, $access === 'scan' && $i === 0 && $batch >= $n ? null : array_slice($keys, $i, $batch)); + $stats['recordsRead'] += count($records); + if(!empty($records) && $visit($records) === false){ + return $stats; + } + } + $done = true; + } + return $stats; + } + + /** + * explain(): what scanRecords() would do, without reading records or building anything (an index that + * a read would build counts as a scan) + */ + public function explainPlan($dbname, array $plan){ + $explain = function() use ($dbname, $plan){ + $out = ['sharded' => $this->segments->dbExists($dbname) && $this->segments->isSharded($dbname), 'segments' => ['total' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0], + 'plan' => [], 'estimatedRead' => 0]; + if(!$this->segments->dbExists($dbname)){ + return $out; + } + $shards = $plan[0] === 'any' ? null : $this->planShards($dbname, $plan); + foreach($this->segments->listSegments($dbname) as $segment){ + $out['segments']['total']++; + if($shards !== null && !isset($shards[$segment])){ + $out['segments']['skipped']++; + $entry = ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; + } else { + $keys = $plan[0] === 'any' ? null : $this->planCandidates($dbname, $segment, $plan); + if($keys === null){ + $out['segments']['scanned']++; + $index = $this->checkpoints->readJsonlIndex($dbname, $segment); + $entry = ['segment' => $segment, 'access' => 'scan', 'candidates' => $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index)]; + } else { + $out['segments']['indexed']++; + $entry = ['segment' => $segment, 'access' => 'index', 'candidates' => count($keys)]; + } + } + $out['estimatedRead'] += $entry['candidates']; + if(count($out['plan']) < 50){ + $out['plan'][] = $entry; + } + unset($index, $keys); + $this->cache->evictBetweenSegments(); + } + $out['planTruncated'] = $out['segments']['total'] > 50; + return $out; + }; + $this->noWrites++; + try { + return $this->locks->withLock($dbname, false, $explain); + } finally { + $this->noWrites--; + } + } + + + /** + * Visit the records matching the filter parameter of find(), count(), sum(), ... (FindFilter::normalizeFilter) + * in find()'s order, none kept: $visit([key => record]) gets the matching records of each batch read and + * returns false to stop. One segment's records are held at a time (a "key" filter: the records of its keys, + * in its order, in one batch); indexes plan the scan. False for an invalid filter or a missing database + * without autoCreateDB (where find() returns false). Caller holds the lock. + * @param bool $whole The visitor never stops (count, aggregations): each segment is read in one go (RecordIO::BATCH records at a time when it is larger) + * @return bool + */ + public function eachMatch($dbname, $filter, callable $visit, $whole = false){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null){ + return false; + } + if(!$this->segments->dbExists($dbname)){ + return (bool)$this->autoCreateDB; + } + if(array_key_exists('key', $filters)){ + $records = $this->findFilter->selectRecords($dbname, $filters); + if(!empty($records)){ + $visit($records); + } + return true; + } + $and = []; + foreach($filters as $field => $value){ + $and[] = is_array($value) ? ['any'] : ['eq', (string)$field, $value]; + } + $plan = count($and) === 1 ? $and[0] : ['and', $and]; + if($plan[0] === 'and'){ + $plan[1] = array_values(array_filter($plan[1], function($node){ return $node[0] !== 'any'; })); + $plan = empty($plan[1]) ? ['any'] : (count($plan[1]) === 1 ? $plan[1][0] : $plan); + } + $this->scanRecords($dbname, $plan, empty($filters) ? $visit : function(array $records) use ($filters, $visit){ + $matches = []; + foreach($records as $key => $record){ + if($this->findFilter->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + return empty($matches) || $visit($matches) !== false; + }, $whole); + return true; + } +} diff --git a/src/Internal/Query/QueryMethods.php b/src/Internal/Query/QueryMethods.php new file mode 100644 index 0000000..838eecf --- /dev/null +++ b/src/Internal/Query/QueryMethods.php @@ -0,0 +1,186 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->findFilter, $this->layout, $this->locks, $this->planner, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->findFilter = null; + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // QUERY METHODS + // ========================================== + + /** + * Get unique values for a field + * @param string $dbname + * @param string $field + * @return array|false + */ + public function distinct($dbname, $field){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->distinctUnlocked($dbname, $field); + }); + } + + private function distinctUnlocked($dbname, $field){ + $seen = []; + $values = []; + $found = $this->planner->eachMatch($dbname, [], function(array $records) use ($field, &$seen, &$values){ + \noneDB\Internal\Query\Values::collectUnique($records, $field, $seen, $values); + }, true); + return $found ? $values : false; + } + + /** + * Sort array by field + * @param array $array Result from find() + * @param string $field Field to sort by + * @param string $order 'asc' or 'desc' + * @return array|false + */ + public function sort($array, $field, $order = 'asc'){ + if(!is_array($array)) return false; + if(count($array) === 0) return []; + $desc = strtolower((string)$order) === 'desc'; + $items = []; + foreach(array_values($array) as $i => $row){ + $items[] = [\noneDB\Internal\Query\Values::sortKey(is_array($row) && array_key_exists($field, $row) ? $row[$field] : null), $i, $row]; + } + usort($items, function($a, $b) use ($desc){ + $result = $a[0] <=> $b[0]; + if($desc){ + $result = -$result; + } + return $result !== 0 ? $result : $a[1] <=> $b[1]; // stable + }); + return array_column($items, 2); + } + + /** + * Count records matching filter + * @param string $dbname + * @param mixed $filter + * @return int + */ + public function count($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->countUnlocked($dbname, $filter); + }); + } + + private function countUnlocked($dbname, $filter = 0){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null){ + return 0; // a filter find() rejects matches nothing + } + // Fast-path: no filter = use index/meta counters + if(empty($filters)){ + return $this->countFast($dbname); + } + $count = 0; + $this->planner->eachMatch($dbname, $filters, function(array $records) use (&$count){ $count += count($records); }, true); + return $count; + } + + /** + * Count without reading records: index entries (non-sharded) or meta totalRecords (sharded) + * @param string $dbname Already normalized + * @return int + */ + public function countFast($dbname){ + if(!$this->segments->dbExists($dbname)){ + return 0; + } + if($this->segments->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + return $meta !== null ? (int)($meta['totalRecords'] ?? 0) : 0; + } + $index = $this->checkpoints->readJsonlIndex($dbname, null); + return $index !== null ? \noneDB\Internal\Storage\KeyIndex::count($index) : 0; + } + + /** + * Pattern matching search (LIKE) + * @param string $dbname + * @param string $field + * @param string $pattern Use ^ for starts with, $ for ends with + * @return array|false + */ + public function like($dbname, $field, $pattern){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $pattern){ + return $this->likeUnlocked($dbname, $field, $pattern); + }); + } + + private function likeUnlocked($dbname, $field, $pattern){ + if(!is_scalar($pattern)){ + return false; // an array is never a pattern subject + } + $result = []; + $like = \noneDB\Internal\Query\Values::compileLike((string)$pattern); + $found = $this->planner->eachMatch($dbname, [], function(array $records) use ($field, $like, &$result){ + foreach($records as $record){ + $value = $record[$field] ?? null; + // Only strings and numbers (true used to match "1", like in the query builder it never does) + if(is_scalar($value) && !is_bool($value) && \noneDB\Internal\Query\Values::likeMatch($like, (string)$value)){ + $result[] = $record; + } + } + }, true); + return $found ? $result : false; + } +} diff --git a/src/Internal/Query/SpatialQueries.php b/src/Internal/Query/SpatialQueries.php new file mode 100644 index 0000000..6b1f5a2 --- /dev/null +++ b/src/Internal/Query/SpatialQueries.php @@ -0,0 +1,797 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->geoJson, $this->geometry, $this->layout, $this->locks, $this->queryMethods, $this->records, $this->rtree, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->indexedCandidates = &$this->k->indexedCandidates; + $this->spatialIndexEnabled = &$this->k->spatialIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->geoJson = null; + $this->geometry = null; + $this->layout = null; + $this->locks = null; + $this->queryMethods = null; + $this->records = null; + $this->rtree = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SPATIAL QUERIES + // ========================================== + + /** + * Records whose geometry may overlap the search boxes, in key order, one segment at a time (the cache may + * drop what a segment loaded once it is done): yields key => record. Uses the R-tree (and shard boxes) + * when the field is indexed; otherwise every record. + * @param array $boxes List of MBRs + * @return \Generator + */ + private function spatialCandidates(string $dbname, string $field, array $boxes) { + if (!$this->segments->dbExists($dbname)) { + return; + } + $indexed = $this->spatialIndexEnabled && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + $this->indexedCandidates = []; + $sharded = $this->segments->isSharded($dbname); + $global = ($indexed && $sharded) ? $this->rtree->readGlobalSpatialIndex($dbname, $field) : null; + + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach ($this->segments->listSegments($dbname) as $segment) { + if ($done) { + $this->cache->evictBetweenSegments(); + $done = false; + } + $keys = null; // null: every record of the segment + // Shard skip: only with an entry that fits the shard, misses the boxes, and no newer geometry of + // the shard in them. A missing entry (or global file) means unknown: searched. + if ($indexed && $global !== null && $this->rtree->globalSpatialSkips($dbname, $field, $segment, $global, $boxes)) { + continue; + } + if ($indexed) { + try { + $keys = $this->rtree->rtreeSearchMBR($dbname, $field, $boxes, $segment); + foreach ($keys as $key) { + $this->indexedCandidates[$key] = true; + } + } catch (\Throwable $e) { + if ($e->getCode() === \noneDB::LOCK_FAILED) { + throw $e; + } + $keys = null; // the index can't be built: scan + } + } + if ($keys === null || !empty($keys)) { + foreach ($this->records->eachSegmentBatch($dbname, $segment, $keys) as $batch) { + yield from $batch; + } + } + $done = true; + } + } + + /** + * Search boxes for a circle; splits at the antimeridian and covers all longitudes near the poles + * @return array List of [minLon, minLat, maxLon, maxLat] + */ + private function circleToBBoxes(float $lon, float $lat, float $radiusMeters): array { + $latDeg = $radiusMeters / 111000.0; + $minLat = $lat - $latDeg; + $maxLat = $lat + $latDeg; + if ($minLat <= -90 || $maxLat >= 90) { + return [[-180.0, max(-90.0, $minLat), 180.0, min(90.0, $maxLat)]]; + } + $cos = cos(deg2rad(max(abs($minLat), abs($maxLat)))); + $lonDeg = $cos > 1e-9 ? $radiusMeters / (111000.0 * $cos) : 360.0; + if ($lonDeg >= 180) { + return [[-180.0, $minLat, 180.0, $maxLat]]; + } + $minLon = $lon - $lonDeg; + $maxLon = $lon + $lonDeg; + if ($minLon < -180) { + return [[-180.0, $minLat, $maxLon, $maxLat], [$minLon + 360.0, $minLat, 180.0, $maxLat]]; + } + if ($maxLon > 180) { + return [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon - 360.0, $maxLat]]; + } + return [[$minLon, $minLat, $maxLon, $maxLat]]; + } + + /** + * Distance in meters from a point to a geometry: 0 inside a polygon, otherwise to the nearest + * vertex or edge (edges use a local equirectangular approximation) + */ + public function distanceToGeometry(float $lon, float $lat, array $geometry): float { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + switch ($type) { + case 'Point': + return $this->geometry->haversineDistance($lon, $lat, (float)$coords[0], (float)$coords[1]); + case 'MultiPoint': + $best = INF; + foreach ($coords as $p) { + $best = min($best, $this->geometry->haversineDistance($lon, $lat, (float)$p[0], (float)$p[1])); + } + return $best; + case 'LineString': + return $this->distanceToPath($lon, $lat, $coords); + case 'MultiLineString': + $best = INF; + foreach ($coords as $line) { + $best = min($best, $this->distanceToPath($lon, $lat, $line)); + } + return $best; + case 'Polygon': + if ($this->geometry->pointInPolygon($lon, $lat, $coords)) { + return 0.0; + } + $best = INF; + foreach ($coords as $ring) { + $best = min($best, $this->distanceToPath($lon, $lat, $ring)); + } + return $best; + case 'MultiPolygon': + $best = INF; + foreach ($coords as $polygon) { + $best = min($best, $this->distanceToGeometry($lon, $lat, ['type' => 'Polygon', 'coordinates' => $polygon])); + } + return $best; + case 'GeometryCollection': + $best = INF; + foreach ($geometry['geometries'] ?? [] as $g) { + $best = min($best, $this->distanceToGeometry($lon, $lat, $g)); + } + return $best; + } + return INF; + } + + /** + * Distance from a point to a polyline (meters): to its nearest point, an edge being straight in + * longitude/latitude. Exact up to floating point: every vertex is measured, and every edge whose + * bounding box could hold a nearer point is searched (golden section on the haversine of the point at + * t along the edge, which wraps at the antimeridian by itself); the other edges are skipped by a lower + * bound of the haversine over their box. + */ + private function distanceToPath(float $lon, float $lat, array $points): float { + $n = count($points); + if ($n === 0) { + return INF; + } + $cosLat = cos(deg2rad($lat)); + $xs = []; + $ys = []; + $best = INF; // the haversine "a" of the nearest point found (monotonic in the distance) + foreach ($points as $i => $point) { + $xs[$i] = (float)$point[0]; + $ys[$i] = (float)$point[1]; + $best = min($best, $this->haversineA($lon, $lat, $cosLat, $xs[$i], $ys[$i])); + } + $points = array_keys($xs); + $bounds = []; + for ($e = 0; $e < $n - 1; $e++) { + $i = $points[$e]; + $j = $points[$e + 1]; + $bound = $this->edgeBoxLowerBound($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j]); + if ($bound < $best) { + $bounds[$e] = $bound; + } + } + asort($bounds); + foreach ($bounds as $e => $bound) { + if ($bound >= $best) { + break; // every other edge is at least this far + } + $i = $points[$e]; + $j = $points[$e + 1]; + $best = min($best, $this->edgeMinimumA($lon, $lat, $cosLat, $xs[$i], $ys[$i], $xs[$j], $ys[$j])); + } + return 6371000.0 * (2 * atan2(sqrt($best), sqrt(1 - $best))); + } + + /** + * The "a" of haversineDistance() (the same operations, so a vertex gives the same distance) + */ + private function haversineA(float $lon, float $lat, float $cosLat, float $x, float $y): float { + $latDelta = deg2rad($y - $lat); + $lonDelta = deg2rad($x - $lon); + return sin($latDelta / 2) * sin($latDelta / 2) + $cosLat * cos(deg2rad($y)) * sin($lonDelta / 2) * sin($lonDelta / 2); + } + + /** + * A lower bound of haversineA() over the bounding box of an edge: a = hav(dlat) + cos(lat) cos(lat2) + * hav(dlon), each term bounded by the box's smallest latitude and longitude difference to the point + * and its smallest cosine + */ + private function edgeBoxLowerBound(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $loY = min($ay, $by); + $hiY = max($ay, $by); + $dLat = $lat < $loY ? $loY - $lat : ($lat > $hiY ? $lat - $hiY : 0.0); + $loX = min($ax, $bx); + $width = max($ax, $bx) - $loX; + $dLon = 0.0; + if ($width < 360) { + $offset = fmod($lon - $loX, 360.0); + if ($offset < 0) { + $offset += 360.0; + } + if ($offset > $width) { + $dLon = min($offset - $width, 360.0 - $offset); + } + } + $cosMin = max(0.0, min(cos(deg2rad($loY)), cos(deg2rad($hiY)))); + $sLat = sin(deg2rad($dLat) / 2); + $sLon = sin(deg2rad($dLon) / 2); + return $sLat * $sLat + $cosLat * $cosMin * $sLon * $sLon; + } + + /** + * The smallest haversineA() of the points of an edge: golden-section search over t (60 steps: t to + * about 1e-13), plus the point a local flat projection picks, whichever is nearer + */ + private function edgeMinimumA(float $lon, float $lat, float $cosLat, float $ax, float $ay, float $bx, float $by): float { + $dx = $bx - $ax; + $dy = $by - $ay; + $at = function (float $t) use ($lon, $lat, $cosLat, $ax, $ay, $dx, $dy) { + return $this->haversineA($lon, $lat, $cosLat, $ax + $dx * $t, $ay + $dy * $t); + }; + // Local projection around the point, the longitude of the edge's start taken on the point's side + $px = fmod($ax - $lon + 540.0, 360.0) - 180.0; + $qx = ($px + $dx) * $cosLat; + $px *= $cosLat; + $py = $ay - $lat; + $qy = $by - $lat; + $ex = $qx - $px; + $ey = $qy - $py; + $len2 = $ex * $ex + $ey * $ey; + $best = $at($len2 > 0 ? max(0.0, min(1.0, -($px * $ex + $py * $ey) / $len2)) : 0.0); + $ratio = 0.6180339887498949; + $lo = 0.0; + $hi = 1.0; + $c = $hi - $ratio * ($hi - $lo); + $d = $lo + $ratio * ($hi - $lo); + $fc = $at($c); + $fd = $at($d); + for ($step = 0; $step < 60; $step++) { + if ($fc < $fd) { + $hi = $d; + $d = $c; + $fd = $fc; + $c = $hi - $ratio * ($hi - $lo); + $fc = $at($c); + } else { + $lo = $c; + $c = $d; + $fc = $fd; + $d = $lo + $ratio * ($hi - $lo); + $fd = $at($d); + } + } + return min($best, $fc, $fd); + } + + /** + * Find records within a distance from a point (distance to the nearest point of the geometry) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @param array $options ['includeDistance' => false] + * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) + */ + public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['includeDistance'], 'option', ' of withinDistance()'); + if(array_key_exists('includeDistance', $options)){ + \noneDB\Internal\Config\Configuration::optionBool('includeDistance', $options['includeDistance']); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $distanceMeters, $options){ + return $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $distanceMeters, $options); + }); + } + + private function withinDistanceUnlocked(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + $includeDistance = $options['includeDistance'] ?? false; + if ($distanceMeters < 0 || is_nan($distanceMeters) || !is_finite($lon) || !is_finite($lat)) { + return []; + } + $distanceMeters = min($distanceMeters, self::EARTH_HALF_CIRCUMFERENCE); // infinite: everything + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $this->circleToBBoxes($lon, $lat, $distanceMeters)) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && $distance <= $distanceMeters) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + } + } + usort($results, function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + return $results; + } + + /** + * Find records within a bounding box (geometries whose bounding box overlaps it). + * minLon > maxLon means the box crosses the antimeridian. + * @return array Matching records + */ + public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $minLon, $minLat, $maxLon, $maxLat){ + return $this->withinBBoxUnlocked($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); + }); + } + + private function withinBBoxUnlocked(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + if (is_nan($minLon) || is_nan($minLat) || is_nan($maxLon) || is_nan($maxLat)) { + return []; + } + // Infinite edges reach the edge of the map + $minLon = max(-180.0, min(180.0, $minLon)); + $maxLon = max(-180.0, min(180.0, $maxLon)); + $minLat = max(-90.0, min(90.0, $minLat)); + $maxLat = max(-90.0, min(90.0, $maxLat)); + if ($minLat > $maxLat) { + [$minLat, $maxLat] = [$maxLat, $minLat]; + } + $boxes = $minLon <= $maxLon + ? [[$minLon, $minLat, $maxLon, $maxLat]] + : [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon, $maxLat]]; + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + $recordMBR = $this->geometry->calculateMBR($geometry); + if ($recordMBR === null) { + continue; + } + foreach ($boxes as $box) { + if ($this->geometry->mbrOverlaps($recordMBR, $box)) { + $results[] = $record; + break; + } + } + } + return $results; + } + + /** + * Find K nearest records to a point + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] + * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + */ + public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + \noneDB\Internal\Config\Configuration::checkNames($options, ['maxDistance', 'includeDistance'], 'option', ' of nearest()'); + if(array_key_exists('maxDistance', $options) && $options['maxDistance'] !== null && !is_int($options['maxDistance']) && !is_float($options['maxDistance'])){ + throw new \InvalidArgumentException("noneDB: 'maxDistance' must be a number of meters or null"); + } + if(array_key_exists('includeDistance', $options)){ + \noneDB\Internal\Config\Configuration::optionBool('includeDistance', $options['includeDistance']); + } + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $lon, $lat, $k, $options){ + return $this->nearestUnlocked($dbname, $field, $lon, $lat, $k, $options); + }); + } + + /** + * True when spatial queries on the field use the R-tree (used by the query builder) + */ + public function usesSpatialIndex(string $dbname, string $field): bool{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field){ + return $this->spatialIndexEnabled && $this->segments->dbExists($dbname) + && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + }); + } + + private function nearestUnlocked(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + $maxDistance = isset($options['maxDistance']) ? (float)$options['maxDistance'] : null; // an int or a float (nearest()) + $includeDistance = $options['includeDistance'] ?? true; + // NaN matches nothing (and NaN radii made the expanding search below loop forever) + if ($k <= 0 || !is_finite($lon) || !is_finite($lat) || ($maxDistance !== null && is_nan($maxDistance)) || !$this->segments->dbExists($dbname)) { + return []; + } + + $indexed = $this->spatialIndexEnabled && in_array($field, $this->fieldIndex->getIndexManifest($dbname)['spatial'], true); + // Half the earth's circumference covers everything (also an infinite maxDistance) + $limit = min($maxDistance ?? self::EARTH_HALF_CIRCUMFERENCE, self::EARTH_HALF_CIRCUMFERENCE); + $results = []; + + if ($indexed && $k < $this->queryMethods->countFast($dbname)) { + // Expanding search: nothing outside radius r is closer than r. Results are ordered by the + // rounded distance, so only records rounded below round(r) are certainly in final order + // (a record just outside r can round to the same value and has to win a key tie). + $radius = 1000.0; + while (true) { + $radius = min($radius, $limit); + $found = $this->withinDistanceUnlocked($dbname, $field, $lon, $lat, $radius, ['includeDistance' => true]); + if ($radius >= $limit || (count($found) >= $k && $found[$k - 1]['_distance'] < round($radius, 2))) { + $results = $found; + break; + } + $radius *= 4; + } + } else { + // Every record by its distance; only the best k are kept (pruned as they come) + $order = function ($a, $b) { + return $a['_distance'] <=> $b['_distance'] ?: $a['key'] <=> $b['key']; + }; + $prune = max(4 * $k, 1024); + foreach ($this->segments->listSegments($dbname) as $segment) { + foreach ($this->records->eachSegmentBatch($dbname, $segment) as $batch) { + foreach ($batch as $record) { + $geometry = $record[$field] ?? null; + if (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid']) { + continue; + } + $distance = $this->distanceToGeometry($lon, $lat, $geometry); + if (is_finite($distance) && ($maxDistance === null || $distance <= $maxDistance)) { + $record['_distance'] = round($distance, 2); + $results[] = $record; + if (count($results) > $prune) { + usort($results, $order); + $results = array_slice($results, 0, $k); + } + } + } + } + $this->cache->evictBetweenSegments(); + } + usort($results, $order); + } + + $results = array_slice($results, 0, $k); + if (!$includeDistance) { + foreach ($results as &$r) { + unset($r['_distance']); + } + unset($r); + } + return $results; + } + + /** + * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon or MultiPolygon geometry + * @return array Matching records + */ + public function withinPolygon(string $dbname, string $field, array $polygon): array{ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $polygon){ + return $this->withinPolygonUnlocked($dbname, $field, $polygon); + }); + } + + private function withinPolygonUnlocked(string $dbname, string $field, array $polygon): array{ + $validation = $this->geoJson->validateGeoJSON($polygon); + if (!$validation['valid'] || !in_array($validation['type'], ['Polygon', 'MultiPolygon'], true)) { + return []; + } + $polygons = $validation['type'] === 'Polygon' ? [$polygon['coordinates']] : $polygon['coordinates']; + $boxes = []; + foreach ($polygons as $coords) { + $boxes[] = $this->geometry->mbrFromPoints($coords[0]); + } + + // One point inside each hole of the query: a record polygon that contains one covers part of the hole + $parts = []; + foreach ($polygons as $coords) { + $holePoints = []; + for ($h = 1; $h < count($coords); $h++) { + $point = $this->pointInsideRing($coords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + $parts[] = ['coords' => $coords, 'holePoints' => $holePoints]; + } + + $results = []; + foreach ($this->spatialCandidates($dbname, $field, $boxes) as $record) { + $geometry = $record[$field] ?? null; + if (!isset($this->indexedCandidates[$record['key']]) && (!is_array($geometry) || !$this->geoJson->validateGeoJSON($geometry)['valid'])) { + continue; // a candidate of an R-tree has a valid geometry: checked when it was indexed + } + if ($this->geometryWithinParts($geometry, $parts)) { + $results[] = $record; + } + } + return $results; + } + + /** + * Within a Polygon or MultiPolygon query (its parts): a single geometry must fit one part; every + * component of a multi-part geometry must fit some part. A component spanning two adjacent + * parts doesn't fit (the union of the parts isn't computed). + */ + private function geometryWithinParts(array $geometry, array $parts): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + $components = null; + if ($type === 'MultiPoint') { + $components = array_map(function ($p) { return ['type' => 'Point', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'MultiLineString') { + $components = array_map(function ($l) { return ['type' => 'LineString', 'coordinates' => $l]; }, $coords); + } elseif ($type === 'MultiPolygon') { + $components = array_map(function ($p) { return ['type' => 'Polygon', 'coordinates' => $p]; }, $coords); + } elseif ($type === 'GeometryCollection') { + // Members without a position don't count; a collection without any position is never within + $components = array_values(array_filter($geometry['geometries'] ?? [], function ($g) { + return is_array($g) && $this->geometry->calculateMBR($g) !== null; + })); + } + if ($components !== null) { + if (empty($components)) { + return false; + } + foreach ($components as $component) { + if (!$this->geometryWithinParts($component, $parts)) { + return false; + } + } + return true; + } + foreach ($parts as $part) { + if ($this->geometryWithinPolygon($geometry, $part['coords'], $part['holePoints'])) { + return true; + } + } + return false; + } + + /** + * A point strictly inside a ring: the middle of the first span a horizontal line between the two + * lowest distinct y values has inside the ring. Null for a degenerate ring. + */ + private function pointInsideRing(array $ring): ?array { + $ys = array_values(array_unique(array_map(function ($p) { return (float)$p[1]; }, $ring))); + sort($ys); + if (count($ys) < 2) { + return null; + } + $y = ($ys[0] + $ys[1]) / 2; + $xs = []; + $n = count($ring); + for ($i = 0; $i < $n - 1; $i++) { + [$x1, $y1] = [(float)$ring[$i][0], (float)$ring[$i][1]]; + [$x2, $y2] = [(float)$ring[$i + 1][0], (float)$ring[$i + 1][1]]; + if (($y1 <= $y && $y2 > $y) || ($y2 <= $y && $y1 > $y)) { + $xs[] = $x1 + ($y - $y1) / ($y2 - $y1) * ($x2 - $x1); + } + } + sort($xs); + return count($xs) >= 2 ? [($xs[0] + $xs[1]) / 2, $y] : null; + } + + /** + * Check if a geometry lies entirely within a polygon. The polygon is closed (its boundary, also a + * hole's boundary, belongs to it, as in withinBBox). Every vertex is inside, no edge crosses a ring of + * the polygon, and every piece of an edge between polygon vertices on it is inside; a record polygon + * must also not contain a point inside one of the polygon's holes ($holePoints). + */ + private function geometryWithinPolygon(array $geometry, array $polygonCoords, ?array $holePoints = null): bool { + $type = $geometry['type'] ?? ''; + $coords = $geometry['coordinates'] ?? []; + if ($holePoints === null) { + $holePoints = []; + for ($h = 1; $h < count($polygonCoords); $h++) { + $point = $this->pointInsideRing($polygonCoords[$h]); + if ($point !== null) { + $holePoints[] = $point; + } + } + } + + switch ($type) { + case 'Point': + return $this->pointInPolygonClosed((float)$coords[0], (float)$coords[1], $polygonCoords); + case 'MultiPoint': + foreach ($coords as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + return true; + case 'LineString': + return $this->pathWithinPolygon($coords, $polygonCoords); + case 'MultiLineString': + foreach ($coords as $line) { + if (!$this->pathWithinPolygon($line, $polygonCoords)) { + return false; + } + } + return true; + case 'Polygon': + return $this->polygonWithinPolygon($coords, $polygonCoords, $holePoints); + case 'MultiPolygon': + foreach ($coords as $poly) { + if (!$this->polygonWithinPolygon($poly, $polygonCoords, $holePoints)) { + return false; + } + } + return true; + case 'GeometryCollection': + // Members without a position don't count; a collection without any position is never within + $located = false; + foreach ($geometry['geometries'] ?? [] as $g) { + if (is_array($g) && $this->geometry->calculateMBR($g) === null) { + continue; + } + if (!$this->geometryWithinPolygon($g, $polygonCoords)) { + return false; + } + $located = true; + } + return $located; + } + return false; + } + + /** + * A polygon (record) within a polygon (query): all its rings within, and no point inside one of the + * query's holes inside it (a record around a hole has its whole boundary inside the query) + */ + private function polygonWithinPolygon(array $rings, array $polygonCoords, array $holePoints): bool { + foreach ($rings as $ring) { + if (!$this->pathWithinPolygon($ring, $polygonCoords)) { + return false; + } + } + foreach ($holePoints as [$x, $y]) { + if ($this->geometry->pointInPolygon($x, $y, $rings)) { + return false; + } + } + return true; + } + + /** + * Point inside a polygon or on its boundary (holes excluded, their boundary included) + */ + private function pointInPolygonClosed(float $lon, float $lat, array $polygonCoords): bool { + if (!$this->geoJson->pointOnRing($lon, $lat, $polygonCoords[0]) && !$this->geometry->pointInRing($lon, $lat, $polygonCoords[0])) { + return false; + } + for ($i = 1; $i < count($polygonCoords); $i++) { + if ($this->geometry->pointInRing($lon, $lat, $polygonCoords[$i]) && !$this->geoJson->pointOnRing($lon, $lat, $polygonCoords[$i])) { + return false; + } + } + return true; + } + + /** + * A path within the closed polygon: every vertex inside, no segment properly crossing a ring, and + * every piece of a segment between polygon vertices lying on it inside (a segment can run along a + * boundary or through a hole between two of its vertices) + */ + private function pathWithinPolygon(array $points, array $polygonCoords): bool { + foreach ($points as $point) { + if (!$this->pointInPolygonClosed((float)$point[0], (float)$point[1], $polygonCoords)) { + return false; + } + } + $n = count($points); + for ($i = 0; $i < $n - 1; $i++) { + $a = [(float)$points[$i][0], (float)$points[$i][1]]; + $b = [(float)$points[$i + 1][0], (float)$points[$i + 1][1]]; + $ts = [0.0, 1.0]; + $alongX = abs($b[0] - $a[0]) >= abs($b[1] - $a[1]); + foreach ($polygonCoords as $ring) { + $m = count($ring); + for ($j = 0; $j < $m - 1; $j++) { + if ($this->segmentsCrossProperly($a, $b, $ring[$j], $ring[$j + 1])) { + return false; + } + $v = [(float)$ring[$j][0], (float)$ring[$j][1]]; + if ($this->geometry->crossProductDirection($a, $b, $v) == 0 && $this->geometry->onSegment($a, $b, $v) && $a != $b) { + $ts[] = $alongX ? ($v[0] - $a[0]) / ($b[0] - $a[0]) : ($v[1] - $a[1]) / ($b[1] - $a[1]); + } + } + } + if (count($ts) > 2) { + sort($ts); + for ($k = 0; $k < count($ts) - 1; $k++) { + if ($ts[$k + 1] - $ts[$k] <= 0) { + continue; + } + $t = ($ts[$k] + $ts[$k + 1]) / 2; + if (!$this->pointInPolygonClosed($a[0] + $t * ($b[0] - $a[0]), $a[1] + $t * ($b[1] - $a[1]), $polygonCoords)) { + return false; + } + } + } elseif (!$this->pointInPolygonClosed(($a[0] + $b[0]) / 2, ($a[1] + $b[1]) / 2, $polygonCoords)) { + return false; // e.g. a chord between two boundary vertices across a concave notch + } + } + return true; + } + + /** + * True when two segments cross at a single interior point (touching endpoints don't count) + */ + private function segmentsCrossProperly(array $p1, array $p2, array $p3, array $p4): bool { + $d1 = $this->geometry->crossProductDirection($p3, $p4, $p1); + $d2 = $this->geometry->crossProductDirection($p3, $p4, $p2); + $d3 = $this->geometry->crossProductDirection($p1, $p2, $p3); + $d4 = $this->geometry->crossProductDirection($p1, $p2, $p4); + return (($d1 > 0 && $d2 < 0) || ($d1 < 0 && $d2 > 0)) && (($d3 > 0 && $d4 < 0) || ($d3 < 0 && $d4 > 0)); + } + + public const EARTH_HALF_CIRCUMFERENCE = 20040000.0; // meters: no two points are farther apart +} diff --git a/src/Internal/Query/Utility.php b/src/Internal/Query/Utility.php new file mode 100644 index 0000000..653fbed --- /dev/null +++ b/src/Internal/Query/Utility.php @@ -0,0 +1,200 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->findFilter, $this->layout, $this->locks, $this->planner, $this->records, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->fieldIndexEnabled = &$this->k->fieldIndexEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->findFilter = null; + $this->layout = null; + $this->locks = null; + $this->planner = null; + $this->records = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // UTILITY METHODS + // ========================================== + + /** + * Get first matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function first($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->firstUnlocked($dbname, $filter); + }); + } + + private function firstUnlocked($dbname, $filter = 0){ + $first = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use (&$first){ $first = reset($records); return false; }); + return $first; + } + + /** + * Get last matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function last($dbname, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->lastUnlocked($dbname, $filter); + }); + } + + private function lastUnlocked($dbname, $filter = 0){ + $last = null; + $this->planner->eachMatch($dbname, $filter, function(array $records) use (&$last){ $last = end($records); }, true); + return $last; + } + + /** + * Check if records exist matching filter + * @param string $dbname + * @param mixed $filter + * @return bool + */ + public function exists($dbname, $filter){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $filter){ + return $this->existsUnlocked($dbname, $filter); + }); + } + + private function existsUnlocked($dbname, $filter){ + $found = false; + $this->planner->eachMatch($dbname, $filter, function() use (&$found){ $found = true; return false; }); + return $found; + } + + /** + * Range query (min <= value <= max) + * @param string $dbname + * @param string $field + * @param mixed $min + * @param mixed $max + * @param mixed $filter Additional filter + * @return array|false + */ + public function between($dbname, $field, $min, $max, $filter = 0){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname, $field, $min, $max, $filter){ + return $this->betweenUnlocked($dbname, $field, $min, $max, $filter); + }); + } + + private function betweenUnlocked($dbname, $field, $min, $max, $filter = 0){ + // Same comparison as query()->between(): numbers (and numeric strings) numerically, strings + // lexically; null, booleans, arrays and number-vs-text are never in range + $result = []; + $collect = function(array $records) use ($field, $min, $max, &$result){ + foreach($records as $record){ + if(!array_key_exists($field, $record)){ + continue; + } + $lo = \noneDB\Internal\Query\Values::compareValues($record[$field], $min); + $hi = \noneDB\Internal\Query\Values::compareValues($record[$field], $max); + if($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0){ + $result[] = $record; + } + } + }; + // An indexed field: only the records whose value the field index puts in range are read + if(!$this->eachRangeCandidate($dbname, $field, ['$gte' => $min, '$lte' => $max], $filter, $collect) + && !$this->planner->eachMatch($dbname, $filter, $collect, true)){ + return false; + } + return $result; + } + + /** + * Visit the records matching a filter parameter whose field value may satisfy the range conditions, read + * through the field index of every segment (a superset: the visitor checks the range), in key order, one + * segment at a time: $visit([key => record]) gets a segment's (never none). False when it can't: no index + * of the field, a key filter decides, an invalid filter or no database (the caller scans). + * @return bool + */ + private function eachRangeCandidate($dbname, $field, array $conditions, $filter, callable $visit){ + $filters = $this->findFilter->normalizeFilter($filter); + if($filters === null || array_key_exists('key', $filters) || !$this->segments->dbExists($dbname) || !$this->fieldIndexEnabled + || !$this->fieldIndex->isFieldIndexed($dbname, $field)){ + return false; + } + $done = false; // a segment was read: the cache may drop what it loaded before the next one (not after the last) + foreach($this->segments->listSegments($dbname) as $segment){ + if($done){ + $this->cache->evictBetweenSegments(); + } + $done = true; + $keys = $this->fieldIndex->fieldIndexRangeCandidates($dbname, $segment, $field, $conditions); + if($keys === null || !empty($keys)){ + foreach($this->records->eachSegmentBatch($dbname, $segment, $keys) as $batch){ + $matches = []; + foreach($batch as $key => $record){ + if($this->findFilter->matchesFilters($record, $filters)){ + $matches[$key] = $record; + } + } + if(!empty($matches)){ + $visit($matches); + } + } + } + } + return true; + } +} diff --git a/src/Internal/Query/Values.php b/src/Internal/Query/Values.php new file mode 100644 index 0000000..a79a174 --- /dev/null +++ b/src/Internal/Query/Values.php @@ -0,0 +1,235 @@ +k = $k; + $this->rt = $k->rt; + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->k = null; + $this->rt = null; + } + + + /** + * valueKey() for the query builder + * @return \Closure + */ + public function valueKeyFunction(){ + return \Closure::fromCallable([self::class, 'valueKey']); + } + + /** + * Add the records' values of a field to the distinct values seen so far (null excluded), in first-seen + * order, same identity as in_array($value, $values, true), in linear time ($seen: a hash set of their + * valueKey()s): distinct() batch by batch, the records never held together + */ + public static function collectUnique(array $records, $field, array &$seen, array &$values){ + foreach($records as $record){ + if(isset($record[$field])){ + $id = self::valueKey($record[$field]); + if(!isset($seen[$id])){ + $seen[$id] = true; + $values[] = $record[$field]; + } + } + } + } + + /** + * A string that is equal for two values exactly when they are identical (===): typed, -0.0 is 0.0, + * arrays by their typed keys and values in order (lengths make the encoding unambiguous). + * Floats by their bits, so neither precision nor locale settings matter. + */ + public static function valueKey($v){ + if($v === null) return 'n'; + if($v === true) return 't'; + if($v === false) return 'f'; + if(is_int($v)) return 'i' . $v; + if(is_float($v)) return 'd' . bin2hex(pack('E', $v + 0.0)); + if(is_string($v)) return 's' . $v; + if(is_array($v)){ + $key = 'a' . count($v) . ':'; + foreach($v as $k => $item){ + $kk = (is_int($k) ? 'i' : 's') . $k; + $id = self::valueKey($item); + $key .= strlen($kk) . ':' . $kk . strlen($id) . ':' . $id; + } + return $key; + } + return is_object($v) ? 'o' . spl_object_id($v) : 'r' . (int)$v; // identical only to itself + } + + /** + * Total order used by sort(), min() and max() (and the query builder): missing/null < booleans < + * numbers < numeric strings (by their number) < other strings (byte order) < arrays. + * @return int + */ + public static function compareSortValues($a, $b){ + return self::sortKey($a) <=> self::sortKey($b); + } + + /** + * Sort key of a value: [kind, comparable value]. Two keys compare with <=> in the order above + * (kind first, then the value; arrays by their JSON), so sorts compute each key once instead of + * classifying both values in every comparison. + * @return array + */ + public static function sortKey($v){ + if($v === null) return [0, 0]; + if(is_bool($v)) return [1, $v]; + if(is_int($v) || is_float($v)) return [2, $v]; + if(is_string($v)){ + // Numeric strings (is_numeric()) never meet other strings, so <=> never mixes numeric and byte + // comparison (mixed, "9" < "10" < "1a" < "9" would be a cycle). + return is_numeric($v) ? [3, $v + 0] : [4, $v]; + } + return [5, (string)json_encode($v)]; + } + + /** + * sortKey() for the query builder + * @return \Closure + */ + public function sortKeyFunction(){ + return \Closure::fromCallable([self::class, 'sortKey']); + } + /** + * True when mbstring can be used for Unicode case folding (like(), search()) + * @internal used by noneDBQuery; not part of the public API + */ + public static function hasMbstring(): bool { + return function_exists('mb_convert_case'); // @prod + return \noneDB\Internal\Cache\Runtime::get()->mbstringFn !== null ? (bool)(\noneDB\Internal\Cache\Runtime::get()->mbstringFn)() : function_exists('mb_convert_case'); // @test + } + /** + * Compare two values for ordering operators ($gt, between, ...). + * Numbers (and numeric strings) compare numerically, other strings lexically; anything else + * (null, booleans, arrays, number vs. non-numeric string) is not comparable. + * @return int|null + */ + public static function compareValues($a, $b): ?int { + if ($a === null || $b === null || is_bool($a) || is_bool($b) || is_array($a) || is_array($b) || is_object($a) || is_object($b)) { + return null; + } + // NAN compares as greater than everything with <=> ($gt NAN matched every number) + if ((is_float($a) && is_nan($a)) || (is_float($b) && is_nan($b))) { + return null; + } + $numA = self::numberOf($a); + $numB = self::numberOf($b); + if ($numA !== null && $numB !== null) { + return $numA <=> $numB; + } + if (is_string($a) && is_string($b)) { + return strcmp($a, $b) <=> 0; + } + return null; + } + /** + * The number of an int, float or numeric string (is_numeric(): leading and trailing whitespace + * allowed), else null. The sort order and range indexes use the same rule. + * @return int|float|null + */ + public static function numberOf($value) { + if (is_int($value) || is_float($value)) { + return $value; + } + if (is_string($value)) { + return is_numeric($value) ? $value + 0 : null; + } + return null; + } + /** + * Run preg_match, turning compile errors and backtrack-limit failures into exceptions + */ + public static function regexMatch(string $regex, string $subject): bool { + $result = @preg_match($regex, $subject); + if ($result === false) { + $error = preg_last_error(); + if ($error === PREG_BAD_UTF8_ERROR || $error === PREG_BAD_UTF8_OFFSET_ERROR) { + return false; // subject isn't UTF-8: can't match a UTF-8 pattern + } + if ($error === PREG_NO_ERROR) { + throw new \InvalidArgumentException('noneDB: invalid regular expression'); + } + throw new \RuntimeException('noneDB: regular expression failed (' . preg_last_error_msg() . ')'); + } + return $result === 1; + } + /** + * Case folding for matching (Unicode full folding: "ß" and "ẞ" are "ss", "ſ" is "s", "ς" is "σ"). + * "İ" folds to "i" + combining dot (U+0307); the dot is dropped so "İzmir", "IZMIR" and "izmir" all + * match each other. Language independent: "I" and "ı" don't match. + */ + public static function foldCase(string $s): string { + return str_replace("\u{0307}", '', mb_convert_case($s, MB_CASE_FOLD, 'UTF-8')); + } + /** + * Compile a like() pattern: a leading "^" anchors the start, a trailing "$" the end, everything else + * is literal and case-insensitive. + * @return array + */ + public static function compileLike(string $pattern): array { + $start = $pattern !== '' && $pattern[0] === '^'; + $end = strlen($pattern) > ($start ? 1 : 0) && substr($pattern, -1) === '$'; + $body = (string)substr($pattern, $start ? 1 : 0, strlen($pattern) - ($start ? 1 : 0) - ($end ? 1 : 0)); + $utf8 = preg_match('//u', $body) === 1; + return [ + 'start' => $start, 'end' => $end, + // Folded body for UTF-8 subjects (null: no mbstring, or the pattern isn't UTF-8) + 'folded' => $utf8 && self::hasMbstring() ? self::foldCase($body) : null, + // Otherwise a regex (PCRE /iu: simple case folding) + // ("D": the end anchor is the end of the value, not also before a final newline) + 'regex' => '~' . ($start ? '^' : '') . preg_quote($body, '~') . ($end ? '$' : '') . '~D' . ($utf8 ? 'iu' : 'i'), + ]; + } + /** + * Match a compiled like() pattern + */ + public static function likeMatch(array $like, string $subject): bool { + if ($like['folded'] === null || preg_match('//u', $subject) !== 1) { + return self::regexMatch($like['regex'], $subject); + } + $t = self::foldCase($subject); + $b = $like['folded']; + if ($like['start'] && $like['end']) { + return $t === $b; + } + if ($like['start']) { + return strncmp($t, $b, strlen($b)) === 0; + } + if ($like['end']) { + return $b === '' || (strlen($t) >= strlen($b) && substr($t, -strlen($b)) === $b); + } + return $b === '' || strpos($t, $b) !== false; + } +} diff --git a/src/Internal/Recovery/Recovery.php b/src/Internal/Recovery/Recovery.php new file mode 100644 index 0000000..e569cdd --- /dev/null +++ b/src/Internal/Recovery/Recovery.php @@ -0,0 +1,238 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->recoveryLog, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->durable = &$this->k->durable; + $this->faultHook = &$this->rt->faultHook; // @test + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recoveryLog = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + // @test-begin + /** + * Fault point of the test build: calls the hook a test installed (it may throw or exit) + */ + public function fault($point, array $context = []){ + if($this->faultHook !== null){ + ($this->faultHook)($point, $context); + } + } + // @test-end + + /** + * Recovery of a database a writer left half-updated (journal j/J; caller holds the exclusive lock): + * key indexes are checked against the data (field and spatial indexes follow by themselves), temp + * files of interrupted atomic writes are removed and the shard counters recounted from the data + */ + public function recoverPendingIndexes($dbname, $path){ + $pending = $this->lockRegistry[$path]['flags']['j'] ?? []; + $known = $this->segments->listSegments($dbname); + $segments = isset($pending['*']) ? $known : array_map(function($segment){ + return $segment === 'main' ? null : (int)substr($segment, 1); + }, array_keys($pending)); + $segments = array_values(array_filter($segments, function($shardId) use ($known){ + return in_array($shardId, $known, true); + })); + foreach($segments as $shardId){ + $this->checkpoints->readJsonlIndex($dbname, $shardId); + $this->cache->evictBetweenSegments(); + } + $home = $this->layout->dbHome($dbname); + $gen = $this->layout->genDir($dbname); + foreach([$home, $home . 'recovery/', $gen, $gen . 'segments/', $gen . 'indexes/', $gen . 'spatial/', $gen . 'buffer/'] as $dir){ + $this->removeStaleTemp($dir); + } + if($this->segments->isSharded($dbname)){ + // A write may have died between its data and its counters + $this->shardMeta->recountMeta($dbname); + } + unset($this->lockRegistry[$path]['flags']['j']); + $this->locks->saveLockState($path); + } + + /** + * Remove bytes [$from, $to) at the end of a data or buffer file, keeping them in a quarantine file + * first (exclusive lock). Fails closed: if the bytes can't be stored and verified, nothing is cut + * and the write that needed the cut is refused. + * @throws \RuntimeException + */ + public function quarantineAndCut($dbname, $shardId, $path, $from, $to, $action = 'cut-tail'){ + $quarantine = $this->quarantineBytes($dbname, $shardId, $path, $from, $to); + if(!$this->io->fsTruncate($path, $from)){ + throw new \RuntimeException("noneDB: could not cut the torn end of a file of database '{$dbname}'"); + } + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => $action, 'id' => $quarantine['id'], 'file' => basename($path), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']], $quarantine['id']); + return $quarantine; + } + + /** + * Copy bytes [$from, $to) of a file into recovery/q.{segment}.{offset}.{length}.{sha256}.bin + * (segment: main, s{N}, or buffer for a buffer file: $shardId 'b') + * and verify the copy. An existing file with the same name is reused only if its bytes match. + * @return array ['id' => string, 'file' => string, 'durable' => true|false|'unknown'] + * @throws \RuntimeException + */ + public function quarantineBytes($dbname, $shardId, $path, $from, $to){ + $length = $to - $from; + $seg = $shardId === 'b' ? 'buffer' : \noneDB\Internal\Storage\Layout::segmentName($shardId); + $src = @fopen($path, 'rb'); + if($src === false){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + $dir = $this->layout->dbHome($dbname) . 'recovery/'; + if(!is_dir($dir)){ + $this->io->fsMkdir($dir); + } + $tmp = $dir . 'q.tmp' . bin2hex(random_bytes(4)); + $this->locks->noteWrite($tmp); + $out = null; + try { + $this->fault('quarantine:write', ['db' => $dbname, 'path' => $path]); // @test + $out = @fopen($tmp, 'xb'); + if($out === false){ + throw new \RuntimeException("noneDB: could not create a quarantine file for database '{$dbname}'"); + } + $this->io->setPermissions($tmp); + $hash = hash_init('sha256'); + fseek($src, $from); + $left = $length; + while($left > 0){ + $chunk = (string)fread($src, min(65536, $left)); + if($chunk === ''){ + throw new \RuntimeException("noneDB: could not read bytes to quarantine in database '{$dbname}'"); + } + hash_update($hash, $chunk); + if(fwrite($out, $chunk) !== strlen($chunk)){ + throw new \RuntimeException("noneDB: could not write a quarantine file for database '{$dbname}'"); + } + $left -= strlen($chunk); + } + $sha = hash_final($hash); + // Synced in both modes (DURABILITY.md §3): required in durable mode, else reported in the log + $fileSynced = fflush($out) ? $this->sync->syncHandle($out, $tmp) : false; + fclose($out); + $out = null; + if($this->durable){ + $this->sync->noteRequired('file', $fileSynced === true); + if($fileSynced !== true){ + throw $this->sync->failure('file', $tmp, false); + } + } + + $id = "{$seg}.{$from}.{$length}.{$sha}"; + $name = "q.{$id}.bin"; + $target = $dir . $name; + clearstatcache(true, $target); + if(file_exists($target)){ + @unlink($tmp); // the same event again (e.g. the cut didn't happen last time): keep the first copy + $fileSynced = null; + } elseif(!@rename($tmp, $target)){ + throw new \RuntimeException("noneDB: could not publish a quarantine file for database '{$dbname}'"); + } + $dirSynced = $this->sync->syncDirectory(rtrim($dir, '/')); + if($this->durable){ + // The copy must be durable before the source bytes are cut + $this->sync->noteRequired('dir', $dirSynced === true); + if($dirSynced !== true){ + throw $this->sync->failure('dir', $dir, false); + } + } + if(@filesize($target) !== $length || hash_file('sha256', $target) !== $sha){ + throw new \RuntimeException("noneDB: quarantine file {$name} doesn't match the bytes it should hold"); + } + if($fileSynced === false || $dirSynced === false){ + $durable = false; + } elseif($dirSynced === null){ + $durable = 'unknown'; + } else { + $durable = true; + } + return ['id' => $id, 'file' => $name, 'durable' => $durable]; + } finally { + fclose($src); + if($out !== null && $out !== false){ + fclose($out); + } + if(is_file($tmp)){ + @unlink($tmp); + } + } + } + + /** + * Delete temp files ("*.tmp" + 8 hex digits) starting with $prefix: what a writer that died left + * behind. Exclusive lock only: no other writer can be in the middle of an atomic write. + */ + public function removeStaleTemp($prefix){ + if($this->noWrites > 0){ + return; + } + foreach($this->segments->listFilesWithPrefix($prefix, '/\.tmp[0-9a-f]{8}$/') as $file){ + @unlink($file); + } + } +} diff --git a/src/Internal/Recovery/RecoveryLog.php b/src/Internal/Recovery/RecoveryLog.php new file mode 100644 index 0000000..0819d76 --- /dev/null +++ b/src/Internal/Recovery/RecoveryLog.php @@ -0,0 +1,144 @@ +k = $k; + $this->rt = $k->rt; + unset($this->io, $this->layout, $this->locks); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->pendingRecoveryEvents = &$this->rt->pendingRecoveryEvents; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->k = null; + $this->rt = null; + } + + + private function getRecoveryLogPath($dbname){ + return $this->layout->dbHome($dbname) . 'recovery/log.jsonl'; + } + + /** + * Append an entry to the database's recovery log (caller holds the exclusive lock). Best effort: a + * failure to log never fails the operation. Past 1 MiB the log moves to log.1.jsonl (replacing it). + */ + public function logRecoveryEvent($dbname, array $event, $dedupId = null){ + try { + $path = $this->getRecoveryLogPath($dbname); + if($dedupId !== null){ + $id = '"id":' . json_encode($dedupId); + foreach([substr($path, 0, -6) . '.1.jsonl', $path] as $file){ + $content = @file_get_contents($file); + // the id is followed by the next member or ends the entry + if($content !== false && (strpos($content, $id . ',') !== false || strpos($content, $id . '}') !== false)){ + return; // already logged + } + } + } + clearstatcache(true, $path); + if(!is_dir(dirname($path))){ + $this->io->fsMkdir(dirname($path)); + } + if(@filesize($path) > 1048576){ + $this->io->fsRename($path, substr($path, 0, -6) . '.1.jsonl'); + } + $line = $this->io->encodeJson(['t' => time()] + $event); + if($line !== false && $this->io->appendToFile($path, $line . "\n") !== false){ + $this->metrics['recoveryEvents']++; + $this->metrics['recoveryActions'][$event['action']] = ($this->metrics['recoveryActions'][$event['action']] ?? 0) + 1; + } + } catch(\Throwable $ignored){ + } + } + + /** + * A recovery decision: logged at once under the exclusive lock, else kept in memory and logged by the + * next exclusive session of this process (readers never write); getRecoveryLog() shows both + */ + public function reportRecoveryEvent($dbname, array $event){ + $lock = $this->locks->getLockPath($dbname); + if($this->noWrites === 0 && isset($this->lockRegistry[$lock]) && $this->lockRegistry[$lock]['mode'] === LOCK_EX){ + $this->logRecoveryEvent($dbname, $event, $event['id']); + } else { + $this->pendingRecoveryEvents[$lock][$event['id']] = ['t' => time()] + $event; + } + } + + public function getRecoveryLogEntries($dbname){ + $path = $this->getRecoveryLogPath($dbname); + $entries = []; + foreach([substr($path, 0, -6) . '.1.jsonl', $path] as $file){ + $fp = @fopen($file, 'rb'); + if($fp === false){ + continue; + } + while(($line = fgets($fp)) !== false){ + $entry = substr($line, -1) === "\n" ? json_decode($line, true) : null; + if(is_array($entry)){ + $entries[] = $entry; + } + } + fclose($fp); + } + return $entries; + } + + public function getRecoveryLog($dbname, $limit = 100){ + $dbname = $this->layout->normalizeDbName($dbname); + $read = function() use ($dbname, $limit){ + // What readers of this process found and no write session logged yet: "persisted" => false + $pending = []; + foreach($this->pendingRecoveryEvents[$this->locks->getLockPath($dbname)] ?? [] as $event){ + $pending[] = $event + ['persisted' => false]; + } + $limit = max(0, (int)$limit); + return $limit === 0 ? [] : array_slice(array_merge($this->getRecoveryLogEntries($dbname), $pending), -$limit); + }; + return $this->locks->withLock($dbname, false, $read); + } +} diff --git a/src/Internal/Recovery/RollbackScope.php b/src/Internal/Recovery/RollbackScope.php new file mode 100644 index 0000000..687dd8e --- /dev/null +++ b/src/Internal/Recovery/RollbackScope.php @@ -0,0 +1,286 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->locks, $this->recovery, $this->recoveryLog, $this->segments, $this->shardMeta, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->durable = &$this->k->durable; + $this->faultHook = &$this->rt->faultHook; // @test + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + $this->pendingRecoveryEvents = &$this->rt->pendingRecoveryEvents; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Run the outermost exclusive call of a database inside a rollback scope (see above), then the jobs + * deferred until its commit, each in a scope of its own. The call is one write operation: how it ended + * goes to the durability report (docs/DURABILITY.md §2): committed when $fn returned (every required + * sync succeeded, or it would have thrown), unknown for OUTCOME_UNKNOWN, rolled back otherwise. + * @param string $op Name of the operation; null for a deferred job (not an operation of its own) + */ + public function runWriteScope($dbname, $path, callable $fn, $op = 'write'){ + if($op !== null){ + if(!empty($this->pendingRecoveryEvents[$path]) && $this->noWrites === 0){ + // What readers of this process found (they don't write): logged once, deduplicated by id + $events = $this->pendingRecoveryEvents[$path]; + unset($this->pendingRecoveryEvents[$path]); + foreach($events as $id => $event){ + $this->recoveryLog->logRecoveryEvent($dbname, $event, $id); + } + } + $this->lockRegistry[$path]['wrote'] = false; + if($this->durable){ + $this->lockRegistry[$path]['syncs'] = []; // the syncs durable mode requires (Sync::noteRequired) + } + } + $this->lockRegistry[$path]['scope'] = ['marks' => [], 'deferred' => [], 'independent' => false]; + try { + $result = $fn(); + } catch(\Throwable $e){ + try { + $this->rollbackScope($dbname, $path, $e); // rethrows + } catch(\Throwable $thrown){ + if($op !== null){ + $this->finishOperation($dbname, $path, $op, $thrown); + } + throw $thrown; + } + } + $deferred = $this->lockRegistry[$path]['scope']['deferred']; + $this->lockRegistry[$path]['scope'] = null; + if($op !== null){ + $this->finishOperation($dbname, $path, $op, null); + } + if($this->faultHook !== null){ $this->recovery->fault('scope:commit', ['db' => $dbname]); } // @test + foreach($deferred as $job => $work){ + try { + $this->runWriteScope($dbname, $path, function() use ($dbname, $job, $work){ + $this->recovery->fault('job:' . $job, ['db' => $dbname]); // @test + return $this->runIndependent($dbname, $job, $work); + }, null); + } catch(\Throwable $e){ + // The operation itself is committed; the job runs again on a later operation + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'deferred-failed', 'job' => $job, 'error' => $e->getMessage()]); + break; + } + $this->cache->evictBetweenSegments(); // a job works on one segment (compaction of a large write's segments) + } + return $result; + } + + /** + * The outcome of a write operation for getDurability(); an operation that wrote nothing has none + */ + private function finishOperation($dbname, $path, $op, ?\Throwable $e){ + $syncs = []; + if(isset($this->lockRegistry[$path]['syncs'])){ + $syncs = $this->lockRegistry[$path]['syncs']; + unset($this->lockRegistry[$path]['syncs']); + } + if(!$this->lockRegistry[$path]['wrote']){ + return; + } + if($e === null){ + $this->sync->finishOperation($op, $dbname, $syncs, 'committed'); + } else { + $this->sync->finishOperation($op, $dbname, $syncs, $e->getCode() === \noneDB::OUTCOME_UNKNOWN ? 'unknown' : 'rolled_back', $e->getMessage()); + } + } + + /** + * Undo the appends of a failed scope and rethrow; OUTCOME_UNKNOWN when a file can't be cut back + * (in durable mode: or its cut can't be synced), with the journal "J" so the next session recovers + */ + private function rollbackScope($dbname, $path, \Throwable $e){ + $marks = $this->lockRegistry[$path]['scope']['marks']; + $this->lockRegistry[$path]['scope'] = null; + $this->lockRegistry[$path]['rolledBack'] = true; + $failed = []; + foreach($marks as $file => $mark){ + try { + $this->recovery->fault('rollback:truncate', ['db' => $dbname, 'file' => $file]); // @test + if(!$this->truncateBack($file, $mark)){ + $failed[] = $file; + } + } catch(\Throwable $t){ + $failed[] = $file; + } + } + $this->cache->invalidateDbCaches($dbname); + if(!empty($failed) || $e->getCode() === \noneDB::OUTCOME_UNKNOWN){ + // What is on disk is uncertain: the next session recovers the whole database ("J") + $this->lockRegistry[$path]['flags']['j']['*'] = true; + try { + $this->locks->saveLockState($path); + } catch(\Throwable $ignored){ + } + if(!empty($failed)){ + throw new \RuntimeException("noneDB: a write to database '{$dbname}' failed and could not be undone: the outcome is unknown, the database is recovered on its next access", \noneDB::OUTCOME_UNKNOWN, $e); + } + } + if(!empty($marks) && $this->segments->isSharded($dbname)){ + try { + $this->shardMeta->recountMeta($dbname); // keys reserved by the failed write stay unused + } catch(\Throwable $ignored){ + // the journal entry stays, recovery recounts + } + } + throw $e; + } + + /** + * Cut a file back to the size recorded in a mark, if it is still the same file + */ + private function truncateBack($file, array $mark){ + list($size, $dev, $ino) = $mark; + $this->locks->noteWrite($file); + $fp = @fopen($file, 'r+b'); + if($fp === false){ + return false; + } + try { + $stat = fstat($fp); + if($stat === false || ($ino && ($stat['ino'] !== $ino || $stat['dev'] !== $dev))){ + return false; + } + return $stat['size'] <= $size || (ftruncate($fp, $size) && $this->sync->syncUndo($fp, $file)); + } finally { + fclose($fp); + clearstatcache(true, $file); + } + } + + /** + * Record the size a file had before the scope's first append to it + */ + public function markScopeFile($dbname, $file, $size, $fp){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->lockRegistry[$lock]['scope']) && !isset($this->lockRegistry[$lock]['scope']['marks'][$file])){ + $stat = fstat($fp); + $this->lockRegistry[$lock]['scope']['marks'][$file] = [$size, $stat ? $stat['dev'] : 0, $stat ? $stat['ino'] : 0]; + } + } + + /** + * Run work that must not be undone by its caller's failure (buffer flush, the switch to shards): in place while the + * scope has no appends, else deferred until the scope committed (once per $job). Returns null if deferred. + */ + public function runIndependent($dbname, $job, callable $work){ + $path = $this->locks->getLockPath($dbname); + if(!isset($this->lockRegistry[$path]['scope'])){ + return $work(); + } + if(!empty($this->lockRegistry[$path]['scope']['marks'])){ + if(!isset($this->lockRegistry[$path]['scope']['deferred'][$job])){ + $this->lockRegistry[$path]['scope']['deferred'][$job] = $work; + $this->recovery->fault('job:deferred', ['db' => $dbname, 'job' => $job]); // @test + } + return null; + } + if($this->faultHook !== null){ $this->recovery->fault('independent:run', ['db' => $dbname, 'job' => $job]); } // @test + $was = $this->lockRegistry[$path]['scope']['independent']; + $this->lockRegistry[$path]['scope']['independent'] = true; + try { + $result = $work(); + $this->commitIndependentStep($dbname); + return $result; + } finally { + if(isset($this->lockRegistry[$path]['scope'])){ + $this->lockRegistry[$path]['scope']['independent'] = $was; + } + } + } + + /** + * Work that replaces or removes data files (dropDB, compact, convertToSharded) can't be undone by a rollback: + * inside a write of the same database that hasn't committed (a nested call from the caller's own code, which + * a value's jsonSerialize() can make before the first append) it is refused before it touches any file + * @throws \LogicException + */ + public function refuseInUncommittedWrite($dbname, $operation){ + $entry = $this->lockRegistry[$this->locks->getLockPath($dbname)] ?? null; + $scope = $entry['scope'] ?? null; + if($scope !== null && (!empty($scope['marks']) || $entry['depth'] > 1) && empty($scope['independent'])){ + throw new \LogicException("noneDB: {$operation}() can't run inside a write to database '{$dbname}' that hasn't committed yet (it replaces or deletes data files, which the write's rollback couldn't undo); call it after that write"); + } + } + + /** + * Before a file is replaced, renamed or removed: never one that holds appends a rollback may have to cut + * back (whatever path asks for it) + * @throws \LogicException + */ + public function refuseUncommittedFile($file, $operation){ + foreach($this->lockRegistry as $entry){ + if(isset($entry['scope']['marks'][$file]) && empty($entry['scope']['independent'])){ + throw new \LogicException("noneDB: {$operation} of " . basename($file) . " refused: it holds appends of database '{$entry['dbname']}' that aren't committed yet"); + } + } + } + + /** + * Inside independent work: its appends so far are final (the scope had none before it started) + */ + public function commitIndependentStep($dbname){ + $path = $this->locks->getLockPath($dbname); + if(!empty($this->lockRegistry[$path]['scope']['independent'])){ + $this->lockRegistry[$path]['scope']['marks'] = []; + } + } +} diff --git a/src/Internal/Sharding/LayoutSwitch.php b/src/Internal/Sharding/LayoutSwitch.php new file mode 100644 index 0000000..b304de1 --- /dev/null +++ b/src/Internal/Sharding/LayoutSwitch.php @@ -0,0 +1,187 @@ +k = $k; + $this->rt = $k->rt; + unset($this->buffer, $this->cache, $this->checkpoints, $this->fieldIndex, $this->io, $this->layout, $this->locks, $this->records, $this->recovery, $this->scope, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->buffer = null; + $this->cache = null; + $this->checkpoints = null; + $this->fieldIndex = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recovery = null; + $this->scope = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Convert a single-layout database to the sharded layout (keys preserved, indexes rebuilt per shard): + * a generation switch (STORAGE_FORMAT.md §5.2). The new generation is built and verified completely + * before CURRENT names it; an interruption leaves the old generation or the new one, never a mix. + * @param string $dbname + * @return bool + */ + public function convertToSharded($dbname){ + return $this->locks->withLock($dbname, true, function() use ($dbname){ + if($this->segments->isSharded($dbname)){ + return true; + } + $singlePath = $this->segments->getDataPath($dbname); + if(!$this->cache->cachedFileExists($singlePath)){ + return false; + } + $this->scope->refuseUncommittedFile($singlePath, 'The switch to shards'); + $this->buffer->flushPendingBuffers($dbname); + + $index = $this->checkpoints->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + $live = \noneDB\Internal\Storage\KeyIndex::count($index); + $maxKey = \noneDB\Internal\Storage\KeyIndex::maxKey($index); + $size = $this->segments->shardSizeOf($dbname); + $nextKey = max((int)$index['n'], $maxKey + 1); + $numShards = max(1, intdiv(max($nextKey, 1) - 1, $size) + 1); + $manifest = $this->fieldIndex->getIndexManifest($dbname); + $format = $this->layout->readFormatRecord($dbname); + unset($index); + + $this->layout->switchGeneration($dbname, function($dir, $generation) use ($dbname, $numShards, $size, $nextKey, $maxKey, $live, $manifest, $format){ + $this->layout->writeGenerationMetadata($dbname, $generation, $this->layout->identityOf($format), 'sharded'); + $meta = \noneDB\Internal\Sharding\ShardMeta::newShardMeta($size, $nextKey); + $meta['totalRecords'] = $live; + $written = 0; + // One shard at a time: its records read from the single segment, written with its checkpoint + for($shardId = 0; $shardId < $numShards; $shardId++){ + $base = $shardId * $size; + $limit = min($nextKey, $base + $size); + $count = 0; + $shardIndex = $this->checkpoints->newJsonlIndex(); + $shardIndex['n'] = $limit - $base; + $buffer = $this->io->withExactFloats(function() use ($dbname, $base, $limit, &$shardIndex, &$count){ + $buffer = ''; + $offset = 0; + // The shard's records a batch at a time (a large shard size holds one batch, not the shard) + foreach($base < $limit ? $this->records->eachSegmentBatch($dbname, null, range($base, $limit - 1)) : [] as $batch){ + foreach($batch as $key => $record){ + $json = $this->io->encodeRecord($record); + \noneDB\Internal\Storage\KeyIndex::set($shardIndex, $key, $offset, strlen($json)); + $buffer .= $json . "\n"; + $offset += strlen($json) + 1; + $count++; + } + } + return $buffer; + }); + if($shardId === $numShards - 1 && $nextKey - 1 > $maxKey){ + // Keep the highest key ever used, as compaction does: a key index rebuilt from these + // files must never hand it out again + $buffer .= '{"__nonedb_deleted":' . ($nextKey - 1) . "}\n"; + $shardIndex['t'] = 1; + } + $shardPath = $dir . 'segments/s' . $shardId . '.jsonl'; + if(!$this->io->writeFileAtomic($shardPath, $buffer, 'replace')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards failed while writing shard {$shardId}"); + } + $shardIndex['s'] = strlen($buffer); + $shardIndex['e'] = bin2hex(random_bytes(4)); + $buffer = null; + $this->recovery->fault('switch:shard-written', ['db' => $dbname, 'shard' => $shardId]); // @test + // Verified from the file: every record of the shard is a line of it + $written += $this->countRecordLines($shardPath, $base, $base + $size); + $this->checkpoints->writeCheckpointFile($shardPath, $shardId, $size, $shardIndex); // derived: best effort + $shardIndex = null; + $entry = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shardId); + $entry['count'] = $count; + $meta['shards'][] = $entry; + } + if($written !== $live){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not be verified"); + } + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($dir . 'shards.json', $json . "\n", 'durable')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not write its shard list"); + } + if(!empty($manifest['fields']) || !empty($manifest['spatial'])){ + $list = ['kind' => 'indexes', 'v' => 1, 'fields' => $manifest['fields'], 'spatial' => $manifest['spatial'], + 'spatialOptions' => array_intersect_key($manifest['spatialOptions'], array_flip($manifest['spatial']))]; + if(!$this->io->atomicWrite($dir . 'indexes.json', $list, false, 'durable')){ + throw new \RuntimeException("noneDB: the switch of database '{$dbname}' to shards could not write its index list"); + } + } + }); + // The new generation is CURRENT: its shards' field and spatial indexes + $this->fieldIndex->rebuildIndexesAfterSwitch($dbname); + return true; + }, 'convertToSharded'); + } + + /** + * Complete lines of a data file that are records with keys in [$base, $limit) + */ + private function countRecordLines($path, $base, $limit){ + $count = 0; + $fp = @fopen($path, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $base, $limit)) !== null && $entry[0] === 'r'){ + $count++; + } + } + if($fp !== false){ + fclose($fp); + } + return $count; + } +} diff --git a/src/Internal/Sharding/ShardInfo.php b/src/Internal/Sharding/ShardInfo.php new file mode 100644 index 0000000..3169d77 --- /dev/null +++ b/src/Internal/Sharding/ShardInfo.php @@ -0,0 +1,105 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->layout, $this->locks, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->layout = null; + $this->locks = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // SHARDING PUBLIC METHODS + // ========================================== + + /** + * Get sharding information for a database + * @param string $dbname + * @return array|false + */ + public function getShardInfo($dbname){ + $dbname = $this->layout->normalizeDbName($dbname); + return $this->locks->withReadLock($dbname, function() use ($dbname){ + return $this->getShardInfoUnlocked($dbname); + }); + } + + /** + * getShardInfo() of a database whose lock the caller holds (backup() and restore() read the counters) + */ + public function getShardInfoUnlocked($dbname){ + if(!$this->segments->dbExists($dbname)){ + return false; + } + if(!$this->segments->isSharded($dbname)){ + $index = $this->checkpoints->readJsonlIndex($dbname, null); + if($index === null){ + return false; + } + return array( + "sharded" => false, + "shards" => 0, + "totalRecords" => \noneDB\Internal\Storage\KeyIndex::count($index), + "shardSize" => $this->segments->shardSizeOf($dbname), // fixed in FORMAT at creation (§4.1) + "nextKey" => (int)$index['n'] + ); + } + + $meta = $this->shardMeta->getCachedMeta($dbname); + if($meta === null){ + return false; + } + return array( + "sharded" => true, + "shards" => count($meta['shards']), + "totalRecords" => (int)$meta['totalRecords'], + "deletedCount" => (int)($meta['deletedCount'] ?? 0), + "shardSize" => (int)$meta['shardSize'], + "nextKey" => (int)$meta['nextKey'] + ); + } +} diff --git a/src/Internal/Sharding/ShardMeta.php b/src/Internal/Sharding/ShardMeta.php new file mode 100644 index 0000000..69c1561 --- /dev/null +++ b/src/Internal/Sharding/ShardMeta.php @@ -0,0 +1,246 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->recoveryLog, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->metaCache = &$this->k->metaCache; + $this->shardedCache = &$this->k->shardedCache; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->noWrites = &$this->rt->noWrites; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recoveryLog = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Shard list and counters of a sharded database + */ + public function getMetaPath($dbname){ + return $this->layout->genPath($dbname, 'shards.json'); + } + + /** + * Read shard metadata + * @param string $dbname + * @return array|null + */ + public function readMeta($dbname){ + $meta = $this->io->atomicRead($this->getMetaPath($dbname), null); + return is_array($meta) && ($meta['kind'] ?? null) === 'shards' && ($meta['v'] ?? null) === 1 + && is_int($meta['shardSize'] ?? null) && is_int($meta['nextKey'] ?? null) + && isset($meta['shards']) && is_array($meta['shards']) ? $meta : null; + } + + /** + * Get cached meta data. The cache is dropped whenever another process writes the + * database (generation check in withLock), so no TTL is needed. + * @param string $dbname + * @param bool $forceRefresh Force refresh from disk + * @return array|null + */ + public function getCachedMeta($dbname, $forceRefresh = false){ + $metaPath = $this->getMetaPath($dbname); + + $this->cacheTouch[$metaPath] = ++$this->cacheTick; + if(!$forceRefresh && isset($this->metaCache[$metaPath])){ + return $this->metaCache[$metaPath]; + } + + $meta = $this->readMeta($dbname); + if($meta === null){ + clearstatcache(true, $metaPath); + if(is_file($metaPath)){ + // There but unreadable (e.g. zero-length after a power loss): rebuilt from the shards. + // checkIntegrity() writes nothing: it works with a meta derived in memory, not kept + if($this->noWrites > 0){ + try { + return $this->deriveShardMeta($dbname); + } catch(\RuntimeException $e){ + return null; + } + } + $meta = $this->recoverMeta($dbname); + } + } + if($meta !== null){ + $this->metaCache[$metaPath] = $meta; + $this->cacheGrew = true; + return $meta; + } + unset($this->metaCache[$metaPath]); + return null; + } + + /** + * Rebuild an unreadable shards.json from the shard files (exclusive lock): the shard size is FORMAT's, + * nextKey follows the highest key ever used. A missing shards.json is never "recovered": shard files + * without one are leftovers (see setOrphanShardsAside). + * @return array|null + */ + private function recoverMeta($dbname){ + return $this->locks->withLock($dbname, true, function() use ($dbname){ + $metaPath = $this->getMetaPath($dbname); + unset($this->metaCache[$metaPath]); + $meta = $this->readMeta($dbname); + clearstatcache(true, $metaPath); + if($meta !== null || !is_file($metaPath)){ + return $meta; + } + $meta = $this->deriveShardMeta($dbname); + $this->writeMeta($dbname, $meta, true); + $this->recountMeta($dbname); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'meta-rebuilt', 'shards' => count($meta['shards']), 'shardSize' => $meta['shardSize']]); + return $this->getCachedMeta($dbname); + }, 'recovery'); + } + + /** + * A shard list derived from the shard files alone (reads only; counters 0, recountMeta() fills them). + * The shard size is FORMAT's; the next key is above every key a shard file holds (tombstones included). + * @return array + */ + public function deriveShardMeta($dbname){ + $size = $this->segments->shardSizeOf($dbname); + $prefix = $this->layout->genDir($dbname) . 'segments/s'; + $ids = []; + $nextKey = 0; + foreach($this->segments->listFilesWithPrefix($prefix, '/^\d+\.jsonl$/') as $file){ + $shardId = (int)substr($file, strlen($prefix)); + $ids[] = $shardId; + $fp = @fopen($file, 'rb'); + while($fp !== false && ($line = fgets($fp)) !== false){ + if(substr($line, -1) === "\n" && ($entry = $this->checkpoints->classifyLine(substr($line, 0, -1), $shardId * $size, ($shardId + 1) * $size)) !== null){ + $nextKey = max($nextKey, $entry[1] + 1); + } + } + if($fp !== false){ + fclose($fp); + } + } + sort($ids); + $meta = self::newShardMeta($size, $nextKey); + foreach($ids as $shardId){ + $meta['shards'][] = self::newShardEntry($shardId); + } + return $meta; + } + + /** + * An empty shards.json record (docs/STORAGE_FORMAT.md §5.1) + */ + public static function newShardMeta($size, $nextKey){ + return ['kind' => 'shards', 'v' => 1, 'shardSize' => (int)$size, 'nextKey' => (int)$nextKey, + 'totalRecords' => 0, 'deletedCount' => 0, 'shards' => []]; + } + + public static function newShardEntry($shardId){ + return ['id' => (int)$shardId, 'count' => 0, 'deleted' => 0]; + } + + /** + * Write shard metadata (caller holds the exclusive lock) + * @param string $dbname + * @param array $meta + * @return bool + * @throws \RuntimeException when shards.json can't be written + */ + public function writeMeta($dbname, $meta, $sync = false){ + $path = $this->getMetaPath($dbname); + $json = $this->io->encodeJson($meta); + if($json === false || !$this->io->writeFileAtomic($path, $json . "\n", $sync ? 'replace' : 'durable')){ + throw new \RuntimeException("noneDB: failed to write metadata of database '{$dbname}'"); + } + $this->metaCache[$path] = $meta; + $this->shardedCache[$path] = true; + $this->cache->markFileExists($path); + return true; + } + + /** + * Recompute the per-shard and total counters of shards.json from the shard indexes + */ + public function recountMeta($dbname){ + $meta = $this->getCachedMeta($dbname, true); + if($meta === null){ + return; + } + $size = (int)$meta['shardSize']; + $total = 0; + $nextKey = 0; + foreach($meta['shards'] as $i => $shard){ + $index = $this->checkpoints->readJsonlIndex($dbname, (int)$shard['id']); + $count = $index === null ? 0 : \noneDB\Internal\Storage\KeyIndex::count($index); + $meta['shards'][$i]['count'] = $count; + $total += $count; + if($index !== null){ + // n of a shard counts every key it ever had (deleted ones and the kept top tombstone too) + $nextKey = max($nextKey, \noneDB\Internal\Storage\KeyIndex::maxKey($index) + 1); + if((int)$index['n'] > 0){ + $nextKey = max($nextKey, (int)$shard['id'] * $size + (int)$index['n']); + } + } + unset($index); + $this->cache->evictBetweenSegments(); + } + $meta['totalRecords'] = $total; + $meta['nextKey'] = max((int)($meta['nextKey'] ?? 0), $nextKey); + $this->writeMeta($dbname, $meta); + } +} diff --git a/src/Internal/Sharding/ShardRouter.php b/src/Internal/Sharding/ShardRouter.php new file mode 100644 index 0000000..2b23fea --- /dev/null +++ b/src/Internal/Sharding/ShardRouter.php @@ -0,0 +1,193 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->layout, $this->locks, $this->records, $this->recovery, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->faultHook = &$this->rt->faultHook; // @test + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->shardFilesUpTo = &$this->k->shardFilesUpTo; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->records = null; + $this->recovery = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Insert into a sharded database. Keys come from meta.nextKey; key K lives in shard intdiv(K, shardSize). + */ + public function insertShardedRecords($dbname, array $items, $keyed = null, $endKey = null){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + $size = (int)$meta['shardSize']; + $nextKey = (int)$meta['nextKey']; + // The meta may lag behind the data (a meta update lost in a power loss): never hand out a key the + // last shard has already used + if(!empty($meta['shards'])){ + $last = max(array_map('intval', array_column($meta['shards'], 'id'))); + $lastIndex = $this->checkpoints->readJsonlIndex($dbname, $last); + if($lastIndex !== null){ + $nextKey = max($nextKey, $last * $size + (int)$lastIndex['n']); + } + } + + $byShard = []; + if($keyed !== null){ + foreach($keyed as $key => $item){ + $byShard[intdiv($key, $size)][$key] = $item; + } + $nextKey = max($nextKey, (int)$endKey); + $items = $keyed; + } else { + foreach($items as $item){ + $byShard[intdiv($nextKey, $size)][$nextKey] = $item; + $nextKey++; + } + } + + // Reserve the keys first: a crash later leaves a gap, never a reused key + $shardPos = []; + foreach($meta['shards'] as $i => $shard){ + $shardPos[(int)$shard['id']] = $i; + } + $this->setOrphanShardsAside($dbname, array_keys(array_diff_key($byShard, $shardPos))); + foreach($byShard as $shardId => $records){ + if(!isset($shardPos[$shardId])){ + $meta['shards'][] = \noneDB\Internal\Sharding\ShardMeta::newShardEntry($shardId); + $shardPos[$shardId] = count($meta['shards']) - 1; + } + $meta['shards'][$shardPos[$shardId]]['count'] += count($records); + } + // Journal first: if we die after reserving, recovery recounts the meta from the shards + foreach(array_keys($byShard) as $shardId){ + $this->locks->markIndexesPending($dbname, $shardId); + } + $meta['nextKey'] = $nextKey; + $meta['totalRecords'] = (int)$meta['totalRecords'] + count($items); + $this->shardMeta->writeMeta($dbname, $meta); + + // A failure below is undone by the rollback scope (appends of every shard cut back, counters + // recounted); the reserved keys stay unused and the new shards stay listed (empty) + foreach($byShard as $shardId => $records){ + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + if($index === null){ + $this->records->createJsonlDatabase($dbname, $shardId); + if($this->faultHook !== null){ $this->recovery->fault('shard:created', ['db' => $dbname, 'shard' => $shardId]); } // @test + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + } + $this->records->appendSegmentRecords($dbname, $shardId, $index, $records); + $index['n'] = max((int)$index['n'], max(array_keys($records)) - $shardId * $size + 1); + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + } + } + + /** + * Files of a shard that shards.json doesn't list (left by a shards.json update lost in a power loss, or + * put there by hand) must not be taken over by a new shard with the same number: their records were + * deleted, or belong to keys the meta doesn't know any more. They are renamed to *.orphan.{time} + * (reported by getDB()), never deleted. + */ + private function setOrphanShardsAside($dbname, array $shardIds){ + if(empty($shardIds)){ + return; + } + // The directory is listed only for shards up to the highest one that had files at the last listing: + // above it only this process created shard files since (another process's write drops the number, + // invalidateDbCaches). A listing per new shard made inserts into many shards O(shards²) + // Only data and checkpoint files: field and spatial indexes check themselves against the data + // (segment epoch, covered bytes), so one left by an earlier shard of that number is never used + $base = $this->layout->genDir($dbname) . 'segments/s'; + $upTo = max(array_map('intval', $shardIds)); + if(isset($this->shardFilesUpTo[$base]) && min(array_map('intval', $shardIds)) > $this->shardFilesUpTo[$base]){ + $this->shardFilesUpTo[$base] = $upTo; + return; + } + $wanted = array_flip(array_map('intval', $shardIds)); + $files = []; + foreach($this->segments->listFilesWithPrefix($base, '/^\d+\./') as $file){ + $shardId = (int)substr($file, strlen($base)); + $upTo = max($upTo, $shardId); + if(isset($wanted[$shardId])){ + $files[] = $file; + } + } + $stamp = '.orphan.' . date('YmdHis'); + $moved = false; + foreach($files as $file){ + if(preg_match('/\.orphan\.\d+$/', $file)){ + continue; + } + if(!$moved){ + $this->locks->bumpEpoch($dbname); // a shard file of this name goes away: no cache of it is kept + $moved = true; + } + if(!$this->io->fsRename($file, $file . $stamp)){ + throw new \RuntimeException("noneDB: could not move aside the leftover file " . basename($file) . " of database '{$dbname}'"); + } + unset($this->indexCache[$file], $this->fieldIndexCache[$file], $this->spatialIndexCache[$file]); + $this->cache->markFileNotExists($file); + } + $this->shardFilesUpTo[$base] = $upTo; // after bumpEpoch(), which drops it + } +} diff --git a/src/Internal/Storage/Buffer.php b/src/Internal/Storage/Buffer.php new file mode 100644 index 0000000..8c702dd --- /dev/null +++ b/src/Internal/Storage/Buffer.php @@ -0,0 +1,424 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->db, $this->io, $this->layout, $this->locks, $this->recovery, $this->recoveryLog, $this->scope, $this->segments, $this->shardMeta, $this->sync, $this->writer); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->bufferAutoFlushOnShutdown = &$this->k->bufferAutoFlushOnShutdown; + $this->bufferCountLimit = &$this->k->bufferCountLimit; + $this->bufferFlushInterval = &$this->k->bufferFlushInterval; + $this->bufferLastFlush = &$this->k->bufferLastFlush; + $this->bufferSizeLimit = &$this->k->bufferSizeLimit; + $this->shutdownHandlerRegistered = &$this->k->shutdownHandlerRegistered; + $this->bufferCounts = &$this->rt->bufferCounts; + $this->bufferEnds = &$this->rt->bufferEnds; + $this->flushingNow = &$this->rt->flushingNow; + $this->lockRegistry = &$this->rt->lockRegistry; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->db = null; + $this->io = null; + $this->layout = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->sync = null; + $this->writer = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE BUFFER + // ========================================== + // Optional (off by default). Buffered records have no key yet; flushing renames the buffer + // (buffer/pending.jsonl of the active generation) to buffer/flushing.{firstKey}.jsonl first and then + // inserts its records through insertRecords(), so the normal indexes and sharding logic apply. + // Pending buffers are flushed before every read and write, whether or not buffering is enabled. + + /** + * Get buffer file path of a database + * @param string $dbname + * @return string + */ + public function getBufferPath($dbname){ + return $this->layout->genPath($dbname, 'buffer/pending.jsonl'); + } + + /** + * Check if buffer exists and has content + * @param string $bufferPath + * @return bool + */ + public function hasBuffer($bufferPath){ + clearstatcache(true, $bufferPath); + return file_exists($bufferPath) && filesize($bufferPath) > 0; + } + + /** + * Number of records in a buffer file; counted once, then tracked by size + */ + public function getBufferRecordCount($bufferPath){ + clearstatcache(true, $bufferPath); + if(!file_exists($bufferPath)){ + return 0; + } + $size = filesize($bufferPath); + if(isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size){ + return $this->bufferCounts[$bufferPath][1]; + } + $count = count($this->readBufferRecords($bufferPath)); + $this->bufferCounts[$bufferPath] = [$size, $count]; + return $count; + } + + /** + * Read all records from a buffer file (JSONL). An unterminated last line is not a record (its + * append never completed). + * @param string $bufferPath + * @param bool $clean Set to false when the file holds anything that isn't a complete record line + * @return array + */ + public function readBufferRecords($bufferPath, &$clean = true){ + $clean = true; + $records = []; + $fp = @fopen($bufferPath, 'rb'); + if($fp === false){ + return $records; + } + while(($line = fgets($fp)) !== false){ + if(substr($line, -1) !== "\n"){ + $clean = false; + break; + } + $line = trim($line); + if($line === ''){ + continue; + } + $record = json_decode($line, true); + if(is_array($record)){ + $records[] = $record; + } else { + $clean = false; + } + } + fclose($fp); + return $records; + } + + /** + * Before appending to a buffer: a torn last line (crashed append) goes to quarantine and is cut off, + * so the next record isn't glued to it. Skipped when the file has the size this process wrote last. + */ + private function repairBufferTail($dbname, $bufferPath){ + clearstatcache(true, $bufferPath); + $size = @filesize($bufferPath); + if(!$size || ($this->bufferEnds[$bufferPath] ?? null) === $size){ + return; + } + $last = $this->checkpoints->lastNewlineIn($bufferPath, max(0, $size - 65536), $size); + if($last !== $size - 1){ + if($last < 0){ + $last = $this->checkpoints->lastNewlineIn($bufferPath, 0, $size); + } + $this->recovery->quarantineAndCut($dbname, 'b', $bufferPath, $last + 1, $size); + unset($this->bufferCounts[$bufferPath]); + } + } + + /** + * Before a flushed buffer file is deleted: anything in it that wasn't a record is kept in quarantine + */ + private function quarantineBufferRest($dbname, $file){ + clearstatcache(true, $file); + $size = (int)@filesize($file); + if($size > 0){ + $quarantine = $this->recovery->quarantineBytes($dbname, 'b', $file, 0, $size); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'drop-buffer', 'id' => $quarantine['id'], 'file' => basename($file), + 'quarantine' => $quarantine['file'], 'bytesVerified' => true, 'powerLossDurable' => $quarantine['durable']]); + } + } + + /** + * True if the database has buffered records waiting + */ + private function hasPendingBuffers($dbname){ + // Under the lock the state record says it (flag 'b', set before a buffer gets records and + // cleared after they are flushed), so normal operations don't touch the file system for this + $path = $this->locks->getLockPath($dbname); + if(isset($this->lockRegistry[$path]) && $this->lockRegistry[$path]['epoch'] !== null){ + return !empty($this->lockRegistry[$path]['flags']['b']); + } + return $this->hasBuffer($this->getBufferPath($dbname)) || !empty($this->flushingFiles($dbname)); + } + + /** + * Buffer files whose flush was interrupted: buffer/flushing.{firstKey}.jsonl + */ + public function flushingFiles($dbname){ + return $this->segments->listFilesWithPrefix($this->layout->genDir($dbname) . 'buffer/flushing.', '/^\d+\.jsonl$/'); + } + + /** + * Flush buffered records into the database (exclusive lock) + * @return int Number of records flushed + */ + public function flushPendingBuffers($dbname){ + $base = $this->layout->dbHome($dbname); + // Flushing inserts records, which can switch the database to shards, which flushes again: don't re-enter + if(isset($this->flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + return (int)$this->locks->withLock($dbname, true, function() use ($dbname, $base){ + // In place before the caller appended anything, else after the caller's write committed + return $this->scope->runIndependent($dbname, 'flush', function() use ($dbname, $base){ + if(isset($this->flushingNow[$base]) || !$this->hasPendingBuffers($dbname)){ + return 0; + } + $this->flushingNow[$base] = true; + try { + return $this->flushPendingBuffersLocked($dbname); + } finally { + unset($this->flushingNow[$base]); + } + }); + }, 'flush'); + } + + private function flushPendingBuffersLocked($dbname){ + { + $flushed = 0; + $bufferPath = $this->getBufferPath($dbname); + // Interrupted flushes first. A flush renames the buffer to flushing.{firstKey}.jsonl: the + // rename records atomically which keys its records get, so replaying it is idempotent. + foreach($this->flushingFiles($dbname) as $file){ + preg_match('/flushing\.(\d+)\.jsonl$/', $file, $m); + $flushed += $this->flushBufferFile($dbname, $file, (int)$m[1]); + } + if($this->hasBuffer($bufferPath)){ + if(!$this->segments->dbExists($dbname)){ + $this->db->createDBUnlocked($dbname); + } + $startKey = $this->nextKeyOf($dbname); + $flushingPath = dirname($bufferPath) . '/flushing.' . $startKey . '.jsonl'; + if(!$this->sync->renameWithRetry($bufferPath, $flushingPath)){ + throw new \RuntimeException("noneDB: could not flush the write buffer of database '{$dbname}'"); + } + $this->bufferEnds[$bufferPath] = 0; // the buffer is gone: the next append creates it + // The rename must survive a power loss before records get keys (else replayed twice) + $this->sync->requireSync(null, dirname($bufferPath), 'dir'); + $flushed += $this->flushBufferFile($dbname, $flushingPath, $startKey); + } + unset($this->bufferCounts[$bufferPath]); + $this->locks->setBufferFlag($dbname, false); + $this->bufferLastFlush[$dbname] = time(); + return $flushed; + } + } + + /** + * Insert the records of one buffer file, starting at $startKey + * @return int + */ + private function flushBufferFile($dbname, $file, $startKey){ + $records = $this->readBufferRecords($file, $clean); + if(!empty($records)){ + $result = $this->writer->insertRecords($dbname, $records, $startKey); + if(isset($result['error'])){ + throw new \RuntimeException("noneDB: flushing the write buffer of '{$dbname}' failed: " . $result['error']); + } + } + $this->scope->commitIndependentStep($dbname); // the records are stored for good before their buffer goes + if(!$clean){ + $this->quarantineBufferRest($dbname, $file); + } + $this->io->fsUnlink($file); + return count($records); + } + + /** + * The key the next inserted record gets + */ + private function nextKeyOf($dbname){ + if($this->segments->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + return $meta === null ? 0 : (int)$meta['nextKey']; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + return $index === null ? 0 : (int)$index['n']; + } + /** + * Append validated records to the database's write buffer (exclusive lock) + * @return array + */ + public function insertBuffered($dbname, array $validItems){ + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->db->createDBUnlocked($dbname); + } + $this->registerShutdownHandler(); + + $lines = $this->io->withExactFloats(function() use ($validItems){ + $lines = ''; + foreach($validItems as $item){ + $lines .= $this->io->encodeRecord($item) . "\n"; + } + return $lines; + }); + + $bufferPath = $this->getBufferPath($dbname); + // One size for the flush limits, the torn tail and the count. The size this process's last append + // left holds while no other process wrote the database: a write of another process moves the lock + // generation, which drops this cache (CacheManager::invalidateDbCaches), so no stat per insert. + $size = $this->bufferEnds[$bufferPath] ?? null; + if($size === null){ + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + $known = isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size ? $this->bufferCounts[$bufferPath][1] : null; + if($size > 0 && $this->shouldFlushBuffer($bufferPath, $dbname, $size, $known)){ + $this->flushPendingBuffers($dbname); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + if($size > 0 && ($this->bufferEnds[$bufferPath] ?? null) !== $size){ + $this->repairBufferTail($dbname, $bufferPath); + clearstatcache(true, $bufferPath); + $size = (int)@filesize($bufferPath); + } + $count = $size === 0 ? 0 : (isset($this->bufferCounts[$bufferPath]) && $this->bufferCounts[$bufferPath][0] === $size + ? $this->bufferCounts[$bufferPath][1] : $this->getBufferRecordCount($bufferPath)); + $this->locks->setBufferFlag($dbname, true); + if($this->io->appendToFile($bufferPath, $lines, $dbname) === false){ + throw new \RuntimeException("noneDB: failed to write the write buffer of database '{$dbname}'; nothing was written"); + } + $size += strlen($lines); // the append ended there (checked against the file next time) + $this->bufferCounts[$bufferPath] = [$size, $count + count($validItems)]; + $this->bufferEnds[$bufferPath] = $size; + if(!isset($this->bufferLastFlush[$dbname])){ + $this->bufferLastFlush[$dbname] = time(); + } + + if($this->shouldFlushBuffer($bufferPath, $dbname, $size, $count + count($validItems))){ + $this->flushPendingBuffers($dbname); + } + return ["n" => count($validItems)]; + } + + /** + * Check if buffer needs flushing (size, count, or time based) + * @param string $bufferPath + * @param string $flushKey Key for tracking last flush time + * @return bool + */ + private function shouldFlushBuffer($bufferPath, $flushKey, $size = null, $count = null){ + if($size === null){ + clearstatcache(true, $bufferPath); + $size = file_exists($bufferPath) ? (int)filesize($bufferPath) : 0; + } + if($size <= 0){ + return false; + } + if($size >= $this->bufferSizeLimit){ + return true; + } + if(($count ?? $this->getBufferRecordCount($bufferPath)) >= $this->bufferCountLimit){ + return true; + } + if($this->bufferFlushInterval > 0){ + $lastFlush = $this->bufferLastFlush[$flushKey] ?? time(); + if((time() - $lastFlush) >= $this->bufferFlushInterval){ + return true; + } + } + return false; + } + + /** + * Register shutdown handler for flushing all buffers + */ + private function registerShutdownHandler(){ + if($this->shutdownHandlerRegistered){ + return; + } + if($this->bufferAutoFlushOnShutdown){ + // Weak reference: the handler must not keep the instance alive until shutdown + $ref = \WeakReference::create($this->k); + register_shutdown_function(static function() use ($ref){ + $db = $ref->get(); + if($db !== null){ + try { + $db->maintenance->flushAllBuffers(); + } catch(\Throwable $e){ + // The records stay in the buffer (committed): the next access flushes them. An + // exception here would end the script with a fatal error after its work was done. + } + } + }); + $this->shutdownHandlerRegistered = true; + } + } +} diff --git a/src/Internal/Storage/Checkpoints.php b/src/Internal/Storage/Checkpoints.php new file mode 100644 index 0000000..8202622 --- /dev/null +++ b/src/Internal/Storage/Checkpoints.php @@ -0,0 +1,889 @@ + data file => its checkpoint file (a pure memo of ckptPath()) */ + private static $ckptPaths = []; + + private $dataEpochs; + private $fieldIndexCache; + private $indexCache; + private $spatialIndexCache; + private $verifiedIndexes; + private $writableIndexes; + private $cacheGrew; + private $sessionCounter; + private $cacheTick; + private $cacheTouch; + private $faultHook; // @test + private $indexStats; // @test + private $lockRegistry; + private $metrics; + private $noWrites; + private $uncheckpointed; + private $decodeDue; + private $cache; + private $fieldIndex; + private $io; + private $locks; + private $recovery; + private $recoveryLog; + private $rtree; + private $secondary; + private $segments; + private $shardMeta; + + public function __construct(\noneDB\Internal\Core\Kernel $k){ + $this->k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->fieldIndex, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->rtree, $this->secondary, $this->segments, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dataEpochs = &$this->k->dataEpochs; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->indexCache = &$this->k->indexCache; + $this->spatialIndexCache = &$this->k->spatialIndexCache; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->sessionCounter = &$this->rt->sessionCounter; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->faultHook = &$this->rt->faultHook; // @test + $this->indexStats = &$this->rt->indexStats; // @test + $this->lockRegistry = &$this->rt->lockRegistry; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->uncheckpointed = &$this->rt->uncheckpointed; + $this->decodeDue = &$this->rt->decodeDue; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->fieldIndex = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->rtree = null; + $this->secondary = null; + $this->segments = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // JSONL SEGMENT INDEX + // ========================================== + + /** + * Get JSONL index path + * @param string $dbname + * @param int|null $shardId Null for non-sharded + * @return string + */ + public function getJsonlIndexPath($dbname, $shardId = null){ + return self::ckptPath($this->segments->segDataPath($dbname, $shardId)); + } + + /** + * Checkpoint of a data file: segments/{seg}.jsonl -> segments/{seg}.kix + */ + public static function ckptPath($dataPath){ + if(isset(self::$ckptPaths[$dataPath])){ + return self::$ckptPaths[$dataPath]; + } + if(count(self::$ckptPaths) >= 65536){ + self::$ckptPaths = []; + } + return self::$ckptPaths[$dataPath] = substr($dataPath, 0, -6) . '.kix'; + } + + public function newJsonlIndex(){ + return KeyIndex::emptyIndex(); + } + + /** + * A segment's key index (offset and length of every live record), from the cache, or loaded from its + * checkpoint and the data after it + * @param string $dbname + * @param int|null $shardId + * @return array|null Null when the segment doesn't exist + */ + public function readJsonlIndex($dbname, $shardId = null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + // A cached index was checked against its data file (no file check on this hot path); else the file + // must exist + if(!isset($this->indexCache[$path]) && !$this->cache->cachedFileExists($dataPath)){ + return null; + } + if(isset($this->indexCache[$path])){ + $this->cacheTouch[$path] = ++$this->cacheTick; + if(isset($this->verifiedIndexes[$path])){ + if(($this->indexCache[$path]['kb'] !== '' || empty($this->indexCache[$path]['few'])) && $this->indexCache[$path]['ses'] !== $this->sessionCounter){ + // One more operation: a body is decoded once they have paid for it (a database of few segments + // decodes earlier: loadSegmentIndex(), decodeDueIndexes()); they count on after that for the + // data bytes of a segment (RecordIO::heldData()), which a database of few segments doesn't hold + $this->indexCache[$path]['ses'] = $this->sessionCounter; + if((++$this->indexCache[$path]['uses'] + $this->indexCache[$path]['looked']) * self::KEYS_PER_LOOKUP >= $this->indexCache[$path]['live'] + && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeKeyIndex($path); + } + } + $this->metrics['hits']++; + return $this->indexCache[$path]; + } + // Another process wrote the database since this one cached the index, without replacing a data + // file (same epoch): the data only grew, the new lines are replayed + $index = $this->indexCache[$path]; + unset($this->indexCache[$path]); + clearstatcache(true, $dataPath); + $size = @filesize($dataPath); + if($size !== false && $size >= (int)$index['s']){ + if($size > (int)$index['s']){ + $this->replaySegment($dbname, $shardId, $index, $size); + } + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->maybeReaderCheckpoint($dbname, $shardId); + $this->metrics['hits']++; + return $this->indexCache[$path]; + } + } + return $this->loadSegmentIndex($dbname, $shardId); + } + + /** + * Load a segment index: its checkpoint when it fits the data file, then the lines after it. A + * checkpoint that doesn't fit (the data changed behind it) rebuilds the index and the segment's + * secondary indexes under the exclusive lock; a missing one (or one of an unknown version) only + * replays the whole file. + * @return array|null + */ + private function loadSegmentIndex($dbname, $shardId){ + $this->indexStats['loads']++; // @test + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + clearstatcache(true, $dataPath); + if(!is_file($dataPath)){ + if(file_exists($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + $this->cache->markFileNotExists($dataPath); + return null; + } + $size = (int)filesize($dataPath); + $this->metrics['misses']++; + // A checkpoint whose header, body length or a block crc is not valid is no checkpoint (full replay) + $checkpoint = KeyIndex::readFile($path); + if($checkpoint !== null){ + $this->cache->noteIndexRead(KeyIndex::HEADER + strlen($checkpoint[1])); + if(!$this->checkpointFits($dataPath, $checkpoint[0]['s'], $size, $checkpoint[0]['h'] ?? null)){ + return $this->repairSegmentIndex($dbname, $shardId); + } + $index = KeyIndex::fromFile($checkpoint[0], $checkpoint[1]); + $index['ckh'] = $checkpoint[0]['h']; // checked: the crc of the bytes before ck + } else { + $index = $this->newJsonlIndex(); + } + $this->replaySegment($dbname, $shardId, $index, $size); + $index['ses'] = $this->sessionCounter; // the operation that loads it is its first + $index['uses'] = 1; + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->indexCache[$path]['few'] = $this->segments->segmentCount($dbname) <= Segments::FEW_SEGMENTS; + $this->indexCache[$path]['dp'] = $dataPath; + $this->cacheTouch[$path] = ++$this->cacheTick; + $this->cacheGrew = true; + if($index['ck'] < 0){ + // No checkpoint to continue (missing, empty): the next write session of this process writes one + $this->uncheckpointed[$this->locks->getLockPath($dbname)][$shardId === null ? 'main' : $shardId] = $shardId; + } + $this->maybeReaderCheckpoint($dbname, $shardId); + // A database of few segments (§6.4): a small body decoded at its first use, except in the process's + // first operation (a single cold operation keeps the body); any other at the start of the database's + // next operation (LockManager::lockSession() calls decodeDueIndexes()) + if(!empty($this->indexCache[$path]['few']) && $this->indexCache[$path]['kb'] !== ''){ + if(strlen($this->indexCache[$path]['kb']) <= self::DECODE_AT_FIRST_USE && $this->sessionCounter >= 2){ + $this->decodeKeyIndex($path); + } + if(isset($this->indexCache[$path]) && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeDue[$this->locks->getLockPath($dbname)][$path] = true; + } + } + return $this->indexCache[$path]; + } + + /** + * True when a checkpoint covering $covered bytes belongs to this data file: within it, at a line end, + * and the bytes before $covered have the checksum it recorded + */ + public function checkpointFits($dataPath, $covered, $size, $crc){ + if($covered < 0 || $covered > $size || !is_string($crc)){ + return false; + } + return $this->segmentCrc($dataPath, $covered) === $crc; + } + + /** + * crc32b of the up to 256 bytes before $offset of a data file; null when $offset is not a line end + */ + public function segmentCrc($dataPath, $offset){ + if($offset === 0){ + return sprintf('%08x', crc32('')); + } + $fp = @fopen($dataPath, 'rb'); + if($fp === false){ + return null; + } + $start = max(0, $offset - 256); + fseek($fp, $start); + $bytes = (string)fread($fp, $offset - $start); + fclose($fp); + return strlen($bytes) === $offset - $start && substr($bytes, -1) === "\n" ? sprintf('%08x', crc32($bytes)) : null; + } + + /** + * Apply the complete lines of bytes [s, $size) of a segment's data file to its index: the last line of + * a key wins, a tombstone deletes. An unterminated tail stays outside ("s" ends before it; the next + * writer quarantines and cuts it). A complete line that doesn't decode (a damaged region before it, + * STORAGE_FORMAT.md §6.1) is salvaged when a record or tombstone can be found at its end, and reported + * either way. + * @param array|null $hint an earlier index of the segment (p/l), used to find records in damaged lines + * @param bool $report Report salvaged and skipped lines (not for checkIntegrity()) + */ + public function replaySegment($dbname, $shardId, array &$index, $size, $hint = null, $report = true){ + $from = (int)$index['s']; + if($shardId !== null){ + $index['z'] = $this->segments->shardSizeOf($dbname); + $index['sid'] = $shardId; + } + if($size <= $from){ + return; + } + $fp = @fopen($this->segments->segDataPath($dbname, $shardId), 'rb'); + if($fp === false){ + return; + } + $base = $shardId === null ? 0 : $shardId * $this->segments->shardSizeOf($dbname); + $limit = $shardId === null ? null : $base + $this->segments->shardSizeOf($dbname); + $p = $index['p']; + $l = $index['l']; + $x = $index['x']; + unset($index['p'], $index['l'], $index['x']); // sole owner: changed in place, not copied + $hasBody = $index['kb'] !== ''; + $live = $index['live']; + $n = (int)$index['n']; + $d = (int)$index['d']; + $t = (int)$index['t']; + $lines = 0; + $oldEnds = null; + $events = []; + fseek($fp, $from); + $read = $from; + $offset = $from; // file offset of $buffer[0] + $buffer = ''; + while($read < $size){ + $chunk = fread($fp, min(1048576, $size - $read)); + if($chunk === false || $chunk === ''){ + break; + } + $read += strlen($chunk); + $buffer .= $chunk; + $pos = 0; + while(($newline = strpos($buffer, "\n", $pos)) !== false){ + $json = substr($buffer, $pos, $newline - $pos); + $lineOffset = $offset + $pos; + $entry = $this->classifyLine($json, $base, $limit); // a key of another shard is no line of this one + if($entry === null){ + if($oldEnds === null){ + $oldEnds = []; + [$hintOffsets, $hintLengths] = $hint === null ? [[], []] : KeyIndex::locate($hint, null); + foreach($hintOffsets as $k => $at){ + $oldEnds[(int)$at + (int)$hintLengths[$k]] = [(int)$k, (int)$at]; + } + $hintOffsets = $hintLengths = null; + } + $entry = $this->salvageLine($json, $lineOffset, $oldEnds, $base, $limit); + $events[] = [ + 'action' => $entry === null ? 'skip-line' : 'salvaged', + 'seg' => \noneDB\Internal\Storage\Layout::segmentName($shardId), + 'off' => $lineOffset, 'len' => strlen($json) + 1, + 'sha256' => hash('sha256', $json . "\n"), + ] + ($entry === null ? [] : ['key' => $entry[1]]); + } + if($entry === null){ + $d++; + } else { + $key = $entry[1]; + // The key's record so far: written since the checkpoint (p), deleted since (x), or in the body + $inBody = $hasBody && !isset($x[$key]) && KeyIndex::baseGet($index, $key) !== null; + $existed = isset($p[$key]) || $inBody; + if($entry[0] === 't'){ + if($existed){ + unset($p[$key], $l[$key]); + if($inBody){ + $x[$key] = true; + } + $live--; + $d++; + } + $t++; + } else { + if($existed){ + $d++; + } else { + $live++; + } + $p[$key] = $lineOffset + $entry[2]; + $l[$key] = $entry[3]; + unset($x[$key]); + } + if($key >= $base && $key - $base + 1 > $n){ + $n = $key - $base + 1; + } + } + $lines++; + $pos = $newline + 1; + } + $offset += $pos; + $buffer = (string)substr($buffer, $pos); + } + fclose($fp); + $index['p'] = $p; + $index['l'] = $l; + $index['x'] = $x; + $index['live'] = $live; + $index['n'] = $n; + $index['d'] = $d; + $index['t'] = $t; + $index['s'] = $offset; + $index['tl'] = (int)$index['tl'] + $lines; + $this->indexStats['replayedLines'] += $lines; // @test + foreach($report ? $events : [] as $event){ + $event['id'] = $event['seg'] . '.' . $event['off'] . '.' . $event['len'] . '.' . $event['sha256']; + unset($event['sha256']); + $this->recoveryLog->reportRecoveryEvent($dbname, $event); + } + } + + /** + * A segment index changed by a write (exclusive lock): cached, and checkpointed when the session ends + */ + public function storeSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->lockRegistry[$this->locks->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + if($this->faultHook !== null){ $this->recovery->fault('index:written', ['db' => $dbname, 'shard' => $shardId]); } // @test + } + + /** + * True when the lines after a segment's checkpoint are many enough to write a new one + */ + private function needsCheckpoint(array $index){ + return (int)$index['ck'] < 0 + || (int)$index['tl'] >= self::CHECKPOINT_LINES + intdiv($index['live'], 16) + || (int)$index['s'] - (int)$index['ck'] >= self::CHECKPOINT_BYTES; + } + + /** + * Write the checkpoint file of a data file's key index (an index with its epoch; exclusive lock, or a reader: + * the content is the data's): [layout, slots, body, max, live, header, base], or null when its bytes don't end + * at a line or the file can't be written + * @return array|null + */ + public function writeCheckpointFile($dataPath, $shardId, $shardSize, array $index){ + $crc = $this->segmentCrc($dataPath, (int)$index['s']); + if($crc === null){ + return null; + } + $base = $shardId === null ? 0 : $shardId * $shardSize; + [$layout, $slots, $body, $max, $live] = KeyIndex::encodeBody($index, $base); + $header = ['kind' => 'kix', 'v' => 1, 'layout' => $layout, 'base' => $base, 'slots' => $slots, 'live' => $live, + 'n' => (int)$index['n'], 'd' => (int)$index['d'], 't' => (int)$index['t'], 's' => (int)$index['s'], 'h' => $crc, + 'e' => $index['e'], 'created' => (int)$index['created'], 'updated' => time()]; + if($shardId !== null){ + $header['z'] = $shardSize; + $header['sid'] = $shardId; + } + if(!$this->io->writeDerivedFile(self::ckptPath($dataPath), KeyIndex::encodeFile($header, $body))){ + return null; + } + return [$layout, $slots, $body, $max, $live, $header, $base]; + } + + /** + * Write a segment's checkpoint (best effort: a failure only means a longer replay next time). An + * index without epoch gets one, which only the exclusive lock holder may assign. + * @return bool + */ + public function writeCheckpoint($dbname, $shardId, array &$index){ + if($this->noWrites > 0){ + return false; + } + $lock = $this->locks->getLockPath($dbname); + $exclusive = isset($this->lockRegistry[$lock]) && $this->lockRegistry[$lock]['mode'] === LOCK_EX; + if($index['e'] === null){ + if(!$exclusive){ + return false; + } + $index['e'] = bin2hex(random_bytes(4)); + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $written = $this->writeCheckpointFile($dataPath, $shardId, $shardId === null ? null : $this->segments->shardSizeOf($dbname), $index); + if($written === null){ + return false; + } + [$layout, $slots, $body, $max, $live, $header, $base] = $written; + if($index['kb'] !== ''){ + $index = KeyIndex::rebased($index, $layout, $slots, $body, $max, $base); // compact again + } + // (an index held as arrays stays so: it is in use, or was just replayed whole; eviction compacts it) + $index['live'] = $live; + $index['updated'] = $header['updated']; + $index['ck'] = (int)$index['s']; + $index['ckh'] = $header['h']; + $index['tl'] = 0; + $this->indexStats['checkpoints']++; // @test + $this->indexCache[self::ckptPath($dataPath)] = $index; + $this->dataEpochs[$dataPath] = $index['e']; + if($this->faultHook !== null){ $this->recovery->fault('checkpoint:written', ['db' => $dbname, 'shard' => $shardId]); } // @test + return true; + } + + /** + * End of a successful write session: checkpoint the segments it changed that need one + */ + public function checkpointTouchedSegments($dbname, $lock){ + $touched = $this->lockRegistry[$lock]['touched'] ?? []; + $segments = $touched + ($this->uncheckpointed[$lock] ?? []); + unset($this->uncheckpointed[$lock]); + foreach($segments as $shardId){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + if(!isset($this->indexCache[$path], $this->verifiedIndexes[$path]) || !$this->needsCheckpoint($this->indexCache[$path])){ + continue; + } + $index = $this->indexCache[$path]; + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + // a checkpoint is never needed for correctness + } + } + $this->lockRegistry[$lock]['touched'] = []; + foreach($touched as $shardId){ + if(!isset($this->indexCache[$this->getJsonlIndexPath($dbname, $shardId)])){ + continue; // dropped by the cache limit during the session: the next reader that loads it checkpoints it + } + try { + $this->secondary->checkpointSegmentSecondaries($dbname, $shardId); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + } + } + } + + /** + * A reader that replayed a long tail writes the checkpoint itself (shared lock: the data can't change, + * the content is the same as a writer's; no generation bump). Only on top of an existing checkpoint with + * an epoch, and not inside a write session of this process (that one checkpoints at its end). + */ + private function maybeReaderCheckpoint($dbname, $shardId){ + $lock = $this->locks->getLockPath($dbname); + if($this->noWrites > 0 || !isset($this->lockRegistry[$lock]) || $this->lockRegistry[$lock]['mode'] !== LOCK_SH){ + return; + } + $path = $this->getJsonlIndexPath($dbname, $shardId); + $index = $this->indexCache[$path]; + if($index['e'] === null || (int)$index['ck'] < 0 || !$this->needsCheckpoint($index)){ + return; + } + try { + $this->writeCheckpoint($dbname, $shardId, $index); + } catch(\Throwable $ignored){ + } + } + + /** + * Position of the last "\n" in bytes [$from, $to) of a file, -1 if there is none + */ + public function lastNewlineIn($path, $from, $to){ + $fp = @fopen($path, 'rb'); + if($fp === false){ + return -1; + } + try { + $end = $to; + while($end > $from){ + $start = max($from, $end - 65536); + fseek($fp, $start); + $chunk = (string)fread($fp, $end - $start); + $pos = strrpos($chunk, "\n"); + if($pos !== false){ + return $start + $pos; + } + $end = $start; + } + return -1; + } finally { + fclose($fp); + } + } + + /** + * Rebuild a segment index from its data file (exclusive lock, upgrading a shared one): a checkpoint + * that doesn't fit the data, a key the index points at that isn't there. The rebuilt index has a new + * epoch, so the segment's field and spatial indexes are rebuilt as well (STORAGE_FORMAT.md §6.5). + * @param bool $secondaries Also rebuild field/spatial indexes when needed (recovery does it itself) + * @return array|null + */ + public function repairSegmentIndex($dbname, $shardId, $secondaries = true){ + return $this->locks->withLock($dbname, true, function() use ($dbname, $shardId, $secondaries){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + clearstatcache(true, $dataPath); + if(!file_exists($dataPath)){ + return null; + } + if(!is_file($dataPath)){ + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is not a regular file: " . basename($dataPath)); + } + $old = $this->checkpointOnDisk($path); + $index = $this->rebuildSegmentIndexFromData($dbname, $shardId, $old); + if($shardId !== null){ + $this->shardMeta->recountMeta($dbname); + } + // The rebuilt index has a new epoch: every field and spatial index of the segment is invalid now (§6.5), + // also when the entries are the old ones (the data changed in place). Built again while this exclusive + // lock is held, not by the next reader + if($secondaries){ + $this->rebuildSecondaryIndexes($dbname, [$shardId]); + } + return $index; + }, 'rebuildIndex'); + } + + + /** + * The key index a checkpoint file on disk holds (valid by the format, not checked against the data), or null + */ + private function checkpointOnDisk($path){ + $checkpoint = KeyIndex::readFile($path); + if($checkpoint === null){ + return null; + } + $this->cache->noteIndexRead(KeyIndex::HEADER + strlen($checkpoint[1])); + return KeyIndex::fromFile($checkpoint[0], $checkpoint[1]); + } + + /** + * Rebuild field and spatial indexes of some segments (and the global ones of a sharded database) + * from the data. Caller holds the exclusive lock. + */ + public function rebuildSecondaryIndexes($dbname, array $segments){ + $manifest = $this->fieldIndex->getIndexManifest($dbname); + if(empty($manifest['fields']) && empty($manifest['spatial'])){ + return; + } + $known = $this->segments->listSegments($dbname); + // One index that can't be built must never lock the whole database: it is dropped (queries scan, + // the next write or query builds it again) and logged + $guard = function(callable $build, $what, array $files) use ($dbname){ + try { + $build(); + } catch(\Throwable $e){ + if($e->getCode() === \noneDB::LOCK_FAILED){ + throw $e; + } + foreach($files as $file){ + $this->io->fsUnlink($file); + unset($this->fieldIndexCache[$file], $this->spatialIndexCache[$file]); + } + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'index-dropped', 'index' => $what, 'error' => $e->getMessage()]); + } + }; + foreach($segments as $shardId){ + if(!in_array($shardId, $known, true)){ + continue; + } + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field, $shardId){ + $index = $this->fieldIndex->buildSegmentFieldIndex($dbname, $shardId, $field); + $this->fieldIndex->writeFieldIndex($dbname, $field, $index, $shardId); + }, 'field ' . $field, [$this->fieldIndex->getFieldIndexPath($dbname, $field, $shardId)]); + } + foreach($manifest['spatial'] as $field){ + $guard(function() use ($dbname, $field, $shardId, $manifest){ + $tree = $this->rtree->buildSegmentSpatialIndex($dbname, $field, $shardId, + isset($manifest['spatialOptions'][$field]['nodeSize']) ? ['nodeSize' => $manifest['spatialOptions'][$field]['nodeSize']] : []); + $this->rtree->writeSpatialIndex($dbname, $field, $tree, $shardId); + if($shardId !== null){ + $this->rtree->updateGlobalSpatialShard($dbname, $field, $shardId, $tree); + } + }, 'spatial ' . $field, [$this->rtree->getSpatialIndexPath($dbname, $field, $shardId)]); + } + $this->cache->evictBetweenSegments(); + } + if($this->segments->isSharded($dbname)){ + foreach($manifest['fields'] as $field){ + $guard(function() use ($dbname, $field){ + $this->fieldIndex->rebuildGlobalFieldIndex($dbname, $field); + }, 'global field ' . $field, [$this->fieldIndex->getGlobalFieldIndexPath($dbname, $field)]); + } + } + } + + /** + * Index of a segment a writer is about to change (exclusive lock), taken out of the cache: the + * writer changes it in place and stores it back (storeSegmentIndex). Checked against the data file + * when it was not yet known to match it: an unterminated tail (crashed append) is quarantined and cut + * off, so the next line can't be glued to it. A checked index stays checked until its cache is + * dropped; appendToFile() compares the size it finds with the index anyway (no extra system call), + * which catches anything else. + * @return array|null + */ + public function writableSegmentIndex($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null){ + return null; + } + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $path = self::ckptPath($dataPath); + if(!isset($this->writableIndexes[$path])){ + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + if($size < (int)$index['s']){ + $index = $this->repairSegmentIndex($dbname, $shardId); + clearstatcache(true, $dataPath); + $size = (int)@filesize($dataPath); + } + if($size > (int)$index['s']){ + $this->recovery->quarantineAndCut($dbname, $shardId, $dataPath, (int)$index['s'], $size); + } + $this->writableIndexes[$path] = true; + } + unset($this->indexCache[$path]); // the caller is its only holder until it stores it back + return $index; + } + + /** + * A writer's index goes back to the cache unchanged (nothing to write after all) + */ + public function releaseSegmentIndex($dbname, $shardId, array $index){ + $path = $this->getJsonlIndexPath($dbname, $shardId); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + } + + /** + * Rebuild a segment index from the whole data file and write its checkpoint (exclusive lock): new + * segment epoch, as there is no checkpoint it continues + * @param array|null $old The checkpoint found (created time, key counter, records in damaged lines) + * @return array + */ + public function rebuildSegmentIndexFromData($dbname, $shardId, $old = null){ + $dataPath = $this->segments->segDataPath($dbname, $shardId); + if($old === null){ + $old = $this->checkpointOnDisk(self::ckptPath($dataPath)); + } + $index = $this->newJsonlIndex(); + if($old !== null){ + $index['created'] = $old['created'] ?? $index['created']; + if($shardId === null && isset($old['n'])){ + $index['n'] = (int)$old['n']; + } + } + clearstatcache(true, $dataPath); + $this->replaySegment($dbname, $shardId, $index, (int)@filesize($dataPath), $old); // its entries only if a line is damaged + $index['e'] = bin2hex(random_bytes(4)); + $path = self::ckptPath($dataPath); + $this->indexCache[$path] = $index; + $this->verifiedIndexes[$path] = true; + $this->writeCheckpoint($dbname, $shardId, $index); + return $this->indexCache[$path]; + } + + /** + * Classify one complete data line: ['r', key, 0, length] (record), ['t', key] (tombstone), or null + * when it isn't a valid line of this segment + */ + public function classifyLine($json, $base, $limit){ + $record = json_decode($json, true); + if(!is_array($record)){ + return null; + } + if(count($record) === 1 && isset($record['__nonedb_deleted']) && is_int($record['__nonedb_deleted'])){ + $key = $record['__nonedb_deleted']; + return $key >= $base && ($limit === null || $key < $limit) ? ['t', $key] : null; + } + if(isset($record['key']) && is_int($record['key'])){ + $key = $record['key']; + return $key >= $base && ($limit === null || $key < $limit) ? ['r', $key, 0, strlen($json)] : null; + } + return null; + } + + /** + * Find a record or tombstone at the end of a line that doesn't decode (a damaged region before it). + * First where the earlier index says a record ended; then candidates from the + * rightmost "{" leftwards: inside a record no "{" starts a JSON text that also ends the line, so the + * first one that decodes is where the record starts. + * @return array|null Like classifyLine(), with the record's offset inside the line + */ + public function salvageLine($json, $lineOffset, array $oldEnds, $base, $limit){ + $end = $lineOffset + strlen($json); + if(isset($oldEnds[$end]) && $oldEnds[$end][1] > $lineOffset){ + $start = $oldEnds[$end][1] - $lineOffset; + $entry = $this->classifySalvaged(substr($json, $start), $base, $limit); + if($entry !== null && $entry[0] === 'r' && $entry[1] === $oldEnds[$end][0]){ + return ['r', $entry[1], $start, strlen($json) - $start]; + } + } + $pos = strlen($json); + for($tries = 0; $tries < 4096; $tries++){ + $pos = $pos > 0 ? strrpos($json, '{', $pos - strlen($json) - 1) : false; + if($pos === false || $pos === 0){ + return null; + } + $entry = $this->classifySalvaged(substr($json, $pos), $base, $limit); + if($entry !== null){ + return $entry[0] === 't' ? $entry : ['r', $entry[1], $pos, strlen($json) - $pos]; + } + } + return null; + } + + /** + * classifyLine() for a salvage candidate: also requires an object whose last member is "key" + */ + private function classifySalvaged($candidate, $base, $limit){ + $entry = $this->classifyLine($candidate, $base, $limit); + if($entry === null || $entry[0] === 't'){ + return $entry; + } + $record = json_decode($candidate, true); + $keys = array_keys($record); + return end($keys) === 'key' && $keys !== range(0, count($keys) - 1) ? $entry : null; + } + + // Segment index: checkpoint + replay (docs/STORAGE_FORMAT.md §6.4). The data file is a log; the .kix + // file is a checkpoint of the index at byte offset "s" of it: a 512-byte JSON header ("kind":"kix", "v":1, + // layout, base, slots, live, n next key, d garbage lines, t tombstone lines, s, h = crc32b of the up to 256 + // bytes before s, e segment epoch, z shard size, sid shard id), then the slots in 4 KiB blocks with their crc + // (KeyIndex). Loading reads the checkpoint and replays the complete lines after S (an unterminated tail is never + // part of it, readers never cut it). Writes append to the data and update the cached index only; a + // write session writes a new checkpoint of a segment it changed once its tail reached 64 lines plus + // 1/16 of the live records, or 1 MiB (checkpointTouchedSegments). A checkpoint counts only for its data + // file: S within it, at a line end, h matching; a data file is replaced only after its checkpoint was + // deleted. "e" is the segment's data epoch for the secondary indexes: new when the data file is + // replaced or the index rebuilt without a checkpoint, assigned only under the exclusive lock. + // Runtime keys of a cached index (never written): "ck" = S of the checkpoint on disk (-1: none), + // "tl" = lines after it. + + /** A key index body of at most DECODE_MAX_KEYS keys may be decoded into arrays */ + private const DECODE_MAX_KEYS = 250000; + /** A body of at most this many bytes, in a database of few segments, is decoded at its first use (§6.4) */ + private const DECODE_AT_FIRST_USE = 65536; + /** Estimated memory of a decoded key index per key */ + private const DECODED_KEY_BYTES = 80; + /** + * Keys whose decoding costs about what one key lookup costs more on a body than on arrays (measured: + * 0.17 µs per key to decode, 1.3 µs more per lookup on a body) + */ + private const KEYS_PER_LOOKUP = 8; + + /** + * Decode a cached key index body into arrays (lookups at array speed) when it has at most + * DECODE_MAX_KEYS keys and memory allows (CacheManager::decodeFits()). In a database of few segments + * (Segments::FEW_SEGMENTS: each is in constant use): at its first use when the body is at most + * DECODE_AT_FIRST_USE bytes (loadSegmentIndex()), else at its second operation. Otherwise once the + * lookups through the body have paid for decoding: operations (one key each, counted in readJsonlIndex()) + * and keys looked up in bulk (noteLookups()), KEYS_PER_LOOKUP live keys each. Light use keeps the body: + * a cold query over many segments, a few reads of each segment. + */ + private function decodeKeyIndex($path){ + $live = $this->indexCache[$path]['live']; + if($live <= self::DECODE_MAX_KEYS && \noneDB\Internal\Cache\CacheManager::decodeFits(self::DECODED_KEY_BYTES * $live)){ + $this->indexStats['keyDecodes']++; // @test + $this->indexCache[$path] = KeyIndex::materialize($this->indexCache[$path]); + $this->cacheGrew = true; + } + } + + /** + * Start of an operation (lock session) on a database of few segments: the key index bodies an earlier + * operation loaded are decoded now (§6.4), whether this operation reads them or not. Called once the + * lock state was checked: an index another process made stale is gone from the cache by then. + */ + public function decodeDueIndexes($lockPath){ + foreach($this->decodeDue[$lockPath] as $path => $unused){ + if(isset($this->indexCache[$path], $this->verifiedIndexes[$path]) && $this->indexCache[$path]['kb'] !== ''){ + $this->decodeKeyIndex($path); + } + } + unset($this->decodeDue[$lockPath]); + } + + /** + * A segment's key index for a read of every record: a whole read decodes the body anyway, so the decoded + * index is kept (when memory allows) and the writes that often follow such a read work on arrays + */ + public function decodedIndex($dbname, $shardId){ + $index = $this->readJsonlIndex($dbname, $shardId); + if($index === null || $index['kb'] === ''){ + return $index; + } + $path = self::ckptPath($this->segments->segDataPath($dbname, $shardId)); + $this->decodeKeyIndex($path); + return $this->indexCache[$path] ?? $index; + } + + /** Keys looked up in bulk through the key index body of a data file (see decodeKeyIndex()) */ + public function noteLookups($dataPath, $n){ + $path = self::ckptPath($dataPath); + if(!isset($this->indexCache[$path]) || $this->indexCache[$path]['kb'] === ''){ + return; + } + $this->indexCache[$path]['looked'] += $n; + if(($this->indexCache[$path]['uses'] + $this->indexCache[$path]['looked']) * self::KEYS_PER_LOOKUP >= $this->indexCache[$path]['live']){ + $this->decodeKeyIndex($path); + } + } + public const CHECKPOINT_LINES = 64; + public const CHECKPOINT_BYTES = 1048576; +} diff --git a/src/Internal/Storage/KeyIndex.php b/src/Internal/Storage/KeyIndex.php new file mode 100644 index 0000000..0618d6e --- /dev/null +++ b/src/Internal/Storage/KeyIndex.php @@ -0,0 +1,579 @@ + 0, 'd' => 0, 't' => 0, 's' => 0, 'e' => null, 'created' => $now, 'updated' => $now, + 'ck' => -1, 'ckh' => null, 'tl' => 0, 'kb' => '', 'kl' => 'dense', 'kbase' => 0, 'kslots' => 0, 'kmax' => -1, 'live' => 0, + 'p' => [], 'l' => [], 'x' => [], 'looked' => 0, 'uses' => 0, 'ses' => -1]; + } + + // ------------------------------------------------------------------ lookups + + /** [offset, length] of a key in the body, or null */ + public static function baseGet(array $index, int $key): ?array { + $kb = $index['kb']; + if($kb === ''){ + return null; + } + $local = $key - $index['kbase']; + if($local < 0){ + return null; + } + $slots = $index['kslots']; + if($index['kl'] === 'dense'){ + if($local >= $slots){ + return null; + } + $block = intdiv($local, self::DENSE_SLOTS); + $first = $block * self::DENSE_SLOTS; + $m = $slots - $first < self::DENSE_SLOTS ? $slots - $first : self::DENSE_SLOTS; + $at = $block * self::BLOCK; + $length = unpack('V', $kb, $at + 8 * $m + 4 * ($local - $first))[1]; + return $length === 0 ? null : [unpack('P', $kb, $at + 8 * ($local - $first))[1], $length]; + } + $lo = 0; + $hi = $slots - 1; + while($lo <= $hi){ + $mid = ($lo + $hi) >> 1; + $block = intdiv($mid, self::SPARSE_ENTRIES); + $first = $block * self::SPARSE_ENTRIES; + $i = $mid - $first; + $at = $block * self::BLOCK; + $found = unpack('V', $kb, $at + 4 * $i)[1]; + if($found === $local){ + $m = $slots - $first < self::SPARSE_ENTRIES ? $slots - $first : self::SPARSE_ENTRIES; + return [unpack('P', $kb, $at + 4 * $m + 8 * $i)[1], unpack('V', $kb, $at + 12 * $m + 4 * $i)[1]]; + } + if($found < $local){ + $lo = $mid + 1; + } else { + $hi = $mid - 1; + } + } + return null; + } + + public static function has(array $index, int $key): bool { + return isset($index['p'][$key]) || ($index['kb'] !== '' && !isset($index['x'][$key]) && self::baseGet($index, $key) !== null); + } + + public static function count(array $index): int { + return $index['live']; + } + + /** + * Point a key at a line; true when the key had a record before (its old line becomes garbage) + */ + public static function set(array &$index, int $key, int $offset, int $length): bool { + $existed = isset($index['p'][$key]) || (!isset($index['x'][$key]) && self::baseGet($index, $key) !== null); + $index['p'][$key] = $offset; + $index['l'][$key] = $length; + unset($index['x'][$key]); + if(!$existed){ + $index['live']++; + } + return $existed; + } + + /** + * Point the keys of appended lines at them: $lines [key => line without "\n"] written in this order from + * byte $offset. Returns how many keys had a record before (their old lines are garbage now). + */ + public static function setLines(array &$index, array $lines, int $offset): int { + $replaced = 0; + $added = 0; + $p = &$index['p']; + $l = &$index['l']; + if($index['kb'] === ''){ + foreach($lines as $key => $json){ + if(isset($p[$key])){ + $replaced++; + } else { + $added++; + } + $p[$key] = $offset; + $l[$key] = $length = strlen($json); + $offset += $length + 1; + } + } else { + $x = &$index['x']; + foreach($lines as $key => $json){ + if(isset($p[$key]) || (!isset($x[$key]) && self::baseGet($index, $key) !== null)){ + $replaced++; + } else { + $added++; + } + $p[$key] = $offset; + $l[$key] = $length = strlen($json); + unset($x[$key]); + $offset += $length + 1; + } + unset($x); + } + unset($p, $l); + $index['live'] += $added; + return $replaced; + } + + /** Delete keys; returns how many had a record */ + public static function removeMany(array &$index, array $keys): int { + $removed = 0; + if($index['kb'] === ''){ + // A decoded index: its arrays only, no call per key + $p = &$index['p']; + $l = &$index['l']; + foreach($keys as $key){ + if(isset($p[$key])){ + unset($p[$key], $l[$key]); + $removed++; + } + } + unset($p, $l); + $index['live'] -= $removed; + return $removed; + } + foreach($keys as $key){ + if(self::remove($index, (int)$key)){ + $removed++; + } + } + return $removed; + } + + /** Delete a key; true when it had a record */ + public static function remove(array &$index, int $key): bool { + if(isset($index['p'][$key])){ + unset($index['p'][$key], $index['l'][$key]); + if(self::baseGet($index, $key) !== null){ + $index['x'][$key] = true; + } + $index['live']--; + return true; + } + if(isset($index['x'][$key]) || self::baseGet($index, $key) === null){ + return false; + } + $index['x'][$key] = true; + $index['live']--; + return true; + } + + /** The highest live key (-1: none) */ + public static function maxKey(array $index): int { + $max = $index['kmax'] >= 0 && !isset($index['x'][$index['kmax']]) ? $index['kmax'] : self::bodyMax($index); + foreach($index['p'] as $key => $unused){ + if($key > $max){ + $max = $key; + } + } + return $max; + } + + + /** Every live key: ascending, but in the order of its arrays for a decoded index (the caller sorts) */ + public static function keys(array $index): array { + if($index['kb'] === ''){ + return array_keys($index['p']); + } + if(empty($index['p']) && empty($index['x'])){ + return self::baseKeys($index); + } + // The body's keys, the removed ones out, the delta's in (no array per key) + $live = self::bodyArrays($index)[1]; + foreach($index['x'] as $key => $unused){ + unset($live[$key]); + } + if(!empty($index['l'])){ + foreach($index['l'] as $key => $length){ + $live[$key] = $length; + } + ksort($live); + } + return array_keys($live); + } + + /** + * Offsets and lengths of $keys (all live keys when null) for a batch read: [key => offset], [key => length] + */ + public static function locate(array $index, ?array $keys): array { + $positions = []; + $lengths = []; + if($keys === null){ + // Every live key: the arrays of a decoded index (the body unpacked in a few C-level calls, the + // delta applied); in no particular order, the batch read orders by position + $all = self::materialize($index); + return [$all['p'], $all['l']]; + } + $p = $index['p']; + $l = $index['l']; + if($index['kb'] === ''){ + // Every key in the arrays (a decoded index): the hot path, no call per key + foreach($keys as $key){ + if(isset($p[$key])){ + $positions[$key] = $p[$key]; + $lengths[$key] = $l[$key]; + } + } + return [$positions, $lengths]; + } + $x = $index['x']; + foreach($keys as $key){ + if(isset($p[$key])){ + $positions[$key] = $p[$key]; + $lengths[$key] = $l[$key]; + } elseif(!isset($x[$key]) && ($e = self::baseGet($index, (int)$key)) !== null){ + $positions[$key] = $e[0]; + $lengths[$key] = $e[1]; + } + } + return [$positions, $lengths]; + } + + /** + * The same index with every key in p/l (no body): lookups at array speed, about 5× the memory of a body. + * For a segment in repeated use (plan D6: hot segments are kept decoded within the cache budget). + */ + public static function materialize(array $index): array { + if($index['kb'] === ''){ + return $index; + } + [$p, $l] = self::bodyArrays($index); + foreach($index['x'] as $key => $unused){ + unset($p[$key], $l[$key]); + } + foreach($index['p'] as $key => $offset){ + $p[$key] = $offset; + $l[$key] = $index['l'][$key]; + } + $index['p'] = $p; + $index['l'] = $l; + $index['x'] = []; + $index['kb'] = ''; + $index['kslots'] = 0; + $index['kmax'] = -1; + return $index; + } + + /** The body's live keys as [key => offset], [key => length], ascending (a few C-level calls per block) */ + private static function bodyArrays(array $index): array { + $kb = $index['kb']; + $base = $index['kbase']; + $slots = $index['kslots']; + $p = []; + $l = []; + if($index['kl'] === 'dense'){ + if($slots === 0){ + return [[], []]; + } + // The offset and length runs of every block joined, then decoded at once + $offsets = ''; + $lengths = ''; + for($block = 0, $first = 0; $first < $slots; $block++, $first += self::DENSE_SLOTS){ + $m = min(self::DENSE_SLOTS, $slots - $first); + $at = $block * self::BLOCK; + $offsets .= substr($kb, $at, 8 * $m); + $lengths .= substr($kb, $at + 8 * $m, 4 * $m); + } + // One array at a time, the runs released once used (the peak of a decode stays near its result) + $keys = range($base, $base + $slots - 1); + $l = array_combine($keys, unpack('V*', $lengths)); + $lengths = null; + if(in_array(0, $l, true)){ + $l = array_filter($l); // slots without a record (length 0) + } + $p = array_combine($keys, unpack('P*', $offsets)); + $offsets = $keys = null; + if(count($l) < $slots){ + $p = array_intersect_key($p, $l); + } + return [$p, $l]; + } + for($block = 0, $first = 0; $first < $slots; $block++, $first += self::SPARSE_ENTRIES){ + $m = min(self::SPARSE_ENTRIES, $slots - $first); + $at = $block * self::BLOCK; + $keys = []; + foreach(unpack('V' . $m, $kb, $at) as $local){ + $keys[] = $base + $local; + } + $p += array_combine($keys, unpack('P' . $m, $kb, $at + 4 * $m)); + $l += array_combine($keys, unpack('V' . $m, $kb, $at + 12 * $m)); + } + return [$p, $l]; + } + + + // ------------------------------------------------------------------ the body + + /** Live entries of the body: key => [offset, length], ascending */ + + /** Live keys of the body, ascending */ + private static function baseKeys(array $index): array { + return array_keys(self::bodyArrays($index)[1]); + } + + /** + * Encode the live keys of an index as a checkpoint body for a segment whose first key is $base: + * [layout, slots, body, highest key, live keys] + */ + public static function encodeBody(array $index, int $base): array { + [$p, $l] = self::allArrays($index); + $live = count($p); + $max = $live === 0 ? -1 : (int)array_key_last($p); + $slots = $max < $base ? 0 : $max - $base + 1; + $body = ''; + if($slots <= max(1024, 4 * $live)){ + // Every slot in key order (0 where no record), built without a loop per key + $zeros = $slots === 0 ? [] : array_fill($base, $slots, 0); + $offsets = array_values($live === $slots ? $p : array_replace($zeros, $p)); + $lengths = array_values($live === $slots ? $l : array_replace($zeros, $l)); + unset($zeros); + for($first = 0; $first < $slots; $first += self::DENSE_SLOTS){ + $m = min(self::DENSE_SLOTS, $slots - $first); + $data = pack('P*', ...array_slice($offsets, $first, $m)) . pack('V*', ...array_slice($lengths, $first, $m)); + $body .= $data . pack('V', crc32($data)); + } + return ['dense', $slots, $body, $max, $live]; + } + $keys = []; + $offsets = []; + $lengths = []; + foreach($p as $key => $offset){ + $keys[] = $key - $base; + $offsets[] = $offset; + $lengths[] = $l[$key]; + if(count($keys) === self::SPARSE_ENTRIES){ + $data = str_pad(pack('V*', ...$keys) . pack('P*', ...$offsets) . pack('V*', ...$lengths), self::BLOCK - 4, "\0"); + $body .= $data . pack('V', crc32($data)); + $keys = []; + $offsets = []; + $lengths = []; + } + } + if(!empty($keys)){ + $data = str_pad(pack('V*', ...$keys) . pack('P*', ...$offsets) . pack('V*', ...$lengths), self::BLOCK - 4, "\0"); + $body .= $data . pack('V', crc32($data)); + } + return ['sparse', $live, $body, $max, $live]; + } + + /** Every live key as [key => offset], [key => length], ascending by key (no pair per key) */ + private static function allArrays(array $index): array { + if($index['kb'] === ''){ + $p = $index['p']; + $l = $index['l']; + } else { + [$p, $l] = self::bodyArrays($index); + foreach($index['x'] as $key => $unused){ + unset($p[$key], $l[$key]); + } + foreach($index['p'] as $key => $offset){ + $p[$key] = $offset; + $l[$key] = $index['l'][$key]; + } + } + // Sorted only when not ascending already (sorting copies arrays the index shares), both by key + $previous = PHP_INT_MIN; + foreach($p as $key => $unused){ + if($key < $previous){ + ksort($p); + ksort($l); + break; + } + $previous = $key; + } + return [$p, $l]; + } + + /** A derived file's header: its JSON with its own crc as the last member (§6.5) */ + public static function sealHeader(string $json): string { + return substr($json, 0, -1) . ',"crc":' . crc32($json) . '}'; + } + + /** The members of a derived file's header, or null when its crc doesn't fit */ + public static function openHeader(string $line): ?array { + $line = rtrim($line, " \n"); + $at = strrpos($line, ',"crc":'); + if($at === false || substr($line, $at + 7, -1) !== (string)crc32(substr($line, 0, $at) . '}')){ + return null; + } + $header = json_decode($line, true); + if(!is_array($header)){ + return null; + } + unset($header['crc']); + return $header; + } + + /** The bytes of a checkpoint file: header line and body */ + public static function encodeFile(array $header, string $body): string { + return str_pad(self::sealHeader((string)json_encode($header, JSON_UNESCAPED_SLASHES)), self::HEADER - 1, ' ') . "\n" . $body; + } + + /** + * Read a checkpoint file's bytes: [header, body] when its header, body length and every block crc are + * valid (§6.4), else null + */ + public static function decodeFile($raw): ?array { + if(!is_string($raw) || strlen($raw) < self::HEADER){ + return null; + } + $header = self::headerOf(substr($raw, 0, self::HEADER)); + if($header === null || strlen($raw) !== self::HEADER + self::bodyLength($header)){ + return null; + } + $body = substr($raw, self::HEADER); + return self::bodyFits($body) ? [$header, $body] : null; + } + + /** + * The checkpoint at $path, as decodeFile() returns it, read through one handle: the header, then the + * body it announces (nothing may follow). Only the body is held, never the file and its body at once. + */ + public static function readFile(string $path): ?array { + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + try { + $header = self::headerOf((string)fread($fp, self::HEADER)); + if($header === null){ + return null; + } + $length = self::bodyLength($header); + if((fstat($fp)['size'] ?? null) !== self::HEADER + $length){ + return null; // a body the file doesn't hold is never asked for + } + $body = $length === 0 ? '' : (string)fread($fp, $length); + if(strlen($body) !== $length){ + return null; + } + return self::bodyFits($body) ? [$header, $body] : null; + } finally { + fclose($fp); + } + } + + /** The header of a checkpoint (its first HEADER bytes, ending with a newline), or null when it is not valid */ + private static function headerOf(string $head): ?array { + if(strlen($head) !== self::HEADER || $head[self::HEADER - 1] !== "\n"){ + return null; + } + $header = self::openHeader(substr($head, 0, self::HEADER - 1)); + if($header === null || ($header['kind'] ?? null) !== 'kix' || ($header['v'] ?? null) !== 1 + || !in_array($header['layout'] ?? null, ['dense', 'sparse'], true)){ + return null; + } + foreach(['base', 'slots', 'live', 'n', 'd', 't', 's'] as $member){ + if(!is_int($header[$member] ?? null) || $header[$member] < 0){ + return null; + } + } + // Its body's length is an integer, and so is every key: base + slot, or base + a local key of 32 bits + return $header['slots'] > intdiv(PHP_INT_MAX, self::BLOCK) || $header['base'] > PHP_INT_MAX - $header['slots'] - 0xFFFFFFFF ? null : $header; + } + + /** The body length a valid header announces */ + private static function bodyLength(array $header): int { + $dense = $header['layout'] === 'dense'; + $per = $dense ? self::DENSE_SLOTS : self::SPARSE_ENTRIES; + $slots = $header['slots']; + $blocks = $slots === 0 ? 0 : intdiv($slots - 1, $per) + 1; + return $blocks === 0 ? 0 : ($blocks - 1) * self::BLOCK + ($dense ? ($slots - ($blocks - 1) * $per) * 12 + 4 : self::BLOCK); + } + + /** True when every block of a body is sealed by its crc (one block at a time, no copy of the body) */ + private static function bodyFits(string $body): bool { + for($at = 0, $length = strlen($body); $at < $length; $at += self::BLOCK){ + if(crc32(substr($body, $at, self::BLOCK)) !== self::CRC_RESIDUE){ + return false; + } + } + return true; + } + + /** + * A key index from a decoded checkpoint (fits checked by the caller) + */ + public static function fromFile(array $header, string $body): array { + $index = self::emptyIndex(); + foreach(['n', 'd', 't', 's', 'created', 'updated', 'z', 'sid'] as $member){ + if(isset($header[$member])){ + $index[$member] = $header[$member]; + } + } + $index['e'] = isset($header['e']) && is_string($header['e']) ? $header['e'] : null; + $index['kb'] = $body; + $index['kl'] = $header['layout']; + $index['kbase'] = $header['base']; + $index['kslots'] = $header['slots']; + $index['live'] = $header['live']; + $index['kmax'] = self::bodyMax($index); + $index['ck'] = $header['s']; + return $index; + } + + /** + * The highest key of the body that the delta didn't remove (-1: none), from the top down: no array of the + * body is made (a deleted highest key of a large segment costs a few lookups, not a decode) + */ + private static function bodyMax(array $index): int { + if($index['kl'] === 'sparse'){ + for($i = $index['kslots'] - 1; $i >= 0; $i--){ + $block = intdiv($i, self::SPARSE_ENTRIES); + $key = $index['kbase'] + unpack('V', $index['kb'], $block * self::BLOCK + 4 * ($i - $block * self::SPARSE_ENTRIES))[1]; + if(!isset($index['x'][$key])){ + return $key; + } + } + return -1; + } + for($local = $index['kslots'] - 1; $local >= 0; $local--){ + $key = $index['kbase'] + $local; + if(!isset($index['x'][$key]) && self::baseGet($index, $key) !== null){ + return $key; + } + } + return -1; + } + + /** + * The index after a checkpoint of it was written: the new body, no delta + */ + public static function rebased(array $index, string $layout, int $slots, string $body, int $max, int $base): array { + $index['kb'] = $body; + $index['kl'] = $layout; + $index['kbase'] = $base; + $index['kslots'] = $slots; + $index['kmax'] = $max; + $index['p'] = []; + $index['l'] = []; + $index['x'] = []; + return $index; + } +} diff --git a/src/Internal/Storage/Layout.php b/src/Internal/Storage/Layout.php new file mode 100644 index 0000000..6f65026 --- /dev/null +++ b/src/Internal/Storage/Layout.php @@ -0,0 +1,736 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->io, $this->locks, $this->recovery, $this->recoveryLog, $this->store, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->dbDir = &$this->k->dbDir; + $this->dbHomes = &$this->k->dbHomes; + $this->durable = &$this->k->durable; + $this->formatRecords = &$this->k->formatRecords; + $this->genPaths = &$this->k->genPaths; + $this->lockPaths = &$this->k->lockPaths; + $this->shardSize = &$this->k->shardSize; + $this->directoryListings = &$this->rt->directoryListings; // @test + $this->lockRegistry = &$this->rt->lockRegistry; + $this->noWrites = &$this->rt->noWrites; + $this->staticCacheEnabled = &$this->rt->staticCacheEnabled; + $this->staticFileExistsCache = &$this->rt->staticFileExistsCache; + $this->staticSanitizeCache = &$this->rt->staticSanitizeCache; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->recoveryLog = null; + $this->store = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Validate a database name. + * Allowed characters: A-Z a-z 0-9, space, apostrophe and hyphen. Anything else is rejected, never + * stripped: stripping would let different names collide. + * + * @param mixed $dbname Database name + * @return string The (unchanged) database name + * @throws \InvalidArgumentException + */ + public function normalizeDbName($dbname){ + if(is_int($dbname)){ + $dbname = (string)$dbname; + } + if(!is_string($dbname)){ + throw new \InvalidArgumentException('noneDB: database name must be a string'); + } + if(isset($this->staticSanitizeCache[$dbname])){ + return $dbname; + } + if($dbname === '' || preg_match("/^[A-Za-z0-9' -]+$/D", $dbname) !== 1){ + throw new \InvalidArgumentException( + "noneDB: invalid database name '{$dbname}'. Only letters A-Z/a-z, digits, space, apostrophe (') and hyphen (-) are allowed." + ); + } + if($this->staticCacheEnabled){ + $this->staticSanitizeCache[$dbname] = true; + } + return $dbname; + } + + public function dbHome($dbname){ + if(isset($this->dbHomes[$dbname])){ + return $this->dbHomes[$dbname]; + } + if(count($this->dbHomes) >= 4096){ + $this->dbHomes = $this->lockPaths = []; + } + $home = $this->dbHomes[$dbname] = $this->dbDir . $this->store->locator($dbname) . '.ndb4/'; + $this->lockPaths[$dbname] = $home . 'LOCK'; + return $home; + } + + /** + * Active generation of a database. A database without a valid CURRENT (absent, being created) uses + * generation 1: the state check in prepareDb() runs before any of its files is used. + */ + public function generationNumber($dbname){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->currentGenerations[$lock])){ + return $this->currentGenerations[$lock]; + } + $format = $this->readFormatRecord($dbname); + $current = $format === null ? null : $this->readCurrentRecord($dbname, $format); + $generation = $current === null ? 1 : $current['generation']; + if($current !== null){ + $this->currentGenerations[$lock] = $generation; + } + return $generation; + } + + public function genDir($dbname){ + $home = $this->dbHomes[$dbname] ?? $this->dbHome($dbname); + $generation = $this->currentGenerations[$this->lockPaths[$dbname]] ?? $this->generationNumber($dbname); + if(($this->genPaths[$dbname]['gen'] ?? null) !== $generation){ + // Paths of the active generation are built once: the same strings (their hashes computed once) + // serve every lookup of the caches keyed by them + if(count($this->genPaths) >= 4096){ + $this->genPaths = []; + } + $this->genPaths[$dbname] = ['gen' => $generation, 'dir' => $home . 'generations/' . $generation . '/']; + } + return $this->genPaths[$dbname]['dir']; + } + + /** + * A file of the active generation by its path relative to the generation directory (cached) + */ + public function genPath($dbname, $relative){ + // Hot path: the cached path while its generation is still the active one (no call) + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['f'][$relative]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['f'][$relative]; + } + $dir = $this->genDir($dbname); + return $this->genPaths[$dbname]['f'][$relative] ?? ($this->genPaths[$dbname]['f'][$relative] = $dir . $relative); + } + + public static function segmentName($shardId){ + return $shardId === null ? 'main' : 's' . $shardId; + } + + /** + * FORMAT of a database, validated; null when missing or invalid + */ + public function readFormatRecord($dbname){ + $lock = $this->locks->getLockPath($dbname); + if(isset($this->formatRecords[$lock])){ + return $this->formatRecords[$lock]; + } + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'FORMAT'), 'database'); + if($record !== null && !$this->validIdentity($record, $dbname)){ + $record = null; + } + if($record !== null){ + $this->formatRecords[$lock] = $record; + } + return $record; + } + + /** + * The identity members of a FORMAT or metadata record are valid for a database named $dbname (§4.1) + */ + public function validIdentity(array $record, $dbname){ + return ($record['storageFormat'] ?? null) === \noneDB::STORAGE_FORMAT && is_string($record['databaseId'] ?? null) + && ($record['name'] ?? null) === (string)$dbname && is_int($record['shardSize'] ?? null) && $record['shardSize'] > 0 + && is_int($record['createdAt'] ?? null); + } + + /** + * CURRENT, valid by §4.3 (checksum, databaseId of FORMAT, valid metadata of its generation), or null + */ + private function readCurrentRecord($dbname, array $format){ + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'CURRENT'), 'current'); + if($record === null || ($record['databaseId'] ?? null) !== $format['databaseId'] || !is_int($record['generation'] ?? null) + || $record['generation'] < 1){ + return null; + } + return $this->readGenerationMetadata($dbname, $record['generation'], $format['databaseId']) === null ? null : $record; + } + + private function readGenerationMetadata($dbname, $generation, $databaseId){ + $record = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'generations/' . $generation . '/metadata'), 'generation'); + if($record === null || ($record['databaseId'] ?? null) !== $databaseId || ($record['generation'] ?? null) !== $generation + || !$this->validIdentity($record, $dbname) || !in_array($record['layout'] ?? null, ['single', 'sharded'], true)){ + return null; + } + return $record; + } + + public function identityOf(array $record){ + return ['storageFormat' => $record['storageFormat'], 'databaseId' => $record['databaseId'], 'name' => $record['name'], + 'shardSize' => $record['shardSize'], 'createdAt' => $record['createdAt'], + 'formatOptions' => (object)(is_array($record['formatOptions'] ?? null) ? $record['formatOptions'] : [])]; // always a JSON object + } + + public function writeGenerationMetadata($dbname, $generation, array $identity, $layout){ + $record = ['magic' => 'NONEDB4', 'kind' => 'generation'] + $identity + ['generation' => $generation, 'layout' => $layout]; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'generations/' . $generation . '/metadata', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write the metadata of generation {$generation} of database '{$dbname}'"); + } + } + + private function writeCurrentRecord($dbname, $databaseId, $generation){ + $record = ['magic' => 'NONEDB4', 'kind' => 'current', 'databaseId' => $databaseId, 'generation' => $generation]; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'CURRENT', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write CURRENT of database '{$dbname}'"); + } + $this->currentGenerations[$this->locks->getLockPath($dbname)] = $generation; + } + + private function writeFormatRecord($dbname, array $identity){ + $record = ['magic' => 'NONEDB4', 'kind' => 'database'] + $identity; + if(!$this->io->writeFileAtomic($this->dbHome($dbname) . 'FORMAT', \noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'durable')){ + throw new \RuntimeException("noneDB: cannot write FORMAT of database '{$dbname}'"); + } + $this->formatRecords[$this->locks->getLockPath($dbname)] = \noneDB\Internal\Storage\StoreRecord::openRecord(\noneDB\Internal\Storage\StoreRecord::sealRecord($record), 'database'); + $this->cache->markFileExists($this->dbHome($dbname) . 'FORMAT'); + } + + /** + * True when the database exists (FORMAT valid). Cached until the lock epoch changes. + */ + public function dbPresent($dbname){ + return $this->readFormatRecord($dbname) !== null; + } + + /** + * Entries of the database directory other than LOCK (and temp files) + * @return string[] names + */ + private function dbHomeEntries($dbname){ + $this->directoryListings++; // @test + $entries = []; + foreach(@scandir($this->dbHome($dbname)) ?: [] as $entry){ + if($entry !== '.' && $entry !== '..' && $entry !== 'LOCK' && preg_match('/\.tmp[0-9a-f]{8}$/', $entry) !== 1){ + $entries[] = $entry; + } + } + return $entries; + } + + /** + * Resolve the state of a database directory (§8.3) under its lock; called by prepareDb(). + * Interrupted creation and drop are completed (exclusive); a damaged directory is repaired only with + * proof, otherwise FORMAT_DAMAGED. Under $noWrites nothing is changed: the state is only reported. + */ + public function resolveDatabaseState($dbname, $path){ + $flags = $this->lockRegistry[$path]['flags']; + $format = $this->readFormatRecord($dbname); + if(!empty($flags['d'])){ + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname, $path){ + $this->dropDatabaseFiles($dbname, $path); + }); + return; + } + if(!empty($flags['c'])){ + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname, $path){ + if($this->readFormatRecord($dbname) === null){ + $this->undoCreation($dbname, $path); + return; + } + unset($this->lockRegistry[$path]['flags']['c']); + $this->locks->saveLockState($path); + }); + return; + } + if($format !== null){ + if(isset($this->currentGenerations[$path])){ + return; // CURRENT was validated under this epoch + } + $current = $this->readCurrentRecord($dbname, $format); + if($current === null){ + $this->forgetDatabaseLayout($dbname); // the next operation checks again, never the hot path + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: CURRENT is missing or invalid", \noneDB::FORMAT_DAMAGED); + } + $this->currentGenerations[$path] = $current['generation']; + return; + } + if(empty($this->dbHomeEntries($dbname))){ + return; // absent: only LOCK + } + // Damaged: FORMAT missing or invalid, other entries exist. Repair only with proof (§8.3). + if($this->noWrites > 0){ + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: FORMAT is missing or invalid", \noneDB::FORMAT_DAMAGED); + } + $this->requireExclusiveRecovery($dbname, $path, function() use ($dbname){ + if($this->readFormatRecord($dbname) !== null){ + return; + } + $current = \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'CURRENT'), 'current'); + $metadata = $current === null || !is_int($current['generation'] ?? null) ? null + : \noneDB\Internal\Storage\StoreRecord::openRecord(@file_get_contents($this->dbHome($dbname) . 'generations/' . $current['generation'] . '/metadata'), 'generation'); + if($metadata === null || ($metadata['databaseId'] ?? null) !== ($current['databaseId'] ?? false) + || ($metadata['generation'] ?? null) !== $current['generation'] || !$this->validIdentity($metadata, $dbname)){ + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: FORMAT is missing or invalid and cannot be repaired with proof", \noneDB::FORMAT_DAMAGED); + } + $this->writeFormatRecord($dbname, $this->identityOf($metadata)); + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'format-repaired', 'generation' => $current['generation']]); + }); + } + + /** + * Run a recovery step that needs the exclusive lock; under $noWrites report instead of changing + */ + private function requireExclusiveRecovery($dbname, $path, callable $fn){ + if($this->noWrites > 0){ + return; + } + if($this->lockRegistry[$path]['mode'] === LOCK_EX){ + $fn(); + return; + } + $this->locks->withLock($dbname, true, $fn, 'recovery'); + } + + /** + * Drop every cached view of a database's layout (FORMAT, CURRENT) + */ + public function forgetDatabaseLayout($dbname){ + $lock = $this->locks->getLockPath($dbname); + unset($this->currentGenerations[$lock], $this->formatRecords[$lock], $this->genPaths[$dbname]); + unset($this->staticFileExistsCache[$this->dbHome($dbname) . 'FORMAT']); + } + + /** + * Create a database (§8.1); caller holds its lock exclusively and STORE.LOCK shared. Durable mode + * (DURABILITY.md §3): "c" is synced before anything is created, every new entry with its directory, + * and the creation commits when FORMAT is durable. + * restore() (§10.3) passes the identity and $prepare($dir, 1), which builds and verifies generation 1 from + * a backup; when it fails the creation is undone at once (nothing of the database was committed). + */ + public function createDatabaseFiles($dbname, ?array $identity = null, ?callable $prepare = null){ + $path = $this->locks->getLockPath($dbname); + $this->lockRegistry[$path]['flags']['c'] = true; + $this->locks->saveLockState($path, true); + $this->recovery->fault('create:flagged', ['db' => $dbname]); // @test + // The database directory's entry in dbDir (LOCK's entry in it goes with the next sync of the directory) + $this->sync->requireSync(null, rtrim($this->dbDir, '/'), 'dir'); + $identity = $identity ?? ['storageFormat' => \noneDB::STORAGE_FORMAT, 'databaseId' => \noneDB\Internal\Storage\StoreRecord::uuid4(), 'name' => (string)$dbname, + 'shardSize' => (int)$this->shardSize, 'createdAt' => time(), 'formatOptions' => new \stdClass()]; + $this->forgetDatabaseLayout($dbname); + $this->removeTree($this->dbHome($dbname) . 'generations'); + if($prepare === null){ + $this->writeGenerationMetadata($dbname, 1, $identity, 'single'); + // Generation 1 is complete with its empty main segment + $dataPath = $this->dbHome($dbname) . 'generations/1/segments/main.jsonl'; + if(!$this->io->fsMkdir(dirname($dataPath)) || (!$this->io->fsTouch($dataPath) && !is_file($dataPath))){ + throw new \RuntimeException("noneDB: cannot create database file for '{$dbname}'"); + } + $this->sync->requireSync(null, dirname($dataPath), 'dir'); + } else { + $dir = $this->dbHome($dbname) . 'generations/1/'; + try { + $prepare($dir, 1); + } catch(\Throwable $e){ + try { + $this->undoCreation($dbname, $path); + } catch(\Throwable $ignored){ + // "c" stays: the next exclusive session undoes the creation + } + throw $e; + } + if($this->bufferHoldsFile($dir)){ + // The copied buffer's records are visible once FORMAT is: "b" is synced before CURRENT + $this->lockRegistry[$path]['flags']['b'] = true; + $this->locks->saveLockState($path); + } + } + $this->recovery->fault('create:generation', ['db' => $dbname]); // @test + $this->writeCurrentRecord($dbname, $identity['databaseId'], 1); + $this->recovery->fault('create:current', ['db' => $dbname]); // @test + $this->writeFormatRecord($dbname, $identity); + $this->recovery->fault('create:format', ['db' => $dbname]); // @test + unset($this->lockRegistry[$path]['flags']['c']); + $this->locks->saveLockState($path); + } + + /** + * Undo an interrupted creation (§8.3): nothing of it was committed (creation writes no data, a restore's + * copy is in its backup), so what it left goes; caller holds the lock exclusively + */ + private function undoCreation($dbname, $path){ + foreach(['CURRENT', 'generations'] as $entry){ + $this->removeTree($this->dbHome($dbname) . $entry); + } + $this->forgetDatabaseLayout($dbname); + unset($this->lockRegistry[$path]['flags']['c'], $this->lockRegistry[$path]['flags']['b']); + $this->locks->saveLockState($path); + } + + /** + * Drop a database (§8.2), or finish an interrupted drop; caller holds its lock exclusively. LOCK and + * the directory stay. $request: a dropDB() call, which reports a drop that can't remove FORMAT (its + * commit point) as not committed; finishing an interrupted one throws instead. + * Durable mode: "d" (with a new epoch) is synced before FORMAT goes, the drop commits when FORMAT's + * removal is durable, and "d" is cleared only once the other removals are durable too. + * @return array ['committed' => bool, 'removed' => int files, 'left' => string[] files not removed] + */ + public function dropDatabaseFiles($dbname, $path, $request = false){ + if(empty($this->lockRegistry[$path]['flags']['d'])){ + $before = $this->lockRegistry[$path]['flags']; + $this->lockRegistry[$path]['flags']['d'] = true; + unset($this->lockRegistry[$path]['flags']['b'], $this->lockRegistry[$path]['flags']['j']); + try { + $this->locks->bumpEpoch($dbname); // writes "d" with the new epoch + } catch(\RuntimeException $e){ + if($e->getCode() === \noneDB::DURABILITY_FAILED){ + $this->withdrawDrop($dbname, $path, $before, $e); + } + throw $e; + } + } + $this->recovery->fault('drop:flagged', ['db' => $dbname]); // @test + $home = $this->dbHome($dbname); + clearstatcache(true, $home . 'FORMAT'); + $removed = 0; + if(is_file($home . 'FORMAT')){ + if(!$this->io->fsUnlink($home . 'FORMAT')){ + // The commit point was not reached: the database stays as it was + if(!$request){ + throw new \RuntimeException("noneDB: cannot finish the interrupted drop of database '{$dbname}': its FORMAT can't be removed"); + } + $this->withdrawDrop($dbname, $path, $before, null); + return ['committed' => false, 'removed' => 0, 'left' => $this->dbHomeFiles($dbname)]; + } + $removed++; + $this->forgetDatabaseLayout($dbname); + // The commit point: FORMAT's removal is durable + $this->sync->requireSync(null, rtrim($home, '/'), 'dir', true); + } + $this->recovery->fault('drop:format', ['db' => $dbname]); // @test + foreach(['CURRENT', 'generations', 'recovery'] as $entry){ + $removed += $this->removeTree($home . $entry); + } + // "d" goes only once these removals are durable: else a power loss could leave the files without it + $durable = !$this->durable || $this->sync->syncDirectory(rtrim($home, '/')) === true; + $this->forgetDatabaseLayout($dbname); + $this->cache->invalidateDbCaches($dbname); + $this->recovery->fault('drop:cleaned', ['db' => $dbname]); // @test + $left = $this->dbHomeFiles($dbname); + if(empty($left) && $durable){ + unset($this->lockRegistry[$path]['flags']['d']); + $this->locks->saveLockState($path); + } + // Otherwise "d" stays: the database is gone, the next exclusive session removes what is left (§8.2) + return ['committed' => true, 'removed' => $removed, 'left' => $left]; + } + + /** + * A drop that did not reach its commit point: "d" must go (with FORMAT present it would finish the drop), + * durably in durable mode. If that can't be done either, the outcome is unknown. + * @throws \RuntimeException OUTCOME_UNKNOWN + */ + private function withdrawDrop($dbname, $path, array $flags, ?\Throwable $cause){ + $this->lockRegistry[$path]['flags'] = $flags; + try { + $this->locks->saveLockState($path, false, true); + } catch(\RuntimeException $e){ + throw new \RuntimeException("noneDB: the drop of database '{$dbname}' failed and its drop flag could not be withdrawn durably: the outcome is unknown, read the state again", \noneDB::OUTCOME_UNKNOWN, $cause ?? $e); + } + } + + /** + * Files of a database directory except LOCK, relative to it (a drop's leftovers) + * @return string[] + */ + private function dbHomeFiles($dbname){ + $this->directoryListings++; // @test + $home = $this->dbHome($dbname); + $files = []; + $walk = function($dir) use (&$walk, &$files, $home){ + foreach(@scandir($dir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(is_dir($dir . $entry)){ + $walk($dir . $entry . '/'); + } elseif($dir . $entry !== $home . 'LOCK'){ + $files[] = substr($dir . $entry, strlen($home)); + } + } + }; + $walk($home); + $walk = null; // the closure refers to itself: freed now, not by the garbage collector + sort($files); + return $files; + } + + /** + * Remove a file or a directory tree of a database (fsUnlink for files, rmdir for directories) + * @return int files removed + */ + private function removeTree($path){ + $path = rtrim($path, '/'); + clearstatcache(true, $path); + if(is_file($path) || is_link($path)){ + return $this->io->fsUnlink($path) ? 1 : 0; + } + if(!is_dir($path)){ + return 0; + } + $removed = 0; + foreach(@scandir($path) ?: [] as $entry){ + if($entry !== '.' && $entry !== '..'){ + $removed += $this->removeTree($path . '/' . $entry); + } + } + $this->io->fsRmdir($path); + return $removed; + } + + /** + * Generation switch (§5.2). $prepare($dir, $generation) builds the complete new generation in $dir; + * caller holds the database lock exclusively. Returns the new generation number. + */ + public function switchGeneration($dbname, callable $prepare){ + $path = $this->locks->getLockPath($dbname); + $format = $this->readFormatRecord($dbname); + if($format === null){ + throw new \RuntimeException("noneDB: database '{$dbname}' does not exist", \noneDB::FORMAT_DAMAGED); + } + $old = $this->generationNumber($dbname); + $new = $old + 1; + $newDir = $this->dbHome($dbname) . 'generations/' . $new . '/'; + $this->removeTree($newDir); // an unreferenced leftover of an earlier, interrupted PREPARE + // PREPARE: the callback builds and verifies generation $new; CURRENT still names $old. A failed + // PREPARE leaves nothing behind (the next switch would remove it anyway). + try { + $prepare($newDir, $new); + } catch(\Throwable $e){ + try { + $this->removeTree($newDir); + } catch(\Throwable $ignored){ + } + throw $e; + } + $this->recovery->fault('generation:prepared', ['db' => $dbname, 'from' => $old, 'to' => $new]); // @test + // INTENT: g=old>new and a new epoch, synced; from now on no process uses a cache of the old epoch. + // "b" when the new generation holds buffered records (a restore's), kept when the old one may + $this->lockRegistry[$path]['flags']['g'] = [$old, $new]; + if($this->bufferHoldsFile($newDir)){ + $this->lockRegistry[$path]['flags']['b'] = true; + } + $this->lockRegistry[$path]['epoch'] = bin2hex(random_bytes(4)); + $this->locks->saveLockState($path, true); + $this->recovery->fault('generation:intent', ['db' => $dbname, 'from' => $old, 'to' => $new]); // @test + // PUBLISH: the commit point (a failed directory sync after CURRENT's rename: unknown, "g" stays) + $this->writeCurrentRecord($dbname, $format['databaseId'], $new); + $this->recovery->fault('generation:published', ['db' => $dbname, 'from' => $old, 'to' => $new]); // @test + // CLEANUP + $oldDir = $this->dbHome($dbname) . 'generations/' . $old; + $this->removeTree($oldDir); + clearstatcache(true, $oldDir); + $cleaned = !is_dir($oldDir) && (!$this->durable || $this->sync->syncDirectory($this->dbHome($dbname) . 'generations') === true); + $this->recovery->fault('generation:cleaned', ['db' => $dbname, 'from' => $old, 'to' => $new]); // @test + // DONE ("g" stays while the old generation is there or its removal may not survive a power loss: + // the next session finishes the cleanup) + if($cleaned){ + unset($this->lockRegistry[$path]['flags']['g']); + if(!$this->bufferHoldsFile($newDir)){ + unset($this->lockRegistry[$path]['flags']['b']); + } + $this->locks->saveLockState($path); + } + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $new; + return $new; + } + + /** + * restore() over a damaged database (§10.3): $prepare($dir, X) builds and verifies generation X, one above + * the highest number under generations/; then a new epoch without "g" (and "b" when X holds buffered + * records), CURRENT names X (the commit point), and an invalid FORMAT is rewritten with $identity. The + * earlier generations stay: nothing proves which of them was committed. Caller holds the lock exclusively. + * @return int X + */ + public function replaceDamagedGeneration($dbname, array $identity, callable $prepare){ + $path = $this->locks->getLockPath($dbname); + $highest = 0; + foreach(@scandir($this->dbHome($dbname) . 'generations') ?: [] as $entry){ + if(ctype_digit((string)$entry)){ + $highest = max($highest, (int)$entry); + } + } + $new = $highest + 1; + $dir = $this->dbHome($dbname) . 'generations/' . $new . '/'; + try { + $prepare($dir, $new); + } catch(\Throwable $e){ + try { + $this->removeTree($dir); + } catch(\Throwable $ignored){ + } + throw $e; + } + unset($this->lockRegistry[$path]['flags']['g'], $this->lockRegistry[$path]['flags']['b']); + if($this->bufferHoldsFile($dir)){ + $this->lockRegistry[$path]['flags']['b'] = true; + } + $this->locks->bumpEpoch($dbname); + $this->recovery->fault('restore:intent', ['db' => $dbname, 'to' => $new]); // @test + $this->writeCurrentRecord($dbname, $identity['databaseId'], $new); + $this->recovery->fault('restore:published', ['db' => $dbname, 'to' => $new]); // @test + if($this->readFormatRecord($dbname) === null){ + $this->writeFormatRecord($dbname, $identity); + } + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $new; + return $new; + } + + /** + * State of a database for restore() (§10.3), once its lock resolved what it could: 'present' (FORMAT and + * CURRENT valid, no switch pending), 'absent' (no FORMAT, nothing but LOCK) or 'damaged' + */ + public function restoreTargetState($dbname){ + $format = $this->readFormatRecord($dbname); + if($format === null){ + return empty($this->dbHomeEntries($dbname)) ? 'absent' : 'damaged'; + } + $flags = $this->lockRegistry[$this->locks->getLockPath($dbname)]['flags'] ?? []; + return empty($flags['g']) && $this->readCurrentRecord($dbname, $format) !== null ? 'present' : 'damaged'; + } + + /** + * Run $fn reading generation $generation of a database as if it were the active one, under $identity, + * writing nothing: restore() verifies a generation before it is published (§10.3). Caller holds the lock + * exclusively. + */ + public function withGenerationView($dbname, $generation, array $identity, callable $fn){ + $lock = $this->locks->getLockPath($dbname); + $saved = [$this->currentGenerations[$lock] ?? null, $this->formatRecords[$lock] ?? null]; + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$lock] = $generation; + $this->formatRecords[$lock] = ['magic' => 'NONEDB4', 'kind' => 'database'] + $identity; + unset($this->genPaths[$dbname]); + $this->noWrites++; + try { + return $fn(); + } finally { + $this->noWrites--; + $this->cache->invalidateDbCaches($dbname); + unset($this->genPaths[$dbname], $this->currentGenerations[$lock], $this->formatRecords[$lock]); + if($saved[0] !== null){ + $this->currentGenerations[$lock] = $saved[0]; + } + if($saved[1] !== null){ + $this->formatRecords[$lock] = $saved[1]; + } + } + } + + /** + * True when a generation directory's buffer/ holds a pending or flushing file (§5.2: "b") + */ + private function bufferHoldsFile($dir){ + if(!is_dir($dir . 'buffer')){ + return false; + } + $this->directoryListings++; // @test + foreach(@scandir($dir . 'buffer') ?: [] as $entry){ + if($entry === 'pending.jsonl' || preg_match('/^flushing\.\d+\.jsonl$/', $entry) === 1){ + return true; + } + } + return false; + } + + /** + * Recovery of an interrupted generation switch (§5.2): g=N>X found; caller holds the lock exclusively + */ + public function recoverGenerationSwitch($dbname, $path){ + [$old, $new] = $this->lockRegistry[$path]['flags']['g']; + $this->forgetDatabaseLayout($dbname); + $format = $this->readFormatRecord($dbname); + $current = $format === null ? null : $this->readCurrentRecord($dbname, $format); + if($current === null || !in_array($current['generation'], [$old, $new], true)){ + $this->forgetDatabaseLayout($dbname); + throw new \RuntimeException("noneDB: database '{$dbname}' is damaged: a generation switch from {$old} to {$new} was interrupted and CURRENT is invalid; neither generation is removed (restore a backup, see docs/OPERATIONS.md)", \noneDB::FORMAT_DAMAGED); + } + $obsolete = $current['generation'] === $new ? $old : $new; + $obsoleteDir = $this->dbHome($dbname) . 'generations/' . $obsolete; + $this->removeTree($obsoleteDir); + clearstatcache(true, $obsoleteDir); + if(is_dir($obsoleteDir)){ + // CURRENT decides, so the data is consistent; what is left is reported by checkIntegrity() + // instead of being retried by every later operation + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'generation-cleanup-failed', 'committed' => $current['generation'], 'left' => $obsolete]); + } else { + $this->recoveryLog->logRecoveryEvent($dbname, ['action' => 'generation-recovered', 'committed' => $current['generation'], 'removed' => $obsolete]); + } + unset($this->lockRegistry[$path]['flags']['g']); + $this->locks->saveLockState($path); + $this->cache->invalidateDbCaches($dbname); + $this->currentGenerations[$path] = $current['generation']; + } +} diff --git a/src/Internal/Storage/RecordIO.php b/src/Internal/Storage/RecordIO.php new file mode 100644 index 0000000..af482f5 --- /dev/null +++ b/src/Internal/Storage/RecordIO.php @@ -0,0 +1,487 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->checkpoints, $this->io, $this->locks, $this->segments); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->appendMismatch = &$this->k->appendMismatch; + $this->indexCache = &$this->k->indexCache; + $this->fieldIndexCache = &$this->k->fieldIndexCache; + $this->cacheTouch = &$this->rt->cacheTouch; + $this->sessionLimit = &$this->rt->sessionLimit; + $this->cacheTick = &$this->rt->cacheTick; + $this->cacheGrew = &$this->rt->cacheGrew; + $this->verifiedIndexes = &$this->k->verifiedIndexes; + $this->writableIndexes = &$this->k->writableIndexes; + $this->lockRegistry = &$this->rt->lockRegistry; + $this->segmentReads = &$this->rt->segmentReads; // @test + $this->indexStats = &$this->rt->indexStats; // @test + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->checkpoints = null; + $this->io = null; + $this->locks = null; + $this->segments = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // RECORD I/O + // ========================================== + + /** A batch whose records lie within this many bytes is read with one call */ + private const SPAN = 262144; + + /** + * Batch read multiple JSONL records (one open, 64KB buffered sequential reads) + * @param string $path File path + * @param array $positions [key => offset, ...] + * @param array $lengths [key => length, ...] + * @param bool $mismatch Set to true when a line doesn't hold the expected record + * @return array [key => record, ...] + */ + public function readJsonlRecordsBatch($path, $positions, $lengths, &$mismatch = false, $held = null){ + $mismatch = false; + if(empty($positions)){ + return []; + } + + if(count($positions) === 1){ + // One record (a key lookup): its bytes only, in one call + $key = array_key_first($positions); + $length = (int)($lengths[$key] ?? 0); + if($length > self::SPAN){ + clearstatcache(true, $path); // a length the file doesn't hold is never asked of it, as large as it says + $length = $positions[$key] + $length > (int)@filesize($path) ? 0 : $length; + } + $json = $length <= 0 ? false : ($held !== null && $positions[$key] + $length <= strlen($held) + ? substr($held, $positions[$key], $length) : @file_get_contents($path, false, null, (int)$positions[$key], $length)); + $record = is_string($json) ? json_decode($json, true) : null; + if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ + $mismatch = true; + return []; + } + return [$key => $record]; + } + + $records = []; + $bufferSize = 65536; + $buffer = ''; + $bufferStart = -1; + $bufferEnd = -1; + + // Read in file order. Appends keep the index in that order; updated records moved to the end. + $inOrder = true; + $previous = -1; + foreach($positions as $position){ + if($position < $previous){ + $inOrder = false; + break; + } + $previous = $position; + } + if(!$inOrder){ + asort($positions, SORT_NUMERIC); + } + + // Records within one span of at most SPAN bytes (the keys of a small segment, a region): the span in + // one call. Otherwise records close together on average (a scan): 64 KB at a time; sparse ones: 64 KB + // when the next record ends within them, else the record alone (look-ahead in read order) + $first = (int)reset($positions); + $lastKey = array_key_last($positions); + $spanEnd = (int)$positions[$lastKey] + (int)($lengths[$lastKey] ?? 0); + $handle = null; + if($held !== null && $spanEnd <= strlen($held)){ + $buffer = $held; + $bufferStart = 0; + $bufferEnd = strlen($held); + } elseif($spanEnd - $first <= self::SPAN){ + $buffer = $spanEnd > $first ? @file_get_contents($path, false, null, $first, $spanEnd - $first) : ''; + if(!is_string($buffer)){ + $mismatch = true; + return []; + } + $bufferStart = $first; + $bufferEnd = $first + strlen($buffer); + } else { + $handle = @fopen($path, 'rb'); + if($handle === false){ + $mismatch = true; + return []; + } + $fileEnd = (int)(fstat($handle)['size'] ?? 0); + } + $dense = (int)end($positions) - $first <= ($bufferSize >> 2) * (count($positions) - 1); + $keys = $dense ? null : array_keys($positions); + $i = 0; + foreach($positions as $key => $offset){ + $offset = (int)$offset; + $length = (int)($lengths[$key] ?? 0); + if($length <= 0){ + $mismatch = true; + $i++; + continue; + } + + if($offset >= $bufferStart && ($offset + $length) <= $bufferEnd){ + $json = substr($buffer, $offset - $bufferStart, $length); + } elseif($handle === null || $offset + $length > $fileEnd){ + $mismatch = true; // the record ends past what was read: the data is shorter than its index + $i++; + continue; + } else { + if($dense){ + $near = true; + } else { + $next = $keys[$i + 1] ?? null; + $near = $next !== null && (int)$positions[$next] + (int)($lengths[$next] ?? 0) <= $offset + $bufferSize; + } + fseek($handle, $offset, SEEK_SET); + $buffer = (string)fread($handle, $near ? max($length, $bufferSize) : $length); + $bufferStart = $offset; + $bufferEnd = $offset + strlen($buffer); + $json = substr($buffer, 0, $length); + } + + if(!$dense){ + $i++; + } + $record = json_decode((string)$json, true); + if(!is_array($record) || !isset($record['key']) || $record['key'] !== (int)$key){ + $mismatch = true; + continue; + } + $records[$key] = $record; + } + + if($handle !== null){ + fclose($handle); + } + return $records; + } + + /** Records per batch of a read over a whole segment: the default shard size (such a segment is read at once) */ + public const BATCH = 10000; + + /** + * The records of a segment ($keys of it; every record when null) in batches of at most BATCH records, each + * [key => record] sorted by key, the batches in the order of $keys (every record: ascending). An operation over a + * segment of any size holds one batch (docs/CONFIGURATION.md "Memory of operations over many segments"). + * A segment that fits one batch: an array of that batch (nothing held, a read as it always was); a larger one: a + * generator. Iterated by reference, a caller that writes between batches sets its variable to null first, so + * the records it doesn't keep are released, not held through the write. + * @return iterable + */ + public function eachSegmentBatch($dbname, $shardId, $keys = null, $index = null){ + $index = $index ?? ($keys === null ? $this->checkpoints->decodedIndex($dbname, $shardId) : $this->checkpoints->readJsonlIndex($dbname, $shardId)); + if($index === null){ + return []; + } + if(($keys === null ? KeyIndex::count($index) : count($keys)) <= self::BATCH){ + $records = $this->readSegmentRecords($dbname, $shardId, $keys, $index); + return empty($records) ? [] : [$records]; + } + // Larger than a batch: the index's arrays once (a body unpacked in a few C-level calls, as a whole read + // does), then the keys a batch at a time + return $this->segmentBatches($dbname, $shardId, $keys, KeyIndex::materialize($index)); + } + + /** + * The batches of a segment larger than one (eachSegmentBatch()), yielded by reference. The writes of a caller + * between two batches touch only keys it was given: $index still holds the keys of the batches to come + * @return \Generator + */ + private function &segmentBatches($dbname, $shardId, $keys, array $index){ + if($keys === null){ + $keys = array_keys($index['p']); + sort($keys, SORT_NUMERIC); + } + foreach(array_chunk($keys, self::BATCH) as $chunk){ + $records = $this->readSegmentRecords($dbname, $shardId, $chunk, $index); + if(!empty($records)){ + yield $records; + } + } + } + + /** + * Read records of one segment, verified against their keys. A stale or corrupt index is + * rebuilt from the data file once. + * @param array|null $keys Keys to read (null = all) + * @param array|null $index The segment's key index the caller just read (null: read here) + * @return array [key => record] sorted by key + */ + public function readSegmentRecords($dbname, $shardId, $keys = null, $index = null){ + $this->segmentReads++; // @test + for($attempt = 0; $attempt < 2; $attempt++){ + $index = $attempt > 0 ? $this->checkpoints->repairSegmentIndex($dbname, $shardId, true) + : ($index ?? ($keys === null ? $this->checkpoints->decodedIndex($dbname, $shardId) : $this->checkpoints->readJsonlIndex($dbname, $shardId))); + if($index === null){ + return []; + } + if($keys !== null && $index['kb'] === ''){ + // A decoded index (the hot path): its arrays, no call per key; the lengths as they are + $positions = []; + foreach($keys as $key){ + if(isset($index['p'][$key])){ + $positions[$key] = $index['p'][$key]; + } + } + $lengths = $index['l']; + } else { + [$positions, $lengths] = KeyIndex::locate($index, $keys); + } + $dataPath = $index['dp'] ?? $this->segments->segDataPath($dbname, $shardId); + if($keys !== null && $index['kb'] !== '' && count($keys) >= 16){ + $this->checkpoints->noteLookups($dataPath, count($keys)); // bulk lookups (operations count in readJsonlIndex()) + } + if($keys === null && (int)$index['d'] === 0 && (int)$index['t'] === 0 && ($records = $this->liveLines($dataPath, $positions, (int)$index['s'])) !== null){ + $this->indexStats['liveReads']++; // @test + return $records; + } + $mismatch = false; + $held = null; + if(empty($index['few']) && ($this->sessionLimit ?? ($this->sessionLimit = \noneDB\Internal\Cache\CacheManager::cacheLimitBytes())) > 0){ + $held = $this->heldData($dbname, $dataPath, $index); + } + $records = $this->readJsonlRecordsBatch($dataPath, $positions, $lengths, $mismatch, $held); + if(!$mismatch){ + ksort($records, SORT_NUMERIC); + return $records; + } + unset($this->fieldIndexCache[$dataPath]); + } + throw new \RuntimeException("noneDB: data file of database '{$dbname}' is corrupt and could not be repaired"); + } + + /** + * Every record of a segment without garbage, sorted by key: its lines in file order are its records, so they are + * split at once and not cut out by position (about a tenth of such a read), 64 KB at a time. Each line is + * checked against the index as a read by position is. Null when one doesn't fit: the read by position decides. + * @param array $positions [key => offset] of every live key + * @param int $size The bytes the index covers + * @return array|null [key => record] + */ + private function liveLines($path, array $positions, $size){ + $this->indexStats['liveTries']++; // @test + $fp = @fopen($path, 'rb'); + if($fp === false){ + return null; + } + $records = []; + $carry = ''; + $at = 0; + $last = -1; + $sorted = true; + for($left = $size; $left > 0; $left -= strlen($chunk)){ + $chunk = (string)fread($fp, $left < 65536 ? $left : 65536); + if($chunk === ''){ + break; + } + $lines = explode("\n", $carry . $chunk); + $carry = array_pop($lines); + foreach($lines as $line){ + $record = json_decode($line, true); + $key = $record['key'] ?? null; + if(!is_int($key) || ($positions[$key] ?? -1) !== $at){ + fclose($fp); + return null; + } + $records[$key] = $record; + $sorted = $sorted && $key > $last; + $last = $key; + $at += strlen($line) + 1; + } + } + fclose($fp); + if(count($records) !== count($positions)){ + return null; + } + if(!$sorted){ + ksort($records, SORT_NUMERIC); + } + return $records; + } + + /** A record read from a data file costs an open, a seek and a read: about what reading this many of its bytes at once does */ + private const HELD_BYTES_PER_USE = 131072; + + /** + * The bytes of a data file in repeated use, held whole within the cache's budget: a record is then a substr(), + * not an open (§ "Memory" of docs/ARCHITECTURE.md). The caller asks for a database of many segments, in a process + * with a limit; the bytes are read once the operations have paid for it, under the shared lock: bytes no rollback + * takes back. They stay while the key index keeps its epoch (the bytes it covers can't change within one), newer + * lines come from the file, and they are dropped like any cached index. + * @return string|null + */ + private function heldData($dbname, $dataPath, array $index){ + $entry = $this->fieldIndexCache[$dataPath] ?? null; + if($entry !== null){ + if($entry['e'] === $index['e']){ + $this->cacheTouch[$dataPath] = ++$this->cacheTick; + $this->indexStats['heldReads']++; // @test + return $entry['raw']; + } + unset($this->fieldIndexCache[$dataPath]); // the file was replaced + } + $size = (int)$index['s']; + if($index['e'] === null || $index['uses'] < 2 || $index['uses'] * self::HELD_BYTES_PER_USE < $size + || ($this->lockRegistry[$this->locks->getLockPath($dbname)]['mode'] ?? null) !== LOCK_SH + || !\noneDB\Internal\Cache\CacheManager::decodeFits(2 * $size)){ + return null; + } + $raw = @file_get_contents($dataPath, false, null, 0, $size); + if(!is_string($raw) || strlen($raw) !== $size){ + return null; + } + $this->indexStats['dataHeld']++; // @test + $this->fieldIndexCache[$dataPath] = ['kind' => 'data', 'e' => $index['e'], 'raw' => $raw]; + $this->cacheTouch[$dataPath] = ++$this->cacheTick; + $this->cacheGrew = true; + return $raw; + } + + /** + * Append records ([key => record]) to a segment and point the index at them. + * Everything is encoded before anything is written, so an unencodable record writes nothing. + * @throws \InvalidArgumentException|\RuntimeException + */ + public function appendSegmentRecords($dbname, $shardId, array &$index, array $records){ + if(empty($records)){ + return; + } + $lines = $this->io->withExactFloats(function() use ($records){ + $lines = []; + foreach($records as $key => $record){ + unset($record['key']); + $record['key'] = (int)$key; + $lines[$key] = $this->io->encodeRecord($record); + } + return $lines; + }); + // After encoding: a record that can't be encoded leaves no journal entry behind + $this->locks->markIndexesPending($dbname, $shardId); + $buffer = implode("\n", $lines) . "\n"; + $dataPath = $this->segments->segDataPath($dbname, $shardId); + $offset = $this->io->appendToFile($dataPath, $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + $this->cache->markFileExists($dataPath); + $index['d'] += KeyIndex::setLines($index, $lines, $offset); // a replaced key's old line is garbage now + $index['tl'] = (int)$index['tl'] + count($lines); + } + + /** + * Append tombstones for deleted keys, so an index rebuilt from the data won't resurrect them + */ + public function appendTombstones($dbname, $shardId, array &$index, array $keys){ + if(empty($keys)){ + return; + } + $this->locks->markIndexesPending($dbname, $shardId); + $buffer = ''; + foreach($keys as $key){ + $buffer .= '{"__nonedb_deleted":' . (int)$key . "}\n"; + } + $offset = $this->io->appendToFile($this->segments->segDataPath($dbname, $shardId), $buffer, $dbname, $index['s'] ?? null, $shardId); + if($offset === false){ + $this->failedAppend($dbname, $shardId); + } + $index['s'] = $offset + strlen($buffer); + $index['d'] += KeyIndex::removeMany($index, $keys); // the old record lines + $index['t'] = (int)$index['t'] + count($keys); // tombstone lines + $index['tl'] = (int)$index['tl'] + count($keys); + } + + /** + * An append to a segment wrote nothing: throw. When the data file didn't have the size the index + * covers (it changed behind this process's back), the cached index is dropped as well. + * @throws \RuntimeException + */ + private function failedAppend($dbname, $shardId){ + if($this->appendMismatch){ + $path = $this->checkpoints->getJsonlIndexPath($dbname, $shardId); + unset($this->indexCache[$path], $this->verifiedIndexes[$path], $this->writableIndexes[$path]); + throw new \RuntimeException("noneDB: data file of database '{$dbname}' changed unexpectedly; nothing was written, its index is rebuilt on the next access"); + } + throw new \RuntimeException("noneDB: failed to write data of database '{$dbname}'"); + } + + /** + * Create an empty segment (data file + index) + * @param string $dbname + * @param int|null $shardId + * @return bool + */ + public function createJsonlDatabase($dbname, $shardId = null){ + $path = $this->segments->segDataPath($dbname, $shardId); + clearstatcache(true, $path); + if(!file_exists($path) && !$this->io->fsTouch($path)){ + throw new \RuntimeException("noneDB: cannot create data file of database '{$dbname}'"); + } + $this->cache->markFileExists($path); + $index = $this->checkpoints->newJsonlIndex(); + $index['e'] = bin2hex(random_bytes(4)); + $this->indexCache[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = $index; + $this->verifiedIndexes[\noneDB\Internal\Storage\Checkpoints::ckptPath($path)] = true; + // No checkpoint yet (ck -1): the session writes one when it ends, with the first records (until then + // the new segment replays from 0, which is the same index) + $this->lockRegistry[$this->locks->getLockPath($dbname)]['touched'][$shardId === null ? 'main' : $shardId] = $shardId; + return true; + } +} diff --git a/src/Internal/Storage/Segments.php b/src/Internal/Storage/Segments.php new file mode 100644 index 0000000..3f470f8 --- /dev/null +++ b/src/Internal/Storage/Segments.php @@ -0,0 +1,195 @@ +k = $k; + $this->rt = $k->rt; + unset($this->layout, $this->shardMeta); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->currentGenerations = &$this->k->currentGenerations; + $this->genPaths = &$this->k->genPaths; + $this->lockPaths = &$this->k->lockPaths; + $this->shardSize = &$this->k->shardSize; + $this->shardedCache = &$this->k->shardedCache; + $this->directoryListings = &$this->rt->directoryListings; // @test + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->layout = null; + $this->shardMeta = null; + $this->k = null; + $this->rt = null; + } + + + /** + * Path of the main data file (single layout) + */ + public function getDataPath($dbname){ + return $this->segDataPath($dbname, null); + } + // ========================================== + // STORAGE CORE + // ========================================== + // + // A database is one "segment" in the single layout (segments/main.jsonl + main.kix) or one segment per + // shard (segments/s{N}.jsonl + s{N}.kix) listed in shards.json, inside its active generation + // ({locator}.ndb4/generations/{N}/, docs/STORAGE_FORMAT.md §5). Segment ids are null (single) or the + // shard id. Every write runs under the database's exclusive lock (withLock), reads under a shared + // lock, so index read-modify-write cycles are never interleaved. + // + // Data files are append-only JSONL. Every record line carries its "key"; deletions append a + // tombstone line {"__nonedb_deleted":key}. The .kix checkpoint maps key => offset/length and stores + // "s", the data size it covers; loading replays the complete lines after it (Checkpoints, KeyIndex). + + /** + * Files of a directory whose path starts with $prefix (a full path: directory + start of the name) and + * whose remaining name matches $regex (if given), sorted. Not glob(): a path containing [ ] * ? is a + * pattern to glob() and matches nothing. + * @return string[] + */ + public function listFilesWithPrefix($prefix, $regex = null){ + $this->directoryListings++; // @test + $slash = strrpos($prefix, '/'); + $dir = substr($prefix, 0, $slash + 1); + $start = substr($prefix, $slash + 1); + $length = strlen($start); + $files = []; + foreach(@scandir($dir) ?: [] as $file){ + if($file !== '.' && $file !== '..' && strncmp($file, $start, $length) === 0 + && ($regex === null || preg_match($regex, substr($file, $length)) === 1)){ + $files[] = $dir . $file; + } + } + return $files; + } + + /** + * Data file of a shard + */ + public function getShardPath($dbname, $shardId){ + return $this->segDataPath($dbname, (int)$shardId); + } + + /** + * Data file of a segment (null = single layout) + */ + public function segDataPath($dbname, $shardId){ + $k = $shardId ?? -1; + // Hot path: the cached path while its generation is still the active one (no call) + $paths = $this->genPaths[$dbname] ?? null; + if(isset($paths['seg'][$k]) && $paths['gen'] === ($this->currentGenerations[$this->lockPaths[$dbname]] ?? null)){ + return $paths['seg'][$k]; + } + $dir = $this->layout->genDir($dbname); + return $this->genPaths[$dbname]['seg'][$k] ?? ($this->genPaths[$dbname]['seg'][$k] = $dir . 'segments/' . \noneDB\Internal\Storage\Layout::segmentName($shardId) . '.jsonl'); + } + + /** + * Check if a database is sharded + * @param string $dbname + * @return bool + */ + public function isSharded($dbname){ + $metaPath = $this->layout->genPath($dbname, 'shards.json'); // ShardMeta::getMetaPath(), without the hop + if(isset($this->shardedCache[$metaPath])){ + return $this->shardedCache[$metaPath]; + } + + clearstatcache(true, $metaPath); + $result = file_exists($metaPath); + $this->shardedCache[$metaPath] = $result; + return $result; + } + + /** + * True when the database exists (STORAGE_FORMAT.md §8.3: FORMAT valid) + */ + public function dbExists($dbname){ + return $this->layout->dbPresent($dbname); + } + + /** + * Records per shard of a database: fixed in FORMAT when the database was created + */ + public function shardSizeOf($dbname){ + $format = $this->layout->readFormatRecord($dbname); + return $format !== null ? (int)$format['shardSize'] : (int)$this->shardSize; + } + + /** + * Segment ids of a database: [null] when not sharded, the shard ids otherwise, [] if it doesn't exist + */ + public function listSegments($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + if($meta === null){ + throw new \RuntimeException("noneDB: metadata of sharded database '{$dbname}' is unreadable"); + } + return array_map('intval', array_column($meta['shards'], 'id')); + } + return $this->layout->dbPresent($dbname) ? [null] : []; + } + + /** Segments of a database without listing them (0: no database) */ + public function segmentCount($dbname){ + if($this->isSharded($dbname)){ + $meta = $this->shardMeta->getCachedMeta($dbname); + return $meta === null ? 0 : count($meta['shards']); + } + return $this->layout->dbPresent($dbname) ? 1 : 0; + } + + /** A database of at most this many segments decodes the indexes it uses at once (every segment is hot) */ + public const FEW_SEGMENTS = 16; + + /** + * Segment that holds a key, or false if that shard doesn't exist + * @return int|null|false + */ + public function segmentForKey($dbname, $key){ + if(!$this->isSharded($dbname)){ + return null; + } + $shardId = intdiv($key, $this->shardSizeOf($dbname)); + return in_array($shardId, $this->listSegments($dbname), true) ? $shardId : false; + } +} diff --git a/src/Internal/Storage/StoreRecord.php b/src/Internal/Storage/StoreRecord.php new file mode 100644 index 0000000..8b1eb24 --- /dev/null +++ b/src/Internal/Storage/StoreRecord.php @@ -0,0 +1,508 @@ +k = $k; + $this->rt = $k->rt; + unset($this->cache, $this->config, $this->io, $this->locks, $this->recovery, $this->sync); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->dbDir = &$this->k->dbDir; + $this->hashCache = &$this->k->hashCache; + $this->directoryListings = &$this->rt->directoryListings; // @test + $this->maintenanceDepth = &$this->rt->maintenanceDepth; + $this->metrics = &$this->rt->metrics; + $this->noWrites = &$this->rt->noWrites; + $this->storeLocks = &$this->rt->storeLocks; + $this->storeState = &$this->rt->storeState; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->cache = null; + $this->config = null; + $this->io = null; + $this->locks = null; + $this->recovery = null; + $this->sync = null; + $this->k = null; + $this->rt = null; + } + + + /** + * keyCheck of the configured key: identifies it in STORE without revealing it + */ + public function keyCheck(){ + return substr(hash_hmac('sha256', 'nonedb4/key', (string)$this->config->getSecretKey()), 0, 32); + } + + public static function keyCheckOf($key){ + return substr(hash_hmac('sha256', 'nonedb4/key', (string)$key), 0, 32); + } + + /** + * Directory name of a database for the configured key (stable, unpredictable without the key) + */ + public function locator($dbname){ + $cacheKey = $this->cache->getKeyFingerprint() . '|' . $dbname; + return $this->hashCache[$cacheKey] ?? ($this->hashCache[$cacheKey] = self::locatorOf($dbname, (string)$this->config->getSecretKey())); + } + + public static function locatorOf($dbname, $key){ + return substr(hash_hmac('sha256', 'nonedb4/db/' . $dbname, (string)$key), 0, 32); + } + + /** + * Encode a metadata record with its checksum (§2) + */ + public static function sealRecord(array $record){ + unset($record['checksum']); + $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION; + $json = json_encode($record, $flags); + if($json === false){ + throw new \RuntimeException('noneDB: cannot encode a metadata record'); + } + $record['checksum'] = hash('sha256', $json . "\n"); + return json_encode($record, $flags) . "\n"; + } + + /** + * Decode and verify a metadata record: null when it is not a valid record of $kind. The checksum is + * checked on the bytes as written (the record without its last member "checksum"), not on a + * re-encoding, which could differ (an empty object decodes to an empty array). + */ + public static function openRecord($raw, $kind){ + if(!is_string($raw) || $raw === '' || substr($raw, -1) !== "\n" + || preg_match('/,"checksum":"([0-9a-f]{64})"\}\n\z/', $raw, $m, PREG_OFFSET_CAPTURE) !== 1){ + return null; + } + $body = substr($raw, 0, $m[0][1]) . "}\n"; + if(!hash_equals(hash('sha256', $body), $m[1][0])){ + return null; + } + $record = json_decode(substr($raw, 0, -1), true); + if(!is_array($record) || ($record['magic'] ?? null) !== 'NONEDB4' || ($record['kind'] ?? null) !== $kind){ + return null; + } + unset($record['checksum']); + return $record; + } + + /** + * One STORE copy: [exists, record or null, raw bytes or null] + */ + public function readStoreCopy($name){ + $path = $this->dbDir . $name; + $raw = @file_get_contents($path); + if($raw === false){ + clearstatcache(true, $path); + return [is_file($path), null, null]; + } + return [true, $this->validStoreRecord($raw), $raw]; + } + + private function validStoreRecord($raw){ + $record = self::openRecord($raw, 'store'); + $valid = $record !== null && ($record['storageFormat'] ?? null) === \noneDB::STORAGE_FORMAT + && is_string($record['storeId'] ?? null) && is_int($record['seq'] ?? null) && $record['seq'] >= 1 + && is_array($record['keys'] ?? null) && array_is_list($record['keys']); + if($valid){ + $keys = $record['keys']; + foreach($keys as $keyCheck){ + if(!is_string($keyCheck) || preg_match('/^[0-9a-f]{32}$/', $keyCheck) !== 1){ + $valid = false; + } + } + $sorted = array_values(array_unique($keys)); + sort($sorted, SORT_STRING); + $valid = $valid && $sorted === $keys; + } + return $valid ? $record : null; + } + + /** + * The store record by §3.1, null when there is no store. One valid copy repairs the other (under + * STORE.LOCK; not while this process holds it shared: the next open repairs it); two valid copies that + * differ, or none valid, are FORMAT_DAMAGED. + */ + public function loadStoreRecord(){ + [$exists1, $copy1, $raw1] = $this->readStoreCopy('STORE'); + $raw2 = $raw1 === null ? false : @file_get_contents($this->dbDir . 'STORE.2'); + if($raw1 !== null && $raw2 === $raw1){ + $copy2 = $copy1; // the same bytes: one check + $exists2 = true; + } else { + [$exists2, $copy2] = $raw2 === false ? $this->readStoreCopy('STORE.2') : [true, $this->validStoreRecord($raw2)]; + } + if(!$exists1 && !$exists2){ + return null; + } + if($copy1 !== null && $copy2 !== null){ + if($copy1 === $copy2){ + return $copy1; + } + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged (its two copies differ)", \noneDB::FORMAT_DAMAGED); + } + $valid = $copy1 ?? $copy2; + if($valid === null){ + throw new \RuntimeException("noneDB: the store record in {$this->dbDir} is damaged; see repairStore()", \noneDB::FORMAT_DAMAGED); + } + $held = $this->storeLocks[$this->dbDir]['mode'] ?? null; + if($this->noWrites === 0 && $held !== LOCK_SH){ + // The copies are identical outside maintenance (§3.1): the valid one restores the other + $this->withStoreLock(LOCK_EX, function() use ($valid){ + [, $again1] = $this->readStoreCopy('STORE'); + [, $again2] = $this->readStoreCopy('STORE.2'); + if($again1 === null || $again2 === null){ + $this->writeStoreCopies($valid); + $this->logStoreEvent(['action' => 'store-copy-repaired', 'seq' => $valid['seq']]); + } + }); + } + return $valid; + } + + /** + * Write both STORE copies (STORE.2 first) by atomic replacement; caller holds STORE.LOCK exclusively + */ + public function writeStoreCopies(array $record){ + $content = self::sealRecord($record); + foreach(['STORE.2', 'STORE'] as $name){ + $this->recovery->fault('store:write', ['file' => $name]); // @test + if(!$this->writeStoreFile($this->dbDir . $name, $content)){ + throw new \RuntimeException("noneDB: cannot write the store record {$name} in {$this->dbDir}"); + } + } + } + + /** + * Atomic replacement of a store-level file (STORE copies, journal): temp + rename, synced in durable + * mode (temp file, then the directory after the rename: the commit point of the file). + * Store files belong to no database: STORE.LOCK, not a database lock, protects them. + * @return bool False when the file could not be written (nothing changed) + * @throws \RuntimeException DURABILITY_FAILED (temp file sync, nothing changed), OUTCOME_UNKNOWN + * (directory sync after the rename) + */ + public function writeStoreFile($path, $content){ + $this->recovery->removeStaleTemp($path); // a crashed replacement's temp (§2): STORE.LOCK is held exclusively + $tmp = $path . '.tmp' . bin2hex(random_bytes(4)); + $fp = @fopen($tmp, 'xb'); + if($fp === false){ + return false; + } + $this->io->setPermissions($tmp); + $ok = fwrite($fp, $content) === strlen($content) && fflush($fp); + $synced = $ok && $this->sync->trySync($fp, $tmp); + fclose($fp); + if(!$ok || !$synced){ + @unlink($tmp); + if($ok){ + throw $this->sync->failure('file', $path, false); + } + return false; + } + $this->recovery->fault('store:renaming', ['file' => basename($path)]); // @test + if(!$this->sync->renameWithRetry($tmp, $path)){ + @unlink($tmp); + return false; + } + clearstatcache(true, $path); + $this->sync->requireSync(null, dirname($path), 'dir', true); + return true; + } + + /** + * Store-level recovery events (STORE copy repairs, manual rebuilds): .nonedb-maintenance/log.jsonl + */ + public function logStoreEvent(array $event){ + $dir = $this->dbDir . '.nonedb-maintenance'; + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + return; + } + $created = !is_file($dir . '/log.jsonl'); + if(@file_put_contents($dir . '/log.jsonl', json_encode(['t' => time()] + $event, JSON_UNESCAPED_SLASHES) . "\n", FILE_APPEND | LOCK_EX) !== false){ + if($created){ + $this->io->setPermissions($dir . '/log.jsonl'); + } + $this->metrics['recoveryEvents']++; + $this->metrics['recoveryActions'][$event['action']] = ($this->metrics['recoveryActions'][$event['action']] ?? 0) + 1; + } + } + + /** + * Run $fn holding STORE.LOCK (re-entrant within the process; a shared hold is never upgraded) + */ + public function withStoreLock($mode, callable $fn, $nonBlocking = false){ + $dir = $this->dbDir; + if(isset($this->storeLocks[$dir])){ + if($mode === LOCK_EX && $this->storeLocks[$dir]['mode'] !== LOCK_EX){ + throw new \LogicException('noneDB: STORE.LOCK is held shared and cannot be upgraded'); + } + $this->storeLocks[$dir]['depth']++; + try { + return $fn(); + } finally { + $this->storeLocks[$dir]['depth']--; + } + } + if(!is_dir($dir) && !$this->io->makeDirectory($dir)){ + throw new \RuntimeException("noneDB: cannot create database directory {$dir}"); + } + $created = !is_file($dir . 'STORE.LOCK'); + $fp = @fopen($dir . 'STORE.LOCK', 'c+be'); + if($fp !== false && $created){ + $this->io->setPermissions($dir . 'STORE.LOCK'); + } + if($fp === false){ + throw new \RuntimeException("noneDB: cannot open the store lock in {$dir}", \noneDB::LOCK_FAILED); + } + try { + if($nonBlocking){ + if(!flock($fp, $mode | LOCK_NB)){ + throw new \RuntimeException("noneDB: the store in {$dir} is in use", \noneDB::MAINTENANCE_BUSY); + } + } else { + $this->locks->acquireFlock($fp, $mode, 'STORE'); + } + } catch(\Throwable $e){ + fclose($fp); + throw $e; + } + $this->storeLocks[$dir] = ['fp' => $fp, 'mode' => $mode, 'depth' => 1]; + try { + return $fn(); + } finally { + unset($this->storeLocks[$dir]); + flock($fp, LOCK_UN); + fclose($fp); + } + } + + /** + * Entries of dbDir that are not noneDB 4 objects (§3.4). $withStore: the allow-list of a store that + * has a STORE record; without one only the files an empty location may hold are allowed (§3.2). + * @return string[] + */ + private function foreignEntries($withStore){ + $this->directoryListings++; // @test + $foreign = []; + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if($entry === '.' || $entry === '..'){ + continue; + } + if(in_array($entry, ['.htaccess', 'index.html', '.gitkeep', '.gitignore', '.DS_Store', 'STORE.LOCK'], true) + || preg_match('/^STORE(\.2)?\.tmp[0-9a-f]{8}$/', $entry) === 1){ + continue; + } + if($withStore && (in_array($entry, ['STORE', 'STORE.2', '.nonedb-maintenance', '.nonedb-retained'], true) + || (preg_match('/^[0-9a-f]{32}\.ndb4$/', $entry) === 1 && is_dir($this->dbDir . $entry) && !is_link($this->dbDir . $entry)))){ + continue; // a database directory; a link or a file of that name is foreign (§3.4) + } + $foreign[] = $entry; + } + return $foreign; + } + + /** + * MAINTENANCE_IN_PROGRESS while a maintenance journal is open (except in the process running it) + */ + public function checkMaintenanceJournal(){ + if($this->maintenanceDepth > 0){ + return; + } + $journal = $this->dbDir . '.nonedb-maintenance/journal.json'; + clearstatcache(true, $journal); + if(is_file($journal)){ + throw new \RuntimeException("noneDB: maintenance of the store in {$this->dbDir} is in progress or was interrupted; run it again to finish", \noneDB::MAINTENANCE_IN_PROGRESS); + } + } + + /** + * Open the store for this key (§3.2). $create: create the store when the location is empty. + * @return bool false when there is no store (and $create is false): nothing exists yet + */ + public function openStore($create){ + $dir = $this->dbDir; + $keyCheck = $this->keyCheck(); + if(isset($this->storeState[$dir][$keyCheck])){ + return true; + } + $this->checkMaintenanceJournal(); + $record = $this->loadStoreRecord(); + if($record === null && !empty($this->foreignEntries(false))){ + // The entries may belong to a store another process created while this one looked: decided + // again under STORE.LOCK, which every store creation holds + $record = $this->storeRecordUnderExistingLock(); + if($record === null){ + throw $this->missingStoreError($this->foreignEntries(false)); + } + } + if($record === null){ + if(!$create || $this->noWrites > 0){ + return false; + } + $this->withStoreLock(LOCK_EX, function() use ($keyCheck, &$record){ + $record = $this->loadStoreRecord(); + if($record !== null){ + return; + } + $foreign = $this->foreignEntries(false); + if(!empty($foreign)){ + throw $this->missingStoreError($foreign); + } + foreach(@scandir($this->dbDir) ?: [] as $entry){ + if(preg_match('/^STORE(\.2)?\.tmp[0-9a-f]{8}$/', $entry) === 1){ + @unlink($this->dbDir . $entry); + } + } + $record = ['magic' => 'NONEDB4', 'kind' => 'store', 'storageFormat' => \noneDB::STORAGE_FORMAT, + 'storeId' => self::uuid4(), 'seq' => 1, 'keys' => [$keyCheck]]; + $this->recovery->fault('store:create', []); // @test + $this->writeStoreCopies($record); + foreach(['.htaccess' => "Require all denied\nDeny from all\n", 'index.html' => ''] as $name => $content){ + if(@file_put_contents($this->dbDir . $name, $content) !== false){ + $this->io->setPermissions($this->dbDir . $name); + } + } + }); + } + if(!in_array($keyCheck, $record['keys'], true)){ + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$dir}", \noneDB::KEY_NOT_REGISTERED); + } + $this->storeState[$dir][$keyCheck] = true; + return true; + } + + /** + * The store record read under STORE.LOCK held shared, when STORE.LOCK exists (it is never created + * here: a refused location stays unchanged); null without a store + */ + private function storeRecordUnderExistingLock(){ + $dir = $this->dbDir; + if(isset($this->storeLocks[$dir])){ + return $this->loadStoreRecord(); + } + $fp = @fopen($dir . 'STORE.LOCK', 'rb'); + if($fp === false){ + return null; + } + try { + $this->locks->acquireFlock($fp, LOCK_SH, 'STORE'); + $this->storeLocks[$dir] = ['fp' => $fp, 'mode' => LOCK_SH, 'depth' => 1]; + try { + return $this->loadStoreRecord(); + } finally { + unset($this->storeLocks[$dir]); + flock($fp, LOCK_UN); + } + } finally { + fclose($fp); + } + } + + /** + * The missing-database path (§3.3): run $fn holding STORE.LOCK shared after the store was re-read, + * the key re-checked and dbDir listed for foreign entries. $fn decides with that view. + */ + public function withStoreRevalidated($create, callable $fn){ + if(!$this->storeExists() && !$this->openStore($create)){ + return $fn(); // no store and nothing in dbDir: nothing exists + } + return $this->withStoreLock(LOCK_SH, function() use ($fn){ + // Checked again under the lock: a maintenance may have changed the store since it was opened + $this->checkMaintenanceJournal(); + $record = $this->loadStoreRecord(); + if($record === null || !in_array($this->keyCheck(), $record['keys'], true)){ + unset($this->storeState[$this->dbDir]); + throw new \RuntimeException("noneDB: the configured secretKey is not registered for the store in {$this->dbDir}", \noneDB::KEY_NOT_REGISTERED); + } + $foreign = $this->foreignEntries(true); + if(!empty($foreign)){ + throw $this->unsupportedFormat($foreign); + } + return $fn(); + }); + } + + /** + * No STORE copy but entries: noneDB 4 objects without their store record are a damaged store, + * anything else is not a noneDB 4 store + */ + private function missingStoreError(array $foreign){ + foreach($foreign as $entry){ + if(preg_match('/^[0-9a-f]{32}\.ndb4$/', $entry) === 1 || $entry === '.nonedb-maintenance' || $entry === 'STORE' || $entry === 'STORE.2'){ + return new \RuntimeException("noneDB: the store record in {$this->dbDir} is missing; see repairStore()", \noneDB::FORMAT_DAMAGED); + } + } + return $this->unsupportedFormat($foreign); + } + + /** + * True when this process opened the store for its key (it exists and the key is registered) + */ + public function storeExists(){ + return isset($this->storeState[$this->dbDir][$this->keyCheck()]); + } + + private function unsupportedFormat(array $foreign){ + $shown = array_slice($foreign, 0, 5); + return new \RuntimeException("noneDB: {$this->dbDir} contains files that are not noneDB 4 objects (" + . implode(', ', $shown) . (count($foreign) > 5 ? ', ...' : '') . "): no database is created or answered as missing while they are there (docs/STORAGE_FORMAT.md §3.4)", + \noneDB::UNSUPPORTED_FORMAT); + } + + public static function uuid4(){ + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } +} diff --git a/src/Internal/Storage/Writer.php b/src/Internal/Storage/Writer.php new file mode 100644 index 0000000..e2b27db --- /dev/null +++ b/src/Internal/Storage/Writer.php @@ -0,0 +1,204 @@ +k = $k; + $this->rt = $k->rt; + unset($this->checkpoints, $this->compaction, $this->db, $this->findFilter, $this->layoutSwitch, $this->records, $this->scope, $this->segments, $this->shardMeta, $this->shardRouter); + } + + /** + * Bind what this service uses: the instance's values and the process-wide ones by reference, the + * other services by handle (Kernel calls it once every service exists) + */ + public function bind(): void { + $this->autoCreateDB = &$this->k->autoCreateDB; + $this->shardingEnabled = &$this->k->shardingEnabled; + } + + /** + * A handle to another service, bound on first use + */ + public function __get(string $name) { + return $this->$name = $this->k->$name; + } + + /** + * Drop the handles to the kernel and the other services (the instance is gone) + */ + public function unbind(): void { + $this->checkpoints = null; + $this->compaction = null; + $this->db = null; + $this->findFilter = null; + $this->layoutSwitch = null; + $this->records = null; + $this->scope = null; + $this->segments = null; + $this->shardMeta = null; + $this->shardRouter = null; + $this->k = null; + $this->rt = null; + } + + // ========================================== + // WRITE PATHS (callers hold the exclusive lock) + // ========================================== + + /** + * Insert validated records. Assigns keys, writes data + indexes, updates field/spatial indexes + * and switches the database to the sharded layout once it has assigned shardSize keys (shardingEnabled). + * @param array $items Records without "key" + * @return array ['n' => int] or ['n' => 0, 'error' => string] + */ + public function insertRecords($dbname, array $items, $startKey = null){ + if(empty($items)){ + return ["n" => 0]; + } + if(!$this->segments->dbExists($dbname)){ + if(!$this->autoCreateDB){ + return ["n" => 0, "error" => "Database does not exist"]; + } + $this->db->createDBUnlocked($dbname); + } + + // With a start key the records get keys startKey, startKey+1, ... and keys that are already + // stored are skipped, so replaying the same insert after a crash never duplicates anything. + $keyed = null; + $endKey = null; + if($startKey !== null){ + $keyed = []; + foreach(array_values($items) as $i => $item){ + $keyed[(int)$startKey + $i] = $item; + } + $endKey = (int)$startKey + count($items); + foreach(array_keys($this->findFilter->selectRecords($dbname, ['key' => array_keys($keyed)])) as $existing){ + unset($keyed[$existing]); + } + } + + if($this->segments->isSharded($dbname)){ + $this->shardRouter->insertShardedRecords($dbname, $items, $keyed, $endKey); + return ["n" => count($items)]; + } + + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + if($index === null){ + $this->records->createJsonlDatabase($dbname, null); + $index = $this->checkpoints->writableSegmentIndex($dbname, null); + } + + if($keyed !== null){ + $records = $keyed; + $index['n'] = max((int)$index['n'], $endKey); + } else { + $records = []; + foreach($items as $item){ + $records[(int)$index['n']] = $item; + $index['n']++; + } + } + $this->records->appendSegmentRecords($dbname, null, $index, $records); + $this->checkpoints->storeSegmentIndex($dbname, null, $index); + + if($this->shardingEnabled && $index['n'] >= $this->segments->shardSizeOf($dbname)){ + // Replaces the data file: after this write committed + $this->scope->runIndependent($dbname, 'convertToSharded', function() use ($dbname){ + return $this->layoutSwitch->convertToSharded($dbname); + }); + } + return ["n" => count($items)]; + } + + /** + * Replace records ([key => full new record]) in one segment + * @return int Number of records written + */ + public function updateSegmentRecords($dbname, $shardId, array $newRecords){ + if(empty($newRecords)){ + return 0; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + foreach(array_keys($newRecords) as $key){ + if(!KeyIndex::has($index, $key)){ + unset($newRecords[$key]); + } + } + if(empty($newRecords)){ + $this->checkpoints->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $this->records->appendSegmentRecords($dbname, $shardId, $index, $newRecords); // counts the replaced lines + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + $this->compaction->deferCompaction($dbname, $shardId, $index); + return count($newRecords); + } + + /** + * Delete records ([key => record]) from one segment + * @return int Number of records deleted + */ + public function deleteSegmentRecords($dbname, $shardId, array $oldRecords){ + if(empty($oldRecords)){ + return 0; + } + $index = $this->checkpoints->writableSegmentIndex($dbname, $shardId); + if($index['kb'] === ''){ + $oldRecords = array_intersect_key($oldRecords, $index['p']); // a decoded index: one call + } else { + foreach(array_keys($oldRecords) as $key){ + if(!KeyIndex::has($index, $key)){ + unset($oldRecords[$key]); + } + } + } + if(empty($oldRecords)){ + $this->checkpoints->releaseSegmentIndex($dbname, $shardId, $index); + return 0; + } + $keys = array_keys($oldRecords); + $this->records->appendTombstones($dbname, $shardId, $index, $keys); // drops the keys, counts the lines + $this->checkpoints->storeSegmentIndex($dbname, $shardId, $index); + + if($shardId !== null){ + $meta = $this->shardMeta->getCachedMeta($dbname, true); + foreach($meta['shards'] as $i => $shard){ + if((int)$shard['id'] === $shardId){ + $meta['shards'][$i]['count'] = max(0, $meta['shards'][$i]['count'] - count($keys)); + $meta['shards'][$i]['deleted'] += count($keys); + } + } + $meta['totalRecords'] = max(0, (int)$meta['totalRecords'] - count($keys)); + $meta['deletedCount'] = (int)($meta['deletedCount'] ?? 0) + count($keys); + $this->shardMeta->writeMeta($dbname, $meta); + } + + $this->compaction->deferCompaction($dbname, $shardId, $index); + return count($keys); + } +} diff --git a/src/header.php b/src/header.php new file mode 100644 index 0000000..3cba0ee --- /dev/null +++ b/src/header.php @@ -0,0 +1,13 @@ +/.php holds part of a +// class body in a trait. +return [ + 'header' => 'header.php', + // Namespaced internal classes (noneDB\Internal\...), one per file + 'files' => [ + 'Internal/Cache/Runtime.php', + 'Internal/Core/Kernel.php', + 'Internal/Cache/CacheManager.php', + 'Internal/Concurrency/LockManager.php', + 'Internal/Concurrency/Snapshot.php', + 'Internal/Config/Configuration.php', + 'Internal/Core/Crud.php', + 'Internal/Core/DatabaseOps.php', + 'Internal/Diagnostics/Integrity.php', + 'Internal/Diagnostics/Metrics.php', + 'Internal/Durability/FileIO.php', + 'Internal/Durability/Sync.php', + 'Internal/Index/FieldIndex.php', + 'Internal/Index/PagedFile.php', + 'Internal/Index/FieldIndexApi.php', + 'Internal/Index/SecondaryFiles.php', + 'Internal/Index/ShardFilters.php', + 'Internal/Index/Spatial/GeoJson.php', + 'Internal/Index/Spatial/Geometry.php', + 'Internal/Index/Spatial/RTree.php', + 'Internal/Index/Spatial/SpatialIndexApi.php', + 'Internal/Maintenance/Compaction.php', + 'Internal/Maintenance/BackupRestore.php', + 'Internal/Maintenance/KeyMaintenance.php', + 'Internal/Maintenance/Maintenance.php', + 'Internal/Query/Aggregation.php', + 'Internal/Query/FindFilter.php', + 'Internal/Query/Planner.php', + 'Internal/Query/QueryMethods.php', + 'Internal/Query/SpatialQueries.php', + 'Internal/Query/Utility.php', + 'Internal/Query/Values.php', + 'Internal/Recovery/Recovery.php', + 'Internal/Recovery/RecoveryLog.php', + 'Internal/Recovery/RollbackScope.php', + 'Internal/Sharding/LayoutSwitch.php', + 'Internal/Sharding/ShardInfo.php', + 'Internal/Sharding/ShardMeta.php', + 'Internal/Sharding/ShardRouter.php', + 'Internal/Storage/Buffer.php', + 'Internal/Storage/Checkpoints.php', + 'Internal/Storage/KeyIndex.php', + 'Internal/Storage/Layout.php', + 'Internal/Storage/RecordIO.php', + 'Internal/Storage/Segments.php', + 'Internal/Storage/StoreRecord.php', + 'Internal/Storage/Writer.php', + ], + // Global classes: the public API (noneDB, noneDBQuery), built from their modules + 'classes' => [ + [ + 'name' => 'noneDB', + 'before' => '', + 'modules' => [ + 'Facade', + ], + ], + [ + 'name' => 'noneDBQuery', + 'before' => ' +/** + * Query Builder for noneDB - Fluent Interface + * + * Allows method chaining for queries: + * $db->query("users")->where(["active" => true])->sort("name")->limit(10)->get(); + */ +', + 'modules' => [ + 'Core', + 'Filters', + 'Chain', + 'SpatialChain', + 'Execution', + 'Aggregates', + 'Writes', + ], + ], + ], +]; diff --git a/src/noneDB/Facade.php b/src/noneDB/Facade.php new file mode 100644 index 0000000..633c1e8 --- /dev/null +++ b/src/noneDB/Facade.php @@ -0,0 +1,907 @@ + 'durable' on a platform that can't provide it (only Linux can) */ + public const UNSUPPORTED_DURABILITY = 3306; + /** A sync durable mode requires failed before the commit point: the write was rolled back */ + public const DURABILITY_FAILED = 3307; + /** The configured secretKey is not registered for the store */ + public const KEY_NOT_REGISTERED = 3308; + /** A key maintenance of the store is running or was interrupted (run it again) */ + public const MAINTENANCE_IN_PROGRESS = 3309; + /** restore() refused the backup (incomplete or damaged, of another database without force, another shard size): nothing was changed */ + public const RESTORE_REFUSED = 3310; + /** A key maintenance found a database in use: nothing was changed */ + public const MAINTENANCE_BUSY = 3311; + + /** @var int key of this instance's kernel in Kernel::$instances */ + private $id; + + /** + * @param array|null $config Settings (docs/CONFIGURATION.md), merged over a .nonedb file when one is found; null: + * the file alone (or dev mode) + * @throws \InvalidArgumentException for an invalid setting or secretKey + * @throws \RuntimeException UNSUPPORTED_DURABILITY for 'durable' where it can't be provided + */ + public function __construct($config = null){ + $kernel = new \noneDB\Internal\Core\Kernel($config); + $this->id = spl_object_id($this); + \noneDB\Internal\Core\Kernel::$instances[$this->id] = $kernel; + } + + /** The main settings for var_dump() and print_r(), the secret masked */ + public function __debugInfo(){ + $kernel = \noneDB\Internal\Core\Kernel::$instances[$this->id] ?? null; + return $kernel === null ? [] : $kernel->config->debugInfo(); + } + + /** + * Instances hold a secret and open resources; they can't be serialized + * @throws \LogicException + */ + public function __serialize(): array { + throw new \LogicException('noneDB instances cannot be serialized'); + } + + public function __unserialize(array $data): void { + throw new \LogicException('noneDB instances cannot be unserialized'); + } + + public function __clone(){ + $source = \noneDB\Internal\Core\Kernel::$instances[$this->id]; + $this->id = spl_object_id($this); + \noneDB\Internal\Core\Kernel::$instances[$this->id] = $source->cloneKernel(); + } + + public function __destruct(){ + $kernel = \noneDB\Internal\Core\Kernel::$instances[$this->id] ?? null; + if($kernel !== null){ + unset(\noneDB\Internal\Core\Kernel::$instances[$this->id]); + $kernel->config->destroyed(); + $kernel->release(); + } + } + + // ========================================== + // QUERY BUILDER (FLUENT INTERFACE) + // ========================================== + + /** + * Create a new query builder for fluent interface + * @param string $dbname + * @return noneDBQuery + */ + public function query(string $dbname): noneDBQuery { + return new noneDBQuery($this, \noneDB\Internal\Core\Kernel::$instances[$this->id]->layout->normalizeDbName($dbname)); + } + + /** + * Check if config file exists + * @return bool + */ + public static function configExists(): bool{ + return \noneDB\Internal\Config\Configuration::configExists(); + } + + + /** + * Get the config file template path + * @return string|null + */ + public static function getConfigTemplate(): ?string{ + return \noneDB\Internal\Config\Configuration::getConfigTemplate(); + } + + + /** + * Clear config cache (useful for testing) + * @return void + */ + public static function clearConfigCache(): void{ + \noneDB\Internal\Config\Configuration::clearConfigCache(); + } + + + /** + * True when development mode is enabled (NONEDB_DEV_MODE constant or environment variable) + * @return bool + */ + public static function isDevMode(): bool{ + return \noneDB\Internal\Config\Configuration::isDevMode(); + } + + + /** + * Set development mode programmatically + * Useful for testing or when env vars are not available + * @param bool $enabled + * @return void + */ + public static function setDevMode(bool $enabled): void{ + \noneDB\Internal\Config\Configuration::setDevMode($enabled); + } + + + /** + * Clear all static caches (useful for testing or memory management) + * @return void + */ + public static function clearStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::clearStaticCache(); + } + + + /** + * Disable static caching (each instance uses its own cache) + * @return void + */ + public static function disableStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::disableStaticCache(); + } + + + /** + * Enable static caching (default) + * @return void + */ + public static function enableStaticCache(){ + return \noneDB\Internal\Cache\CacheManager::enableStaticCache(); + } + + + /** + * Durability of this instance and of the last write operation of the process (docs/DURABILITY.md §6): + * - mode: 'performance' or 'durable' + * - platform: ['os', 'durableSupported', 'notes'] + * - lastOperation: null before the first write, else ['op', 'db', 'outcome' => 'committed'|'rolled_back'|'unknown', + * 'required' => syncs the mode needed, 'completed' => syncs that succeeded, 'reason' => ?string] + * - lastSyncError: null or ['path' (relative to dbDir), 'op' => 'file'|'dir', 'time'] + * - syncs: ['file', 'dir', 'failed'] counters of the process + * @return array + */ + public function getDurability(): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->sync->getDurability(); + } + + // ========================================== + // PUBLIC INDEX API + // ========================================== + + /** + * Make the key indexes (.kix checkpoints) of a database again from its data files, then its field and + * spatial indexes where they changed. + * Normal operation never needs this; it is a recovery tool after files were copied or restored by hand. + * @param string $dbname + * @return array ['success' => bool, 'totalRecords' => int, 'time' => string] + */ + public function rebuildIndex($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->rebuildIndex($dbname); + } + + + /** + * Get index information for a database + * @param string $dbname + * @return array|null + */ + public function getIndexInfo($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->getIndexInfo($dbname); + } + + + /** + * Check a database against its own data, read-only: every line of every data file is + * decoded; the key index, field indexes, spatial indexes and shard meta are compared with what the + * data says. Nothing is written, repaired, flushed or recovered (rebuildIndex() repairs). + * @param string $dbname + * @return array ['ok' => bool, 'segments', 'lines', 'records', 'badLines' => [[segment, offset, length]], + * 'tornTail', 'indexMismatches', 'bufferedRecords', 'pendingRecovery', 'quarantineFiles', + * 'recoveryLog' => last entries] + */ + public function checkIntegrity($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->integrity->checkIntegrity($dbname); + } + + + /** + * Cheap counters of the process and, for a database, its metadata (docs/OPERATIONS.md §1): index cache + * hits, misses and evictions, index pages read, checkpoints, compactions, lock waits, recovery events, + * durability, the last checkIntegrity(); a database's counters, sizes and state. Nothing is scanned or written. + * @return array ['process' => [...]] (+ 'database' => [...] for $dbname) + */ + public function getMetrics(?string $dbname = null): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->metrics->getMetrics($dbname); + } + + + /** + * Recovery log of a database: what crash recovery and maintenance did to its files, oldest first. Every + * entry has "t" (time) and "action" (docs/API.md "getRecoveryLog()" lists them). What reads of this + * process found and no write session logged yet is included with "persisted" => false (reads never write). + * @param string $dbname + * @param int $limit Newest entries to return + * @return array + */ + public function getRecoveryLog($dbname, $limit = 100){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->recoveryLog->getRecoveryLog($dbname, $limit); + } + + + /** + * True when the database exists. With autoCreateDB a missing database is created; without it nothing is + * created and the answer is false. + * @param string $dbname + * @return bool + * @throws \InvalidArgumentException for names with characters outside [A-Za-z0-9' -] + */ + public function checkDB($dbname=null){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->checkDB($dbname); + } + + + /** + * Create a database (STORAGE_FORMAT.md §8.1) + * @param string $dbname + * @return bool True if created, false if it already existed + */ + public function createDB($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->createDB($dbname); + } + + + /** + * Delete a database (STORAGE_FORMAT.md §8.2): every file but its lock file + * @param string $dbname + * @return array ['success' => bool, 'files' => int deleted file count] + */ + public function dropDB($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->dropDB($dbname); + } + + + /** + * The databases of this key: their names, sorted (false); the same with each one's information (true); or + * one database's information, false when it doesn't exist (its name) + * @param bool|string $info + * @return array|false + */ + public function getDBs($info=false){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->getDBs($info); + } + + + /** + * The first $limit records of a list of records (a result of find()) + * @param array $array + * @param int $limit At least 1 + * @return array|false False for invalid arguments + */ + public function limit($array, $limit=0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->db->limit($array, $limit); + } + + /** + * The records matching the filters, in key order (a "key" filter: in its order) + * @param string $dbname + * @param mixed $filters 0/null/[] for all records, or an array of field => value (strict ===); + * a "key" entry selects by record key (int or list of ints) + * @return array|false False when the database doesn't exist and autoCreateDB is off, or for invalid filters + */ + public function find($dbname, $filters=0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->find($dbname, $filters); + } + + + /** + * Insert one record or a list of records: each gets the next key + * @param string $dbname + * @param array $data One record, or a list of records + * @return array ['n' => inserted count] or ['n' => 0, 'error' => message] + */ + public function insert($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->insert($dbname, $data); + } + + + /** + * Delete the records matching the filters + * @param string $dbname + * @param array $data Filters (same rules as find); [] deletes every record + * @return array ['n' => deleted count] + */ + public function delete($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->delete($dbname, $data); + } + + + /** + * Set fields of the records matching the filters + * @param string $dbname + * @param array $data [filters, ['set' => [field => value, ...]]] + * @return array ['n' => updated count] + */ + public function update($dbname, $data){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->crud->update($dbname, $data); + } + + // ========================================== + // QUERY METHODS + // ========================================== + + /** + * Get unique values for a field + * @param string $dbname + * @param string $field + * @return array|false + */ + public function distinct($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->distinct($dbname, $field); + } + + + /** + * Sort array by field + * @param array $array Result from find() + * @param string $field Field to sort by + * @param string $order 'asc' or 'desc' + * @return array|false + */ + public function sort($array, $field, $order = 'asc'){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->sort($array, $field, $order); + } + + + /** + * Count records matching filter + * @param string $dbname + * @param mixed $filter + * @return int + */ + public function count($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->count($dbname, $filter); + } + + + /** + * Pattern matching search (LIKE) + * @param string $dbname + * @param string $field + * @param string $pattern Use ^ for starts with, $ for ends with + * @return array|false + */ + public function like($dbname, $field, $pattern){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->queryMethods->like($dbname, $field, $pattern); + } + + // ========================================== + // AGGREGATION METHODS + // ========================================== + + /** + * Sum numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function sum($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->sum($dbname, $field, $filter); + } + + + /** + * Average of numeric field values + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return float|int + */ + public function avg($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->avg($dbname, $field, $filter); + } + + + /** + * Get minimum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function min($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->min($dbname, $field, $filter); + } + + + /** + * Get maximum value of a field + * @param string $dbname + * @param string $field + * @param mixed $filter + * @return mixed|null + */ + public function max($dbname, $field, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->aggregation->max($dbname, $field, $filter); + } + + // ========================================== + // UTILITY METHODS + // ========================================== + + /** + * Get first matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function first($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->first($dbname, $filter); + } + + + /** + * Get last matching record + * @param string $dbname + * @param mixed $filter + * @return array|null + */ + public function last($dbname, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->last($dbname, $filter); + } + + + /** + * Check if records exist matching filter + * @param string $dbname + * @param mixed $filter + * @return bool + */ + public function exists($dbname, $filter){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->exists($dbname, $filter); + } + + + /** + * Range query (min <= value <= max) + * @param string $dbname + * @param string $field + * @param mixed $min + * @param mixed $max + * @param mixed $filter Additional filter + * @return array|false + */ + public function between($dbname, $field, $min, $max, $filter = 0){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->utility->between($dbname, $field, $min, $max, $filter); + } + + // ========================================== + // SHARDING PUBLIC METHODS + // ========================================== + + /** + * Get sharding information for a database + * @param string $dbname + * @return array|false + */ + public function getShardInfo($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->shardInfo->getShardInfo($dbname); + } + + // ========================================== + // FIELD INDEX PUBLIC API + // ========================================== + + /** + * Create an index on a field for faster equality lookups (find / where with field => value) + * @param string $dbname Database name + * @param string $field Field name (any name except "key") + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] (+ 'shards' => int when sharded) + */ + public function createFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->createFieldIndex($dbname, $field); + } + + + /** + * Drop a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => string|null] + */ + public function dropFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->dropFieldIndex($dbname, $field); + } + + + /** + * Check if a field has an index + * @param string $dbname Database name + * @param string $field Field name + * @return bool + */ + public function hasFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->hasFieldIndex($dbname, $field); + } + + + /** + * Get list of field indexes for a database + * @param string $dbname Database name + * @return array ['fields' => array, 'sharded' => bool] + */ + public function getFieldIndexes($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->getFieldIndexes($dbname); + } + + + /** + * Rebuild a field index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createFieldIndex + */ + public function rebuildFieldIndex($dbname, $field){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->rebuildFieldIndex($dbname, $field); + } + + + /** + * Get keys matching a field value using field index + * Returns null if no index exists (caller should fall back to scan) + * @param string $dbname Database name + * @param string $field Field name + * @param mixed $value Value to match + * @param int|null $shardId Shard ID for sharded databases (null = all shards) + * @return array|null Array of matching keys, or null if no index + */ + public function getKeysFromFieldIndex($dbname, $field, $value, $shardId = null){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->fieldIndexApi->getKeysFromFieldIndex($dbname, $field, $value, $shardId); + } + + // ========================================== + // SPATIAL INDEX - GEOJSON SUPPORT + // ========================================== + + /** + * Validate a GeoJSON geometry object + * Supports: Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection + * + * @param array $geometry GeoJSON geometry object + * @return array ['valid' => bool, 'error' => ?string, 'type' => ?string] + */ + public function validateGeoJSON($geometry): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->geoJson->validateGeoJSON($geometry); + } + + + /** + * Calculate Haversine distance between two points (in meters). Not memoized: a key for two points costs + * more than the formula + * @param float $lon1 Longitude of point 1 + * @param float $lat1 Latitude of point 1 + * @param float $lon2 Longitude of point 2 + * @param float $lat2 Latitude of point 2 + * @return float Distance in meters + */ + public function haversineDistance(float $lon1, float $lat1, float $lon2, float $lat2): float{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->geometry->haversineDistance($lon1, $lat1, $lon2, $lat2); + } + + + /** + * Check if a spatial index exists + * @param string $dbname Database name + * @param string $field Field name + * @param int|null $shardId Shard ID + * @return bool + */ + public function hasSpatialIndex(string $dbname, string $field, ?int $shardId = null): bool{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->rtree->hasSpatialIndex($dbname, $field, $shardId); + } + + // ========================================== + // SPATIAL INDEX - PUBLIC API + // ========================================== + + /** + * Create a spatial index on a GeoJSON field + * @param string $dbname Database name + * @param string $field Field name containing GeoJSON + * @param array $options Options ['nodeSize' => 32] + * @return array ['success' => bool, 'indexed' => int, 'error' => ?string] + */ + public function createSpatialIndex(string $dbname, string $field, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->createSpatialIndex($dbname, $field, $options); + } + + + /** + * Drop a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array ['success' => bool, 'error' => ?string] + */ + public function dropSpatialIndex(string $dbname, string $field): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->dropSpatialIndex($dbname, $field); + } + + + /** + * Get list of spatial indexes for a database + * @param string $dbname Database name + * @return array List of indexed field names + */ + public function getSpatialIndexes(string $dbname): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->getSpatialIndexes($dbname); + } + + + /** + * Rebuild a spatial index + * @param string $dbname Database name + * @param string $field Field name + * @return array Result from createSpatialIndex + */ + public function rebuildSpatialIndex(string $dbname, string $field): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialIndexApi->rebuildSpatialIndex($dbname, $field); + } + + + /** + * Find records within a distance from a point (distance to the nearest point of the geometry) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param float $distanceMeters Distance in meters + * @param array $options ['includeDistance' => false] + * @return array Matching records (sorted by distance, with _distance in meters if includeDistance=true) + */ + public function withinDistance(string $dbname, string $field, float $lon, float $lat, float $distanceMeters, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinDistance($dbname, $field, $lon, $lat, $distanceMeters, $options); + } + + + /** + * Find records within a bounding box (geometries whose bounding box overlaps it). + * minLon > maxLon means the box crosses the antimeridian. + * @return array Matching records + */ + public function withinBBox(string $dbname, string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinBBox($dbname, $field, $minLon, $minLat, $maxLon, $maxLat); + } + + + /** + * Find K nearest records to a point + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @param array $options ['maxDistance' => null (meters), 'includeDistance' => true] + * @return array Up to k records sorted by distance (fewer when maxDistance cuts them off) + */ + public function nearest(string $dbname, string $field, float $lon, float $lat, int $k, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->nearest($dbname, $field, $lon, $lat, $k, $options); + } + + + /** + * Find records whose geometry lies within a Polygon or MultiPolygon (holes excluded) + * @param string $dbname Database name + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon or MultiPolygon geometry + * @return array Matching records + */ + public function withinPolygon(string $dbname, string $field, array $polygon): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->spatialQueries->withinPolygon($dbname, $field, $polygon); + } + + // ========================================== + // WRITE BUFFER PUBLIC API + // ========================================== + + /** + * Manually flush buffer for a database + * @param string $dbname + * @return array ['success' => bool, 'flushed' => int, 'error' => string|null] + */ + public function flush($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->flush($dbname); + } + + + /** + * Flush the write buffers of every database (of this secretKey) in dbDir + * @return array ['databases' => int, 'flushed' => int] + */ + public function flushAllBuffers(){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->flushAllBuffers(); + } + + + /** + * Get buffer information for a database + * @param string $dbname + * @return array + */ + public function getBufferInfo($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->getBufferInfo($dbname); + } + + + /** + * Enable or disable write buffering + * @param bool $enable + */ + public function enableBuffering($enable = true){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->enableBuffering($enable); + } + + + /** + * Check if buffering is enabled + * @return bool + */ + public function isBufferingEnabled(){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->isBufferingEnabled(); + } + + + /** + * Set buffer size limit (in bytes) + * @param int $bytes + */ + public function setBufferSizeLimit($bytes){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferSizeLimit($bytes); + } + + + /** + * Set buffer flush interval (in seconds) + * @param int $seconds 0 to disable time-based flush + */ + public function setBufferFlushInterval($seconds){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferFlushInterval($seconds); + } + + + /** + * Set buffer count limit + * @param int $count + */ + public function setBufferCountLimit($count){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->setBufferCountLimit($count); + } + + + /** + * Compact a database: drop old record versions and deleted records from the data files. + * Keys are preserved (sharded and non-sharded alike); indexes stay valid. + * @param string $dbname + * @return array ['success' => bool, 'freedSlots' => int, 'totalRecords' => int, 'sharded' => bool] + */ + public function compact($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->compact($dbname); + } + + /** + * Switch a database of the single layout to the sharded layout (STORAGE_FORMAT.md §5.2). A database + * switches by itself once it has assigned shardSize keys (shardingEnabled) + * @param string $dbname + * @return array ["success" => bool, "status" => string] + */ + public function convertToSharded($dbname){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->convertToSharded($dbname); + } + + + /** + * Check if sharding is enabled + * @return bool + */ + public function isShardingEnabled(){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->isShardingEnabled(); + } + + + /** + * Get current shard size setting + * @return int + */ + public function getShardSize(){ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->maintenance->getShardSize(); + } + + // ========================================== + // BACKUP AND RESTORE (docs/STORAGE_FORMAT.md §10) + // ========================================== + + /** + * Copy a database, as it is at one moment, into $dir: a directory outside dbDir that doesn't exist or is + * empty (0700, files 0600). Its manifest.json is written last; store.json is the STORE record. Writers of + * the database wait while it runs, key maintenance fails with MAINTENANCE_BUSY; nothing of the database is + * written (the buffer is copied, not flushed). + * @return array ['database', 'dir', 'databaseId', 'generation', 'nextKey', 'records', 'bufferedRecords', 'files', 'bytes'] + * @throws \InvalidArgumentException $dir inside dbDir or not empty + * @throws \RuntimeException the database does not exist; MAINTENANCE_IN_PROGRESS, KEY_NOT_REGISTERED, LOCK_FAILED + */ + public function backup(string $dbname, string $dir): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->backupRestore->backup($dbname, $dir); + } + + + /** + * Make the backup in $dir the content of database $dbname, with one commit: the database opens as it was + * or as restored after any crash. A backup of another database is refused unless ['force' => true], which + * loads it under this database's identity. Keys never go back: new records get keys above every key the + * database handed out. Also the way out of FORMAT_DAMAGED: the damaged generations are kept. + * @return array ['database', 'generation', 'nextKey', 'records', 'bufferedRecords', 'target' => 'present'|'absent'|'damaged', 'foreign' => bool] + * @throws \RuntimeException RESTORE_REFUSED (nothing changed), LOCK_FAILED, MAINTENANCE_IN_PROGRESS + */ + public function restore(string $dir, string $dbname, array $options = []): array{ + return \noneDB\Internal\Core\Kernel::$instances[$this->id]->backupRestore->restore($dir, $dbname, $options); + } + + // ========================================== + // KEY MAINTENANCE (docs/STORAGE_FORMAT.md §9) + // ========================================== + // + // Every operation follows the maintenance order: STORE.LOCK (exclusive, non-blocking), then every + // target database's LOCK (non-blocking; one busy lock: nothing written, MAINTENANCE_BUSY), then the + // journal .nonedb-maintenance/journal.json, then "m" and a new epoch in every target's state record. + // The logical commit point is the journal in state COMMITTED with the complete target STORE record; + // the STORE copies, clearing "m" and closing the journal follow and are repeated by a re-run. + + /** + * Rotate the secret key of every database in $dbDir that belongs to $oldKey: their directories are + * renamed to the new key's locators (contents unchanged: FORMAT and metadata hold no key) and STORE + * registers $newKey instead of $oldKey. Application processes must be stopped. Re-running after an + * interruption finishes the rotation; ['abort' => true] undoes one that did not commit yet. + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'databases' => string[], 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function migrateKey(string $dbDir, string $oldKey, string $newKey, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::migrateKey($dbDir, $oldKey, $newKey, $options); + } + + + /** + * Register a further key for the store in $dbDir; $registeredKey must already be registered. + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'storeManifest' => array|null, 'error' => ?string, 'code' => ?int] + */ + public static function registerKey(string $dbDir, string $registeredKey, string $newKey, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::registerKey($dbDir, $registeredKey, $newKey, $options); + } + + + /** + * Rebuild a store record whose two copies are both invalid (STORAGE_FORMAT.md §9.4). Every database + * directory must belong to one of $keys. ['manifest' => path or array]: a verified STORE record (a + * backup's store.json, a maintenance result's storeManifest) is restored (complete). Without one, only + * ['confirmManualRebuild' => true] rebuilds the record from $keys (not complete: keys without + * databases that were registered before are lost). + * ['durability' => 'durable'] syncs every step as durable mode does (docs/DURABILITY.md). + * @return array ['success' => bool, 'mode' => 'manifest'|'manual'|null, 'complete' => bool, 'storeManifest' => array|null, + * 'error' => ?string (lists databases no key owns), 'code' => ?int] + */ + public static function repairStore(string $dbDir, array $keys, array $options = []): array{ + return \noneDB\Internal\Maintenance\KeyMaintenance::repairStore($dbDir, $keys, $options); + } +} diff --git a/src/noneDBQuery/Aggregates.php b/src/noneDBQuery/Aggregates.php new file mode 100644 index 0000000..b215fc4 --- /dev/null +++ b/src/noneDBQuery/Aggregates.php @@ -0,0 +1,93 @@ +eachResult(function (array $record) use ($field, &$sum, &$count) { + $number = \noneDB\Internal\Query\Values::numberOf($record[$field] ?? null); + if ($number !== null) { + $sum += $number; + $count++; + } + }); + return [$sum, $count]; + } + + /** + * Sum of field values + * @param string $field + * @return float + */ + public function sum(string $field): float { + return (float)$this->numericTotal($field)[0]; + } + + /** + * Average of field values + * @param string $field + * @return float + */ + public function avg(string $field): float { + [$sum, $count] = $this->numericTotal($field); + return $count > 0 ? $sum / $count : 0.0; + } + + /** + * Minimum value of field (same ordering as sort()) + * @param string $field + * @return mixed|null + */ + public function min(string $field) { + $min = null; + $found = false; + $this->eachResult(function (array $record) use ($field, &$min, &$found) { + if (isset($record[$field]) && (!$found || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $min) < 0)) { + $min = $record[$field]; + $found = true; + } + }); + return $min; + } + + /** + * Maximum value of field (same ordering as sort()) + * @param string $field + * @return mixed|null + */ + public function max(string $field) { + $max = null; + $found = false; + $this->eachResult(function (array $record) use ($field, &$max, &$found) { + if (isset($record[$field]) && (!$found || \noneDB\Internal\Query\Values::compareSortValues($record[$field], $max) > 0)) { + $max = $record[$field]; + $found = true; + } + }); + return $max; + } + + /** + * Get unique values of field + * @param string $field + * @return array + */ + public function distinct(string $field): array { + $seen = []; + $values = []; + $this->eachResult(function (array $record) use ($field, &$seen, &$values) { + \noneDB\Internal\Query\Values::collectUnique([$record], $field, $seen, $values); + }); + return $values; + } + +} diff --git a/src/noneDBQuery/Chain.php b/src/noneDBQuery/Chain.php new file mode 100644 index 0000000..64f3653 --- /dev/null +++ b/src/noneDBQuery/Chain.php @@ -0,0 +1,271 @@ + 10], with operators. + * @param array $filters + * @return self + */ + public function where(array $filters): self { + if (!empty($filters)) { + foreach ($filters as $value) { + $this->isOperatorSpec($value); // validates early + } + $this->whereFilters[] = $filters; + } + return $this; + } + + /** + * Add pattern matching filter + * @param string $field + * @param string $pattern "^" anchors the start, a trailing "$" the end; everything else is literal + * @return self + */ + public function like(string $field, string $pattern): self { + $this->likeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; + return $this; + } + + /** + * Add range filter (min <= value <= max) + * @param string $field + * @param mixed $min + * @param mixed $max + * @return self + */ + public function between(string $field, $min, $max): self { + $this->betweenFilters[] = ['field' => $field, 'min' => $min, 'max' => $max]; + return $this; + } + + /** + * Add a sort key. The first call is the primary key; later calls break ties. + * @param string $field + * @param string $order 'asc' or 'desc' + * @return self + */ + public function sort(string $field, string $order = 'asc'): self { + $this->sortKeys[] = ['field' => $field, 'order' => strtolower($order) === 'desc' ? 'desc' : 'asc']; + return $this; + } + + /** + * Limit results + * @param int $count + * @return self + */ + public function limit(int $count): self { + $this->limitCount = max(0, $count); + return $this; + } + + /** + * Skip first N results (for pagination) + * @param int $count + * @return self + */ + public function offset(int $count): self { + $this->offsetCount = max(0, $count); + return $this; + } + + /** + * Alias for offset() - skip first N results + * @param int $count + * @return self + */ + public function skip(int $count): self { + return $this->offset($count); + } + + /** + * Alias for sort() + * @param string $field + * @param string $order 'asc' or 'desc' + * @return self + */ + public function orderBy(string $field, string $order = 'asc'): self { + return $this->sort($field, $order); + } + + /** + * Add an OR group: a record matches if all where() groups match OR any orWhere() group does + * @param array $filters + * @return self + */ + public function orWhere(array $filters): self { + foreach ($filters as $value) { + $this->isOperatorSpec($value); + } + $this->orWhereFilters[] = $filters; + return $this; + } + + /** + * Filter where field value is in given array (strict comparison) + * @param string $field + * @param array $values + * @return self + */ + public function whereIn(string $field, array $values): self { + $this->whereInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values)), 'raw' => array_values($values)]; + return $this; + } + + /** + * Value list for strict membership tests: longer lists become a hash set of exact identities + * (noneDB::valueKey: same result as in_array(..., true), without scanning the list per record) + * @return array|\stdClass + */ + private function membership(array $values) { + if (count($values) <= 8) { + return $values; + } + $valueKey = $this->kernel()->values->valueKeyFunction(); + $set = []; + foreach ($values as $value) { + $set[$valueKey($value)] = true; + } + return (object)['set' => $set, 'key' => $valueKey]; + } + + private function isMember($value, $membership): bool { + if ($membership instanceof \stdClass) { + return isset($membership->set[($membership->key)($value)]); + } + return in_array($value, $membership, true); + } + + /** + * Filter where field value is NOT in given array (strict comparison) + * @param string $field + * @param array $values + * @return self + */ + public function whereNotIn(string $field, array $values): self { + $this->whereNotInFilters[] = ['field' => $field, 'values' => $this->membership(array_values($values))]; + return $this; + } + + /** + * Exclude records where a field equals the value (strict comparison) + * @param array $filters + * @return self + */ + public function whereNot(array $filters): self { + if (!empty($filters)) { + // Per field: the field matches when all its conditions match (AND, like in where()); + // a record is excluded when any field matches. Operators are checked now, like where(). + $fields = []; + foreach ($filters as $field => $value) { + $fields[] = $this->compileGroups([[$field => $value]])[0]; + } + $this->whereNotFilters[] = $fields; + } + return $this; + } + + /** + * Filter where field does NOT match pattern + * @param string $field + * @param string $pattern + * @return self + */ + public function notLike(string $field, string $pattern): self { + $this->notLikeFilters[] = ['field' => $field, 'pattern' => $pattern, 'like' => $this->likeSpec($pattern)]; + return $this; + } + + /** + * Filter where field value is NOT between min and max + * @param string $field + * @param mixed $min + * @param mixed $max + * @return self + */ + public function notBetween(string $field, $min, $max): self { + $this->notBetweenFilters[] = ['field' => $field, 'min' => $min, 'max' => $max]; + return $this; + } + + /** + * Select only specific fields in results ("key" is always included) + * @param array $fields + * @return self + */ + public function select(array $fields): self { + $this->selectFields = array_values($fields); + return $this; + } + + /** + * Exclude specific fields from results + * @param array $fields + * @return self + */ + public function except(array $fields): self { + $this->exceptFields = array_values($fields); + return $this; + } + + /** + * Group results by field. get() then returns groups: ['_group' => value, '_items' => [...], '_count' => n] + * @param string $field + * @return self + */ + public function groupBy(string $field): self { + $this->groupByField = $field; + return $this; + } + + /** + * Filter groups by aggregate condition. Only used together with groupBy(); without it, and for + * unknown aggregates or operators, the condition is ignored (passes) + * @param string $aggregate 'count', 'sum:field', 'avg:field', 'min:field', 'max:field' + * @param string $operator '>', '<', '>=', '<=', '=', '!=' + * @param mixed $value + * @return self + */ + public function having(string $aggregate, string $operator, $value): self { + $this->havingFilters[] = ['aggregate' => $aggregate, 'operator' => $operator, 'value' => $value]; + return $this; + } + + /** + * Case-insensitive full-text search across fields + * @param string $term Search term + * @param array $fields Fields to search in (empty = all fields except "key") + * @return self + */ + public function search(string $term, array $fields = []): self { + $this->searchFilters[] = ['term' => $term, 'fields' => array_values($fields)]; + return $this; + } + + /** + * Join with another database + * @param string $foreignDb Foreign database name + * @param string $localKey Local key field + * @param string $foreignKey Foreign key field + * @param string|null $alias Field that receives the joined record (default: the foreign database name; + * it replaces a field of the same name in the result, never in storage) + * @return self + */ + public function join(string $foreignDb, string $localKey, string $foreignKey, ?string $alias = null): self { + $this->joinConfigs[] = [ + 'foreignDb' => $foreignDb, + 'localKey' => $localKey, + 'foreignKey' => $foreignKey, + 'alias' => $alias ?? $foreignDb + ]; + return $this; + } + +} diff --git a/src/noneDBQuery/Core.php b/src/noneDBQuery/Core.php new file mode 100644 index 0000000..b901986 --- /dev/null +++ b/src/noneDBQuery/Core.php @@ -0,0 +1,69 @@ + ..., 'order' => 'asc'|'desc'], ...] */ + private array $sortKeys = []; + private ?int $limitCount = null; + private int $offsetCount = 0; + + // Spatial query support + private array $spatialFilters = []; + private ?array $distanceConfig = null; + /** @var array|null Statistics of the last scan (explain()) */ + private ?array $lastScan = null; + + + private const OPERATORS = ['$gt', '$gte', '$lt', '$lte', '$ne', '$eq', '$in', '$nin', '$exists', '$like', '$regex', '$contains']; + + /** + * @param noneDB $db + * @param string $dbname + */ + public function __construct(noneDB $db, string $dbname) { + $this->db = $db; + $this->dbname = $dbname; + } + + /** + * The engine of the database instance (looked up, not held: a dump of the query shows no internals) + */ + private function kernel(): \noneDB\Internal\Core\Kernel { + return \noneDB\Internal\Core\Kernel::of($this->db); + } + + /** + * Debug output without the parent instance's internals + */ + public function __debugInfo() { + return [ + 'dbname' => $this->dbname, + 'where' => $this->whereFilters, + 'orWhere' => $this->orWhereFilters, + 'sort' => $this->sortKeys, + 'limit' => $this->limitCount, + 'offset' => $this->offsetCount, + ]; + } + +} diff --git a/src/noneDBQuery/Execution.php b/src/noneDBQuery/Execution.php new file mode 100644 index 0000000..2ddcda8 --- /dev/null +++ b/src/noneDBQuery/Execution.php @@ -0,0 +1,926 @@ + record]); nearest() returns its candidates + * in distance order with _distance set. Null when there are no spatial filters. + * @param float|null $nearestRadius Only the records within this many meters for a nearest() filter + * (null: all records in distance order) + * @param float|null $nearestBoundary Set when the radius was used: records with a smaller _distance + * are exactly those of the full list (anything outside the radius is at least this far) + */ + private function spatialCandidates(?float $nearestRadius = null, ?float &$nearestBoundary = null): ?array { + $nearestBoundary = null; + if (empty($this->spatialFilters)) { + return null; + } + $result = null; + foreach ($this->spatialFilters as $filter) { + switch ($filter['type']) { + case 'withinDistance': + $rows = $this->db->withinDistance($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $filter['distance'], ['includeDistance' => true]); + break; + case 'withinBBox': + $rows = $this->db->withinBBox($this->dbname, $filter['field'], $filter['minLon'], $filter['minLat'], $filter['maxLon'], $filter['maxLat']); + break; + case 'nearest': + // Records in distance order; the k cut happens after the other filters + if ($filter['k'] <= 0) { + $rows = []; + } elseif ($nearestRadius !== null) { + $rows = $this->db->withinDistance($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $nearestRadius, ['includeDistance' => true]); + $nearestBoundary = round($nearestRadius, 2); + } else { + $all = max(1, $this->db->count($this->dbname)); + $rows = $this->db->nearest($this->dbname, $filter['field'], $filter['lon'], $filter['lat'], $all, ['includeDistance' => true]); + } + break; + case 'withinPolygon': + $rows = $this->db->withinPolygon($this->dbname, $filter['field'], $filter['polygon']); + break; + default: + $rows = []; + } + $keyed = []; + foreach ($rows as $row) { + if (isset($row['key'])) { + $keyed[$row['key']] = $row; + } + } + if ($result === null) { + $result = $keyed; + } elseif ($filter['type'] === 'nearest') { + // nearest() orders and cuts by its own distance: its order and _distance win + $result = array_intersect_key($keyed, $result); + } else { + // Intersect by key, keep the order and fields of the earlier result + $result = array_intersect_key($result, $keyed); + foreach ($result as $key => $row) { + if (isset($keyed[$key]['_distance']) && !isset($row['_distance'])) { + $result[$key]['_distance'] = $keyed[$key]['_distance']; + } + } + } + } + return $result ?? []; + } + + /** + * The plan of the query's filters (see noneDB's Planner): what field indexes, global field indexes and + * key lookups can narrow down. Every record read is still checked against every filter. + * @return array + */ + private function buildPlan(): array { + $where = []; + foreach ($this->whereFilters as $group) { + $where[] = $this->groupPlan($group); + } + $node = count($where) === 1 ? $where[0] : (empty($where) ? null : ['and', $where]); + if (!empty($this->orWhereFilters)) { + // where groups (all) OR any orWhere group; an empty orWhere() group matches everything + $branches = $node === null ? [] : [$node]; + foreach ($this->orWhereFilters as $group) { + $branches[] = $this->groupPlan($group); + } + $node = ['or', $branches]; + } + $and = $node === null ? [] : [$node]; + foreach ($this->whereInFilters as $filter) { + $and[] = $this->inPlan($filter['field'], $filter['raw'] ?? []); + } + foreach ($this->betweenFilters as $between) { + $and[] = ['range', $between['field'], '$gte', $between['min']]; + $and[] = ['range', $between['field'], '$lte', $between['max']]; + } + return self::simplifyPlan(['and', $and]); + } + + /** + * Plan of one where()/orWhere() group (all its conditions) + */ + private function groupPlan(array $group): array { + $and = []; + foreach ($group as $field => $value) { + $field = (string)$field; + if (!$this->isOperatorSpec($value)) { + if ($field === 'key') { + $and[] = ['keys', $this->kernel()->findFilter->normalizeKeyFilter($value)]; + } elseif (!is_array($value)) { + $and[] = ['eq', $field, $value]; + } + continue; + } + foreach ($value as $operator => $operand) { + if ($operator === '$eq' && !is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->kernel()->findFilter->normalizeKeyFilter($operand)] : ['eq', $field, $operand]; + } elseif ($operator === '$in' && is_array($operand)) { + $and[] = $field === 'key' ? ['keys', $this->kernel()->findFilter->normalizeKeyFilter(array_values($operand))] : $this->inPlan($field, $operand); + } elseif (in_array($operator, ['$gt', '$gte', '$lt', '$lte'], true) && $field !== 'key' && !is_array($operand)) { + $and[] = ['range', $field, $operator, $operand]; + } + // $ne, $nin, $exists, $like, $regex, $contains: checked on the records only + } + } + return self::simplifyPlan(['and', $and]); + } + + private function inPlan(string $field, array $values): array { + foreach ($values as $value) { + if (is_array($value) || is_object($value)) { + return ['any']; // indexes hold no arrays + } + } + return ['in', $field, array_values($values)]; + } + + private static function simplifyPlan(array $node): array { + if ($node[0] !== 'and' && $node[0] !== 'or') { + return $node; + } + $children = []; + foreach ($node[1] as $child) { + $child = self::simplifyPlan($child); + if ($child[0] === 'any') { + if ($node[0] === 'or') { + return ['any']; // a branch that matches anything + } + continue; // AND with "anything" is the rest + } + $children[] = $child; + } + if (empty($children)) { + return ['any']; + } + return count($children) === 1 ? $children[0] : [$node[0], $children]; + } + + /** + * True when the filters are one plain where() group with a key filter: find() keeps the order of + * the keys asked for, which a scan in key order wouldn't + */ + private function keyOrderLookup(): bool { + if (count($this->whereFilters) !== 1 || !empty($this->orWhereFilters) || !array_key_exists('key', $this->whereFilters[0])) { + return false; + } + foreach ($this->whereFilters[0] as $value) { + if ($this->isOperatorSpec($value)) { + return false; + } + } + return true; + } + + /** + * Filtered records (no join, grouping or projection) + * @param bool $earlyExit Stop once offset+limit records are collected (only when order can't change) + */ + private function filteredRecords(bool $earlyExit, ?int $target = null, ?int $topK = null): array { + // One shared lock for every read the query makes (count, spatial rounds, records), so they + // all see the same state of the database + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withReadLock($dbname, function () use ($earlyExit, $target, $topK) { + return $this->filteredRecordsLocked($earlyExit, $target, $topK); + }); + } + + /** + * @param int|null $target Matches enough to stop early (exists()); default: offset + limit without sort + */ + private function filteredRecordsLocked(bool $earlyExit, ?int $target = null, ?int $topK = null): array { + $nearest = null; + $nearestFilters = 0; + foreach ($this->spatialFilters as $filter) { + if ($filter['type'] === 'nearest') { + $nearest = $nearest === null ? $filter['k'] : min($nearest, $filter['k']); + $nearestFilters++; + } + } + // A single nearest() as the first spatial filter (its distances are the ones kept) on an indexed + // field doesn't need every record in distance order: search a growing radius until k records match + $nearestRadius = null; + $total = 0; + if ($nearestFilters === 1 && $this->spatialFilters[0]['type'] === 'nearest' && $nearest > 0 + && $this->kernel()->spatialQueries->usesSpatialIndex($this->dbname, $this->spatialFilters[0]['field'])) { + $nearestRadius = 1000.0; + $total = $this->db->count($this->dbname); + } + + [$single, $where, $orWhere, $needWhere, $needAdvanced] = $this->compiledFilters(); + if ($target !== null && $earlyExit && $nearest === null) { + // exists(): independent of sort + } else { + $target = ($earlyExit && $nearest === null && empty($this->sortKeys) && $this->limitCount !== null) + ? $this->offsetCount + $this->limitCount + : PHP_INT_MAX; + } + + while (true) { + $boundary = null; + $spatial = $this->spatialCandidates($nearestRadius, $boundary); + if ($spatial !== null) { + $records = array_values($spatial); + $whereApplied = false; + } elseif ($this->keyOrderLookup()) { + $records = $this->db->find($this->dbname, $this->whereFilters[0]); + $records = $records === false ? [] : $records; + $whereApplied = true; + } else { + // Indexes narrow the segments and keys down; the records are read in batches and the scan + // stops once enough match. A sort with a small limit keeps only the best ones (top-k). + return $this->scanFiltered($target, $topK); + } + + $filtered = []; + foreach ($records as $record) { + if (!$whereApplied) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) continue; + } + if ($needAdvanced && !$this->matchesAdvancedFilters($record)) continue; + $filtered[] = $record; + if (count($filtered) >= $target) break; + } + + if ($boundary !== null) { + // Candidates closer than the radius are exactly the full list's (at the radius itself + // rounding may leave some out). Enough matches among them: done. + $certain = []; + foreach ($filtered as $record) { + if (($record['_distance'] ?? INF) < $boundary) { + $certain[] = $record; + } + } + if (count($certain) >= $nearest) { + $filtered = $certain; + break; + } + if (count($records) >= $total) { + break; // every record lies within the radius: the list is complete + } + // Matches grow with the area (r²): aim for k from the matches so far, at least doubling + $factor = count($certain) > 0 ? min(16.0, max(2.0, sqrt($nearest / count($certain)) * 1.25)) : 4.0; + // Half the earth's circumference covers everything: then take the complete list + $nearestRadius = $nearestRadius * $factor >= 20040000.0 ? null : $nearestRadius * $factor; + continue; + } + break; + } + + if ($nearest !== null) { + // Stable on every PHP version: equal distances keep the candidate order + $positions = array_keys($filtered); + $distances = array_map(function ($r) { return $r['_distance'] ?? INF; }, $filtered); + array_multisort($distances, SORT_ASC, SORT_NUMERIC, $positions, SORT_ASC, SORT_NUMERIC); + $sorted = []; + foreach (array_slice($positions, 0, $nearest) as $position) { + $sorted[] = $filtered[$position]; + } + $filtered = $sorted; + } + return $filtered; + } + + /** + * The filters compiled once per query: [the one where() group when it is the only one (no AND/OR logic + * per record) or null, where() groups, orWhere() groups, any where/orWhere, any other filter] + */ + private function compiledFilters(): array { + $needWhere = !empty($this->whereFilters) || !empty($this->orWhereFilters); + $where = $needWhere ? $this->compileGroups($this->whereFilters) : []; + $orWhere = $needWhere ? $this->compileGroups($this->orWhereFilters) : []; + $single = ($needWhere && count($where) === 1 && empty($orWhere)) ? $where[0] : null; + return [$single, $where, $orWhere, $needWhere, $this->hasAdvancedFilters()]; + } + + /** + * Visit the records matching the filters in key order through the plan's scan, none kept: $visit($key, + * $record) returns false to stop; the first $offset matches are skipped and at most $limit visited. One + * segment's records are held at a time. Not for spatial filters and key lookups (filteredRecordsLocked()). + * Caller holds the lock. + * @param bool $whole The visitor never stops: each segment is read in one go + * @return array The scan's statistics (explain()) + */ + private function eachMatch(callable $visit, bool $whole, int $offset = 0, ?int $limit = null): array { + [$single, $where, $orWhere, $needWhere, $needAdvanced] = $this->compiledFilters(); + $left = $limit ?? PHP_INT_MAX; + if ($left <= 0) { + return []; + } + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->planner->scanRecords($dbname, $this->buildPlan(), + function (array $records) use ($visit, $single, $where, $orWhere, $needWhere, $needAdvanced, &$offset, &$left) { + foreach ($records as $key => $record) { + if ($single !== null) { + if (!$this->matchesGroup($record, $single)) continue; + } elseif ($needWhere && !$this->matchesWhereLogic($record, $where, $orWhere)) { + continue; + } + if ($needAdvanced && !$this->matchesAdvancedFilters($record)) continue; + if ($offset > 0) { + $offset--; + continue; + } + if ($visit($key, $record) === false || --$left === 0) { + return false; + } + } + return true; + }, $whole); + } + + /** + * filteredRecordsLocked() without spatial filters: the plan's candidates, every filter checked + * @param int $target Stop after this many matches + * @param int|null $topK Keep only the best $topK by the sort keys (records arrive in key order) + */ + private function scanFiltered(int $target, ?int $topK): array { + $filtered = []; + $prune = $topK === null ? PHP_INT_MAX : max(4 * $topK, 1024); + $this->lastScan = $this->eachMatch(function ($key, array $record) use (&$filtered, $target, $topK, $prune) { + $filtered[] = $record; + if ($topK !== null) { + if (count($filtered) > $prune) { + $filtered = array_slice($this->applySort($filtered), 0, $topK); + } + } elseif (count($filtered) >= $target) { + return false; + } + return true; + }, $target === PHP_INT_MAX); + return $filtered; + } + + /** + * True when the results are the matches in key order (no groupBy(), join, sort, spatial filter, + * withDistance() or key lookup): aggregates and last() then read them through eachMatch(), one segment + * held at a time, instead of collecting them + */ + private function streamsResults(): bool { + return $this->groupByField === null && empty($this->joinConfigs) && empty($this->sortKeys) && empty($this->spatialFilters) + && $this->distanceConfig === null && !$this->keyOrderLookup(); + } + + /** + * Every record of the results in order, without projection (the aggregates): $visit($record) + */ + private function eachResult(callable $visit): void { + if (!$this->streamsResults()) { + foreach ($this->results(false) as $record) { + $visit($record); + } + return; + } + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + $this->kernel()->locks->withReadLock($dbname, function () use ($visit) { + $this->eachMatch(function ($key, array $record) use ($visit) { + $visit($record); + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + }); + } + + /** + * Add _distance for withDistance() + */ + private function applyDistanceCalculation(array $records): array { + if ($this->distanceConfig === null) { + return $records; + } + $field = $this->distanceConfig['field']; + foreach ($records as &$record) { + $geometry = $record[$field] ?? null; + if (!is_array($geometry) || !$this->db->validateGeoJSON($geometry)['valid']) { + continue; + } + $distance = $this->kernel()->spatialQueries->distanceToGeometry((float)$this->distanceConfig['lon'], (float)$this->distanceConfig['lat'], $geometry); + if (is_finite($distance)) { // a geometry without any position has no distance + $record['_distance'] = round($distance, 4); + } + } + unset($record); + return $records; + } + + private function applyJoins(array $records): array { + foreach ($this->joinConfigs as $join) { + if ($join['foreignKey'] === 'key') { + // Joining on record keys: read only the keys asked for + $keys = []; + foreach ($records as $record) { + $lk = self::joinKey($record[$join['localKey']] ?? null); + if (is_int($lk) && $lk >= 0) { + $keys[$lk] = $lk; + } + } + $foreignData = empty($keys) ? [] : $this->db->find($join['foreignDb'], ['key' => array_values($keys)]); + } else { + // Only the foreign records whose value can join: the local raw values (a whole number also as + // its float and canonical string, which join it too), through the foreign field index when + // there is one. The join identity (joinKey) is only compared, never used as a filter value. + $values = []; + foreach ($records as $record) { + $value = $record[$join['localKey']] ?? null; + $lk = self::joinKey($value); + if ($lk === null) { + continue; + } + if (is_int($lk)) { + $values['i' . $lk] = $lk; + $values['d' . $lk] = (float)$lk; + $values['s' . $lk] = (string)$lk; + } else { + $values[$lk] = $value; + } + } + $foreignData = empty($values) ? [] : $this->db->query($join['foreignDb'])->whereIn($join['foreignKey'], array_values($values))->get(); + } + if ($foreignData === false) { + $foreignData = []; + } + $foreignIndexed = []; + foreach ($foreignData as $fRecord) { + $fk = self::joinKey($fRecord[$join['foreignKey']] ?? null); + if ($fk !== null) { + $foreignIndexed[$fk] = $fRecord; // the last record with a key wins + } + } + foreach ($records as &$record) { + $lk = self::joinKey($record[$join['localKey']] ?? null); + $record[$join['alias']] = $lk !== null ? ($foreignIndexed[$lk] ?? null) : null; + } + unset($record); + } + return $records; + } + + /** + * Join identity of a value (only to compare, never to query): whole numbers in the int64 range, + * 5.0 and the canonical string "5" are the same as 5 (record keys join with user_id values); + * other floats by their bits (a float used as an array key was truncated: 1.5 joined 1); other + * strings as themselves ("007" is not 7, "1.5" is not 1.5); booleans only booleans (true joined 1 + * and "1"); null and arrays never join. + * @return int|string|null + */ + private static function joinKey($v) { + if (is_int($v)) { + return $v; + } + if (is_float($v)) { + if (is_finite($v) && floor($v) === $v && $v >= -9.2233720368547758E18 && $v < 9.2233720368547758E18) { + return (int)$v; + } + return 'd' . bin2hex(pack('E', $v + 0.0)); + } + if (is_string($v)) { + if (preg_match('/\A(0|-?[1-9][0-9]*)\z/', $v) === 1 && (string)(int)$v === $v) { + return (int)$v; + } + return 's' . $v; + } + if (is_bool($v)) { + return $v ? 'b1' : 'b0'; + } + return null; + } + + private function applyGroupBy(array $records): array { + $groups = []; + // Exact identity (as distinct()): json_encode() followed serialize_precision and split -0.0 from 0.0 + $valueKey = $this->kernel()->values->valueKeyFunction(); + foreach ($records as $record) { + $value = $record[$this->groupByField] ?? null; + $groupKey = $valueKey(is_float($value) ? $value + 0.0 : $value); + if (!isset($groups[$groupKey])) { + $groups[$groupKey] = ['_group' => $value, '_items' => [], '_count' => 0]; + } + $groups[$groupKey]['_items'][] = $record; + $groups[$groupKey]['_count']++; + } + + foreach ($this->havingFilters as $having) { + if (!in_array($having['operator'], ['>', '<', '>=', '<=', '=', '!='], true)) { + continue; // unknown operator: the condition is ignored (for every group) + } + $groups = array_filter($groups, function ($group) use ($having) { + if ($having['aggregate'] === 'count') { + $aggValue = $group['_count']; + } elseif (!preg_match('/^(sum|avg|min|max):.+$/', $having['aggregate'])) { + return true; // unknown aggregate: ignored + } else { + [$aggType, $aggField] = explode(':', $having['aggregate'], 2); + $values = []; + foreach ($group['_items'] as $item) { + $number = \noneDB\Internal\Query\Values::numberOf($item[$aggField] ?? null); + if ($number !== null) { + $values[] = $number; + } + } + switch ($aggType) { + case 'sum': $aggValue = array_sum($values); break; + case 'avg': $aggValue = count($values) > 0 ? array_sum($values) / count($values) : 0; break; + case 'min': $aggValue = count($values) > 0 ? min($values) : null; break; + default: $aggValue = count($values) > 0 ? max($values) : null; break; + } + } + if ($aggValue === null) { + return false; + } + switch ($having['operator']) { + case '>': return $aggValue > $having['value']; + case '<': return $aggValue < $having['value']; + case '>=': return $aggValue >= $having['value']; + case '<=': return $aggValue <= $having['value']; + case '=': return $aggValue == $having['value']; + case '!=': return $aggValue != $having['value']; + default: return true; // unknown operator: ignored + } + }); + } + return array_values($groups); + } + + private function applySort(array $records): array { + if (empty($this->sortKeys) || count($records) < 2) { + return $records; + } + // Every sort field becomes two columns, kind and value (noneDB::sortKey), sorted by + // array_multisort with the same comparison as <=>. The original position is the last column, + // so records that compare equal keep their order (a stable sort on every PHP version). + $records = array_values($records); + $sortKey = $this->kernel()->values->sortKeyFunction(); + $columns = []; + foreach ($this->sortKeys as $sk) { + $kinds = []; + $values = []; + foreach ($records as $record) { + [$kinds[], $values[]] = $sortKey(is_array($record) && array_key_exists($sk['field'], $record) ? $record[$sk['field']] : null); + } + $order = $sk['order'] === 'desc' ? SORT_DESC : SORT_ASC; + array_push($columns, $kinds, $order, SORT_REGULAR, $values, $order, SORT_REGULAR); + } + $columns[] = array_keys($records); + array_multisort(...$columns); + $sorted = []; + foreach ($columns[count($columns) - 1] as $position) { + $sorted[] = $records[$position]; + } + return $sorted; + } + + private function applyProjection(array $records): array { + if (empty($this->selectFields) && empty($this->exceptFields)) { + return $records; + } + if ($this->groupByField !== null) { + // Group rows: _group, _count and _items in select()/except() apply to the group rows; other + // fields apply to the items of each group (select(['name']) must not empty a group) + $groupFields = ['_group', '_count', '_items']; + $select = $this->selectFields; + $except = $this->exceptFields; + $this->selectFields = array_values(array_diff($select, $groupFields)); + $this->exceptFields = array_values(array_diff($except, $groupFields)); + $groupBy = $this->groupByField; + $this->groupByField = null; + try { + foreach ($records as &$group) { + if (!empty($this->selectFields) || !empty($this->exceptFields)) { + $group['_items'] = $this->applyProjection($group['_items']); + } + $keep = array_values(array_intersect($select, $groupFields)); + if (!empty($keep)) { + if (!empty($this->selectFields)) { + $keep[] = '_items'; + } + $group = array_intersect_key($group, array_flip($keep)); + } + foreach (array_intersect($except, $groupFields) as $field) { + unset($group[$field]); + } + } + unset($group); + } finally { + $this->groupByField = $groupBy; + $this->selectFields = $select; + $this->exceptFields = $except; + } + return $records; + } + return array_map(function ($record) { + if (!empty($this->selectFields)) { + $filtered = []; + foreach ($this->selectFields as $field) { + if (array_key_exists($field, $record)) { + $filtered[$field] = $record[$field]; + } + } + if (isset($record['key']) && !array_key_exists('key', $filtered)) { + $filtered['key'] = $record['key']; + } + if ($this->distanceConfig !== null && isset($record['_distance']) && !array_key_exists('_distance', $filtered)) { + $filtered['_distance'] = $record['_distance']; + } + return $filtered; + } + foreach ($this->exceptFields as $field) { + unset($record[$field]); + } + return $record; + }, $records); + } + + /** + * How the query reads its records: per segment whether it is skipped (a global field index or + * the keys rule it out), read through an index (candidates) or scanned; how it sorts (top-k, full) and + * whether it stops early. Every record read is checked against every filter ("residual": always). + * Nothing is built or written; an index a query would build first counts as a scan. With $analyze the + * query runs too and "actual" reports what it read and returned. + * @return array + */ + public function explain(bool $analyze = false): array { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + $plan = $this->buildPlan(); + $info = $this->kernel()->planner->explainPlan($dbname, $plan); + $segments = $info['segments']; + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $topK = $this->topKSize($lateJoins); + if (!empty($this->spatialFilters)) { + $access = 'spatial'; + } elseif ($segments['total'] === 0 || $segments['skipped'] === $segments['total']) { + $access = 'none'; + } elseif ($segments['scanned'] === 0) { + $access = 'index'; + } elseif ($segments['indexed'] === 0) { + $access = 'scan'; + } else { + $access = 'mixed'; + } + $joins = []; + foreach ($this->joinConfigs as $join) { + $joins[] = ['db' => $join['foreignDb'], 'on' => $join['localKey'] . ' = ' . $join['foreignKey'], + 'strategy' => $join['foreignKey'] === 'key' ? 'key-lookup' : 'in-lookup', 'when' => $lateJoins ? 'after-slice' : 'before-sort']; + } + $result = [ + 'explain' => 1, + 'db' => $this->dbname, + 'sharded' => $info['sharded'], + 'access' => $access, + 'segments' => $segments, + 'plan' => $info['plan'], + 'planTruncated' => $info['planTruncated'] ?? false, + 'estimatedRead' => $info['estimatedRead'], + 'filter' => $this->keyOrderLookup() ? ['keys-in-filter-order'] : $plan, + 'residual' => true, + 'spatial' => empty($this->spatialFilters) ? null : array_column($this->spatialFilters, 'type'), + 'sort' => empty($this->sortKeys) ? 'none' : ($topK !== null ? 'top-k' : 'full'), + 'earlyExit' => empty($this->sortKeys) && $this->groupByField === null && $this->limitCount !== null + && (empty($this->joinConfigs) || $lateJoins) && empty($this->spatialFilters), + 'joins' => $joins, + 'groupBy' => $this->groupByField, + ]; + if ($analyze) { + $this->lastScan = null; + $start = microtime(true); + $rows = $this->get(); + $result['actual'] = [ + 'recordsRead' => $this->lastScan['recordsRead'] ?? null, + 'segmentsRead' => $this->lastScan === null ? null : $this->lastScan['segments'] - $this->lastScan['skipped'], + 'returned' => count($rows), + 'ms' => round((microtime(true) - $start) * 1000, 3), + ]; + } + return $result; + } + + /** + * Execute query and get all results + * @return array + */ + public function get(): array { + return $this->results(true); + } + + /** + * get() with or without select()/except() (aggregates read the fields themselves) + */ + private function results(bool $project): array { + if (!empty($this->joinConfigs)) { + // Every database of the query read in one consistent state (shared locks held together) + $databases = array_merge([$this->dbname], array_column($this->joinConfigs, 'foreignDb')); + return $this->kernel()->snapshot->withSnapshot($databases, function () use ($project) { + return $this->resultsLocked($project); + }); + } + return $this->resultsLocked($project); + } + + private function resultsLocked(bool $project): array { + // Joins only add fields: without groupBy() and without sorting by a join alias they run on the + // records left after sort, offset and limit, which also allows stopping early + $lateJoins = !empty($this->joinConfigs) && $this->groupByField === null && !$this->sortsByJoinAlias(); + $canEarlyExit = (empty($this->joinConfigs) || $lateJoins) && $this->groupByField === null && empty($this->sortKeys); + $results = $this->filteredRecords($canEarlyExit, null, $this->topKSize($lateJoins)); + $results = $this->applyDistanceCalculation($results); + if (!$lateJoins) { + $results = $this->applyJoins($results); + } + if ($this->groupByField !== null) { + $results = $this->applyGroupBy($results); // having() only applies to groups + } + $results = $this->applySort($results); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $results = array_slice($results, $this->offsetCount, $this->limitCount); + } + if ($lateJoins) { + $results = $this->applyJoins($results); + } + return $project ? $this->applyProjection($results) : $results; + } + + /** + * Size of the top-k a sorted query with a small limit needs (offset + limit), or null: no limit, a + * large one, grouping, joins before the sort, or a sort by withDistance()'s _distance + */ + private function topKSize(bool $lateJoins): ?int { + if (empty($this->sortKeys) || $this->limitCount === null || $this->groupByField !== null + || (!empty($this->joinConfigs) && !$lateJoins) || $this->distanceConfig !== null) { + return null; + } + $k = $this->offsetCount + $this->limitCount; + return $k <= 1000 ? max(1, $k) : null; + } + + private function sortsByJoinAlias(): bool { + $aliases = array_column($this->joinConfigs, 'alias'); + foreach ($this->sortKeys as $sk) { + if (in_array($sk['field'], $aliases, true)) { + return true; + } + } + return false; + } + + /** + * The records a write (update/delete/removeFields) applies to, one segment at a time: $apply($dbname, + * segment, [key => record]) once a segment's matches are complete. Filters, sort, offset and limit apply; + * select/except/join don't matter. Without sort, spatial filters and a key lookup the matches stream from + * the scan (one segment held); otherwise the target keys come first and their records are read segment by + * segment. Caller holds the exclusive lock. + */ + private function eachTargetSegment(callable $apply): void { + if ($this->groupByField !== null) { + throw new \LogicException('noneDB: update(), delete() and removeFields() cannot be combined with groupBy()'); + } + $kernel = $this->kernel(); + $dbname = $kernel->layout->normalizeDbName($this->dbname); + $kernel->buffer->flushPendingBuffers($dbname); + // A missing database has no segments: nothing is written, and the filters are still checked + if (!empty($this->sortKeys) || !empty($this->spatialFilters) || $this->keyOrderLookup()) { + foreach ($kernel->findFilter->segmentMatches($dbname, ['key' => $this->targetKeys()]) as $segment => $records) { + $apply($dbname, $segment, $records); + } + return; + } + $size = $kernel->segments->isSharded($dbname) ? $kernel->segments->shardSizeOf($dbname) : null; + $segment = null; + $held = []; + $this->eachMatch(function ($key, array $record) use ($apply, $dbname, $size, &$segment, &$held) { + $of = $size === null ? null : intdiv($key, $size); + if (!empty($held) && ($of !== $segment || count($held) >= \noneDB\Internal\Storage\RecordIO::BATCH)) { + $apply($dbname, $segment, $held); // the scan is past that segment, or holds a batch of it + $held = []; + } + $segment = $of; + $held[$key] = $record; + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + if (!empty($held)) { + $apply($dbname, $segment, $held); + } + } + + /** + * Keys of the records a write applies to when sort, spatial filters or a key lookup decide (see + * eachTargetSegment()) + * @return int[] + */ + private function targetKeys(): array { + $records = $this->applySort($this->applyDistanceCalculation($this->filteredRecords(empty($this->sortKeys)))); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $records = array_slice($records, $this->offsetCount, $this->limitCount); + } + $keys = []; + foreach ($records as $record) { + if (isset($record['key']) && is_int($record['key'])) { + $keys[] = $record['key']; + } + } + return $keys; + } + + /** + * Run a write under the database's exclusive lock, so selecting and writing are one atomic step + * ($op: the operation, for getDurability()) + */ + private function withWriteLock(callable $fn, string $op) { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withLock($dbname, true, $fn, $op); + } + + /** + * Get first matching record + * @return array|null + */ + public function first(): ?array { + $originalLimit = $this->limitCount; + $this->limitCount = $originalLimit === null ? 1 : min(1, $originalLimit); // limit(0): nothing + try { + $results = $this->get(); + } finally { + $this->limitCount = $originalLimit; + } + return $results[0] ?? null; + } + + /** + * Get last matching record + * @return array|null + */ + public function last(): ?array { + if (!$this->streamsResults()) { + $results = $this->get(); + return count($results) > 0 ? $results[count($results) - 1] : null; + } + $last = null; + $this->eachResult(function (array $record) use (&$last) { + $last = $record; + }); + return $last === null ? null : $this->applyProjection([$last])[0]; + } + + /** + * Count matching records (groups, with groupBy()) + * @return int + */ + public function count(): int { + if ($this->isUnfiltered()) { + return $this->db->count($this->dbname, 0); + } + if ($this->groupByField !== null) { + return count($this->get()); + } + if (empty($this->spatialFilters) && !$this->keyOrderLookup()) { + return $this->countMatches(); // record by record, none kept in memory + } + $results = $this->filteredRecords(true); + if ($this->offsetCount > 0 || $this->limitCount !== null) { + $results = array_slice($results, $this->offsetCount, $this->limitCount); + } + return count($results); + } + + private function countMatches(): int { + $dbname = $this->kernel()->layout->normalizeDbName($this->dbname); + return $this->kernel()->locks->withReadLock($dbname, function () { + $count = 0; + $this->eachMatch(function () use (&$count) { + $count++; + }, $this->limitCount === null, $this->offsetCount, $this->limitCount); + return $count; + }); + } + + /** + * True when count() can use the stored counters + * @return bool + */ + private function isUnfiltered(): bool { + return empty($this->whereFilters) + && empty($this->orWhereFilters) + && !$this->hasAdvancedFilters() + && empty($this->spatialFilters) + && $this->groupByField === null + && empty($this->havingFilters) + && $this->limitCount === null + && $this->offsetCount === 0; + } + + /** + * Check if any records match + * @return bool + */ + public function exists(): bool { + if ($this->limitCount === 0) { + return false; + } + if ($this->groupByField !== null || !empty($this->havingFilters)) { + return $this->first() !== null; // having() may drop every group: no early exit + } + // Sort, joins and projection don't change whether a record exists: stop at offset + 1 matches + return count($this->filteredRecords(true, $this->offsetCount + 1)) > $this->offsetCount; + } + +} diff --git a/src/noneDBQuery/Filters.php b/src/noneDBQuery/Filters.php new file mode 100644 index 0000000..c172dcd --- /dev/null +++ b/src/noneDBQuery/Filters.php @@ -0,0 +1,301 @@ + 0 && $operators !== count($value)) { + throw new \InvalidArgumentException('noneDB: a filter value cannot mix operators ($gt, $in, ...) with plain keys'); + } + return $operators > 0; + } + + /** + * Compile where()/orWhere() groups once per query run: operators are validated and patterns + * turned into regexes here instead of for every record. + * @return array[] per group a list of [field, operator or null for a plain value, operand] + * @throws \InvalidArgumentException for unknown operators or invalid operands + */ + private function compileGroups(array $groups): array { + $compiled = []; + foreach ($groups as $group) { + $conditions = []; + foreach ($group as $field => $value) { + if ($this->isOperatorSpec($value)) { + foreach ($value as $operator => $operand) { + $conditions[] = [(string)$field, $operator, $this->compileOperand($operator, $operand)]; + } + } elseif ((string)$field === 'key') { + // As find(): key 5, "5" or a list of them (only one plain where() went through find()) + $conditions[] = ['key', '#keys', array_flip($this->kernel()->findFilter->normalizeKeyFilter($value))]; + } else { + $conditions[] = [(string)$field, null, $value]; + } + } + $compiled[] = $conditions; + } + return $compiled; + } + + private function compileOperand(string $operator, $operand) { + switch ($operator) { + case '$in': + case '$nin': + if (!is_array($operand)) { + throw new \InvalidArgumentException("noneDB: {$operator} needs an array"); + } + return $this->membership($operand); + case '$like': + return $this->likeSpec($operand); + case '$regex': + return self::regexOperand($operand); + case '$gt': + case '$gte': + case '$lt': + case '$lte': + case '$ne': + case '$eq': + case '$exists': + case '$contains': + return $operand; + default: + throw new \InvalidArgumentException("noneDB: unknown operator '{$operator}' (supported: " . implode(', ', self::OPERATORS) . ')'); + } + } + + /** + * Check if a record matches one compiled where()/orWhere() group (AND of its conditions). + * Plain values compare with ===; operator conditions apply the operator. + */ + private function matchesGroup(array $record, array $conditions): bool { + foreach ($conditions as [$field, $operator, $operand]) { + if ($operator === null) { + if (!array_key_exists($field, $record) || $record[$field] !== $operand) { + return false; + } + } elseif ($operator === '#keys') { + if (!isset($record['key']) || !is_int($record['key']) || !isset($operand[$record['key']])) { + return false; + } + } elseif (!$this->matchesOperator($record, $field, $operator, $operand)) { + return false; + } + } + return true; + } + + /** + * where() groups AND-ed, OR any orWhere() group. Without where(), only orWhere() groups decide. + * @param array[] $where compiled where() groups + * @param array[] $orWhere compiled orWhere() groups + */ + private function matchesWhereLogic(array $record, array $where, array $orWhere): bool { + if (!empty($where)) { + $all = true; + foreach ($where as $conditions) { + if (!$this->matchesGroup($record, $conditions)) { + $all = false; + break; + } + } + if ($all) { + return true; + } + } + foreach ($orWhere as $conditions) { + if ($this->matchesGroup($record, $conditions)) { + return true; + } + } + return false; + } + + /** + * Compiled like()/$like pattern ("^" start anchor, trailing "$" end anchor, rest literal) + */ + private function likeSpec($pattern): array { + if (!is_scalar($pattern)) { + throw new \InvalidArgumentException('noneDB: like pattern must be a string'); + } + return \noneDB\Internal\Query\Values::compileLike((string)$pattern); + } + + /** + * Regex for a $regex operand: the pattern body, case-insensitive (delimiter added here) + */ + private static function regexOperand($pattern): string { + if (!is_string($pattern) || $pattern === '') { + throw new \InvalidArgumentException('noneDB: $regex needs a non-empty string pattern'); + } + // Escape the delimiter, but not where "~" is already escaped: skip every "\x" pair first (a "\\" + // before "~" is an escaped backslash, the "~" still needs escaping) + $body = preg_replace('/\\\\.(*SKIP)(*FAIL)|~/s', '\\~', $pattern); + $regex = '~' . $body . '~i' . (preg_match('//u', $pattern) === 1 ? 'u' : ''); + if (@preg_match($regex, '') === false) { + throw new \InvalidArgumentException("noneDB: invalid \$regex pattern '{$pattern}'"); + } + return $regex; + } + + /** + * Apply a single comparison operator (operand compiled by compileOperand(): $like/$regex are regexes) + */ + private function matchesOperator(array $record, string $field, string $operator, $operand): bool { + $hasField = array_key_exists($field, $record); + $fieldValue = $hasField ? $record[$field] : null; + + switch ($operator) { + case '$gt': + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; + return $c !== null && $c > 0; + case '$gte': + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; + return $c !== null && $c >= 0; + case '$lt': + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; + return $c !== null && $c < 0; + case '$lte': + $c = $hasField ? \noneDB\Internal\Query\Values::compareValues($fieldValue, $operand) : null; + return $c !== null && $c <= 0; + case '$ne': + return !$hasField || $fieldValue !== $operand; + case '$eq': + return $hasField && $fieldValue === $operand; + case '$in': + return $hasField && $this->isMember($fieldValue, $operand); + case '$nin': + return !$hasField || !$this->isMember($fieldValue, $operand); + case '$exists': + return $operand ? $hasField : !$hasField; + case '$like': + if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { + return false; + } + return \noneDB\Internal\Query\Values::likeMatch($operand, (string)$fieldValue); + case '$regex': + if (!$hasField || !is_scalar($fieldValue) || is_bool($fieldValue)) { + return false; + } + return \noneDB\Internal\Query\Values::regexMatch($operand, (string)$fieldValue); + case '$contains': + if (!$hasField) { + return false; + } + if (is_array($fieldValue)) { + return in_array($operand, $fieldValue, true); + } + if (is_string($fieldValue) && is_scalar($operand) && !is_bool($operand)) { + return (string)$operand === '' || strpos($fieldValue, (string)$operand) !== false; + } + return false; + default: + throw new \InvalidArgumentException("noneDB: unknown operator '{$operator}' (supported: " . implode(', ', self::OPERATORS) . ')'); + } + } + + /** + * Check if a record matches all the other filters (whereNot, whereIn, whereNotIn, like, notLike, + * between, notBetween, search) + */ + private function matchesAdvancedFilters(array $record): bool { + foreach ($this->whereNotFilters as $fields) { + foreach ($fields as $conditions) { + if ($this->matchesGroup($record, $conditions)) { + return false; + } + } + } + + foreach ($this->whereInFilters as $filter) { + if (!array_key_exists($filter['field'], $record)) return false; + if (!$this->isMember($record[$filter['field']], $filter['values'])) return false; + } + + foreach ($this->whereNotInFilters as $filter) { + if (array_key_exists($filter['field'], $record) && $this->isMember($record[$filter['field']], $filter['values'])) { + return false; + } + } + + foreach ($this->likeFilters as $like) { + $value = $record[$like['field']] ?? null; + if (!is_scalar($value) || is_bool($value)) return false; + if (!\noneDB\Internal\Query\Values::likeMatch($like['like'], (string)$value)) return false; + } + + foreach ($this->notLikeFilters as $notLike) { + $value = $record[$notLike['field']] ?? null; + if (is_scalar($value) && !is_bool($value) && \noneDB\Internal\Query\Values::likeMatch($notLike['like'], (string)$value)) { + return false; + } + } + + foreach ($this->betweenFilters as $between) { + if (!array_key_exists($between['field'], $record)) return false; + $value = $record[$between['field']]; + $lo = \noneDB\Internal\Query\Values::compareValues($value, $between['min']); + $hi = \noneDB\Internal\Query\Values::compareValues($value, $between['max']); + if ($lo === null || $hi === null || $lo < 0 || $hi > 0) return false; + } + + foreach ($this->notBetweenFilters as $notBetween) { + if (!array_key_exists($notBetween['field'], $record)) continue; + $value = $record[$notBetween['field']]; + $lo = \noneDB\Internal\Query\Values::compareValues($value, $notBetween['min']); + $hi = \noneDB\Internal\Query\Values::compareValues($value, $notBetween['max']); + if ($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0) return false; + } + + foreach ($this->searchFilters as $search) { + $term = (string)$search['term']; + if ($term === '') continue; + $searchFields = empty($search['fields']) ? array_keys($record) : $search['fields']; + $found = false; + foreach ($searchFields as $field) { + if (empty($search['fields']) && ($field === 'key' || $field === '_distance')) continue; // not content + $value = $record[$field] ?? null; + if (!is_scalar($value) || is_bool($value)) continue; + if (self::containsIgnoreCase((string)$value, $term)) { + $found = true; + break; + } + } + if (!$found) return false; + } + + return true; + } + + /** + * Case-insensitive substring test with Unicode case folding when mbstring is available + */ + private static function containsIgnoreCase(string $haystack, string $needle): bool { + if (\noneDB\Internal\Query\Values::hasMbstring() && preg_match('//u', $haystack) === 1 && preg_match('//u', $needle) === 1) { + return strpos(\noneDB\Internal\Query\Values::foldCase($haystack), \noneDB\Internal\Query\Values::foldCase($needle)) !== false; + } + return stripos($haystack, $needle) !== false; + } + + private function hasAdvancedFilters(): bool { + return !empty($this->whereNotFilters) || !empty($this->whereInFilters) || !empty($this->whereNotInFilters) + || !empty($this->likeFilters) || !empty($this->notLikeFilters) || !empty($this->betweenFilters) + || !empty($this->notBetweenFilters) || !empty($this->searchFilters); + } + +} diff --git a/src/noneDBQuery/SpatialChain.php b/src/noneDBQuery/SpatialChain.php new file mode 100644 index 0000000..42ccf17 --- /dev/null +++ b/src/noneDBQuery/SpatialChain.php @@ -0,0 +1,74 @@ +spatialFilters[] = ['type' => 'withinDistance', 'field' => $field, 'lon' => $lon, 'lat' => $lat, 'distance' => $distanceMeters]; + return $this; + } + + /** + * Filter records within a bounding box + * @param string $field GeoJSON field name + * @param float $minLon Minimum longitude + * @param float $minLat Minimum latitude + * @param float $maxLon Maximum longitude + * @param float $maxLat Maximum latitude + * @return self + */ + public function withinBBox(string $field, float $minLon, float $minLat, float $maxLon, float $maxLat): self { + $this->spatialFilters[] = ['type' => 'withinBBox', 'field' => $field, 'minLon' => $minLon, 'minLat' => $minLat, 'maxLon' => $maxLon, 'maxLat' => $maxLat]; + return $this; + } + + /** + * The K records nearest to a point among those matching the other filters (adds _distance in meters) + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @param int $k Number of nearest records + * @return self + */ + public function nearest(string $field, float $lon, float $lat, int $k): self { + $this->spatialFilters[] = ['type' => 'nearest', 'field' => $field, 'lon' => $lon, 'lat' => $lat, 'k' => max(0, $k)]; + return $this; + } + + /** + * Filter records within a polygon + * @param string $field GeoJSON field name + * @param array $polygon GeoJSON Polygon geometry + * @return self + */ + public function withinPolygon(string $field, array $polygon): self { + $this->spatialFilters[] = ['type' => 'withinPolygon', 'field' => $field, 'polygon' => $polygon]; + return $this; + } + + /** + * Add _distance to every result: meters from the point to the geometry, measured like + * withinDistance()/nearest() (0 inside a polygon, otherwise to the nearest point of the geometry). + * It is computed before sorting, so sort('_distance') works. + * @param string $field GeoJSON field name + * @param float $lon Center longitude + * @param float $lat Center latitude + * @return self + */ + public function withDistance(string $field, float $lon, float $lat): self { + $this->distanceConfig = ['field' => $field, 'lon' => $lon, 'lat' => $lat]; + return $this; + } + +} diff --git a/src/noneDBQuery/Writes.php b/src/noneDBQuery/Writes.php new file mode 100644 index 0000000..251daf5 --- /dev/null +++ b/src/noneDBQuery/Writes.php @@ -0,0 +1,83 @@ + count] + */ + public function update(array $set): array { + $error = $this->kernel()->crud->updateSetError($set); + if ($error !== null) { + return $error; + } + return $this->withWriteLock(function () use ($set) { + $n = 0; + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use ($set, &$n) { + $n += $this->kernel()->crud->updateRecords($dbname, $segment, $records, $set); + }); + return ['n' => $n]; + }, 'update'); + } + + /** + * Delete matching records (sort/offset/limit select which ones) + * @return array ["n" => count] + */ + public function delete(): array { + return $this->withWriteLock(function () { + $n = 0; + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use (&$n) { + $n += $this->kernel()->writer->deleteSegmentRecords($dbname, $segment, $records); + }); + return ['n' => $n]; + }, 'delete'); + } + + /** + * Remove fields from the stored matching records (permanent) + * @param array $fields Fields to remove ("key" can't be removed) + * @return array ["n" => count of updated records, "fields_removed" => array of removed fields] + */ + public function removeFields(array $fields): array { + if (empty($fields)) { + return ['n' => 0, 'fields_removed' => [], 'error' => 'No fields specified']; + } + $fieldsToRemove = array_values(array_filter($fields, function ($f) { return $f !== 'key'; })); + if (empty($fieldsToRemove)) { + return ['n' => 0, 'fields_removed' => [], 'error' => 'Cannot remove reserved field: key']; + } + + return $this->withWriteLock(function () use ($fieldsToRemove) { + $n = 0; + $removed = []; + // Fields are removed from the stored records, never from a projected/joined result + $this->eachTargetSegment(function (string $dbname, ?int $segment, array $records) use ($fieldsToRemove, &$n, &$removed) { + $replacements = []; + foreach ($records as $key => $record) { + $changed = false; + foreach ($fieldsToRemove as $field) { + if (array_key_exists($field, $record)) { + unset($record[$field]); + $removed[$field] = true; + $changed = true; + } + } + if ($changed) { + $replacements[$key] = $record; + } + } + $n += $this->kernel()->writer->updateSegmentRecords($dbname, $segment, $replacements); + }); + return [ + 'n' => $n, + 'fields_removed' => array_values(array_filter($fieldsToRemove, function ($f) use ($removed) { return isset($removed[$f]); })) + ]; + }, 'removeFields'); + } +} diff --git a/tests/Concurrency/BackupConcurrencyTest.php b/tests/Concurrency/BackupConcurrencyTest.php new file mode 100644 index 0000000..defbe1f --- /dev/null +++ b/tests/Concurrency/BackupConcurrencyTest.php @@ -0,0 +1,231 @@ +markTestSkipped('needs proc_open and POSIX signals'); + } + } + + protected function tearDown(): void + { + foreach ($this->outside as $path) { + $this->removeDirectoryTree($path); + } + $this->outside = []; + parent::tearDown(); + } + + private function outsidePath(string $prefix): string + { + $path = sys_get_temp_dir() . '/nonedb_' . $prefix . '_' . bin2hex(random_bytes(6)); + $this->outside[] = $path; + return $path; + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + /** Start a child without a shell (its pid is PHP's); [process, stdout pipe] */ + private function start(string $body): array + { + $proc = proc_open([PHP_BINARY, $this->childScript($body)], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + return [$proc, $pipes[1], $pipes[2]]; + } + + private function finish(array $child): string + { + [$proc, $out, $err] = $child; + $output = stream_get_contents($out) . stream_get_contents($err); + fclose($out); + fclose($err); + proc_close($proc); + return trim($output); + } + + private function waitFor(string $file): void + { + $deadline = microtime(true) + 20; + while (!is_file($file)) { + $this->assertLessThan($deadline, microtime(true), "waiting for $file"); + usleep(10000); + } + } + + /** A child that backs up 'u' and waits at the first copied file until $release exists */ + private function pausedBackup(string $dir, string $paused, string $release, string $key = self::SECRET): array + { + $child = $this->start(' + $db = new \noneDB(["secretKey" => ' . var_export($key, true) . ', "dbDir" => ' . var_export($this->testDbDir, true) . ']); + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { + static $done = false; + if ($p === "backup:copied" && !$done) { + $done = true; + touch(' . var_export($paused, true) . '); + while (!is_file(' . var_export($release, true) . ')) { usleep(10000); } + } + }); + $db->backup("u", ' . var_export($dir, true) . '); + echo "COMPLETED"; + '); + $this->waitFor($paused); + return $child; + } + + /** @test the copy is the state of one moment: whole insert calls, a prefix of each writer's calls */ + public function aBackupTakenWhileWritersRunIsTheStateOfOneMoment(): void + { + $config = ['shardSize' => 50, 'lockTimeout' => 30]; + $db = $this->db($config); + $db->insert('w', ['writer' => 0, 'batch' => 0, 'i' => 0]); + $stop = $this->outsidePath('stop'); + $writers = []; + for ($w = 1; $w <= 3; $w++) { + $writers[] = $this->start(' + $db = ' . $this->childDb($config) . '; + for ($b = 1; $b <= 400 && !is_file(' . var_export($stop, true) . '); $b++) { + $db->insert("w", array_map(function ($i) use ($b) { return ["writer" => ' . $w . ', "batch" => $b, "i" => $i]; }, range(0, 9))); + } + echo "DONE"; + '); + } + $backups = []; + for ($n = 0; $n < 3; $n++) { + usleep(150000); + $dir = $this->outsidePath('backup'); + $db->backup('w', $dir); + $backups[] = $dir; + } + touch($stop); + foreach ($writers as $writer) { + $this->assertSame('DONE', $this->finish($writer)); + } + foreach ($backups as $n => $dir) { + $copy = 'copy' . $n; + $result = $db->restore($dir, $copy, ['force' => true]); + $records = $db->find($copy, 0); + $manifest = json_decode(file_get_contents("$dir/manifest.json"), true); + $this->assertSame($manifest['records'], count($records), "backup $n: the counters of the same moment"); + $this->assertSame($result['records'], count($records)); + $groups = []; + foreach ($records as $record) { + $groups[$record['writer']][$record['batch']][] = $record['i']; + } + unset($groups[0]); + foreach ($groups as $writer => $batches) { + ksort($batches); + $this->assertSame(range(1, count($batches)), array_keys($batches), "backup $n, writer $writer: a prefix of its inserts"); + foreach ($batches as $batch => $is) { + sort($is); + $this->assertSame(range(0, 9), $is, "backup $n, writer $writer, batch $batch: a whole insert"); + } + } + $this->assertTrue($db->checkIntegrity($copy)['ok'], "backup $n"); + } + $this->assertGreaterThan(0, count($db->find('copy2', 0)) - 1, 'the writers ran while the backups were taken'); + } + + /** @test §10.2 step 2: maintenance fails with MAINTENANCE_BUSY while a backup holds STORE.LOCK */ + public function maintenanceIsBusyWhileABackupRuns(): void + { + $db = $this->db(); + $db->insert('u', array_map(function ($n) { return ['v' => $n]; }, range(0, 4))); + $store = file_get_contents($this->testDbDir . 'STORE'); + $dir = $this->outsidePath('backup'); + $paused = $this->outsidePath('paused'); + $release = $this->outsidePath('release'); + $child = $this->pausedBackup($dir, $paused, $release); + + $register = \noneDB::registerKey($this->testDbDir, self::SECRET, 'another-registered-key-1'); + $this->assertSame([false, \noneDB::MAINTENANCE_BUSY], [$register['success'], $register['code']]); + $rotate = \noneDB::migrateKey($this->testDbDir, self::SECRET, 'a-rotated-secret-key-22'); + $this->assertSame([false, \noneDB::MAINTENANCE_BUSY], [$rotate['success'], $rotate['code']]); + touch($release); + $this->assertSame('COMPLETED', $this->finish($child)); + $this->assertSame($store, file_get_contents($this->testDbDir . 'STORE'), 'nothing written by the busy maintenance'); + $this->assertSame($store, file_get_contents("$dir/store.json"), 'the STORE record of the locked moment'); + + // A key registered before a backup is in its store.json + $register = \noneDB::registerKey($this->testDbDir, self::SECRET, 'another-registered-key-1'); + $this->assertTrue($register['success'], (string)$register['error']); + $second = $this->outsidePath('backup'); + $this->db()->backup('u', $second); + $keys = json_decode(file_get_contents("$second/store.json"), true)['keys']; + $this->assertContains($this->keyCheckOf('another-registered-key-1'), $keys); + $this->assertContains($this->keyCheckOf(), $keys); + + // After a rotation the old key backs up nothing + $rotate = \noneDB::migrateKey($this->testDbDir, self::SECRET, 'a-rotated-secret-key-22'); + $this->assertTrue($rotate['success'], (string)$rotate['error']); + $third = $this->outsidePath('backup'); + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $this->code(function () use ($third) { $this->db()->backup('u', $third); })); + $this->assertFileDoesNotExist($third); + $rotated = new \noneDB(['secretKey' => 'a-rotated-secret-key-22', 'dbDir' => $this->testDbDir]); + $this->assertSame(5, $rotated->backup('u', $third)['records']); + } + + /** @test §10.2 step 2: writers of the database wait; readers and other databases are not affected */ + public function aWriterWaitsForABackup(): void + { + $db = $this->db(); + $db->insert('u', array_map(function ($n) { return ['v' => $n]; }, range(0, 4))); + $db->insert('other', ['v' => 'o']); + $dir = $this->outsidePath('backup'); + $paused = $this->outsidePath('paused'); + $release = $this->outsidePath('release'); + $child = $this->pausedBackup($dir, $paused, $release); + + \noneDB::clearStaticCache(); + $writer = $this->db(['lockTimeout' => 0.3]); + $this->assertSame(\noneDB::LOCK_FAILED, $this->code(function () use ($writer) { $writer->insert('u', ['v' => 'late']); })); + $this->assertSame(5, $writer->count('u'), 'a reader shares the lock'); + $writer->insert('other', ['v' => 'p']); + $this->assertSame(2, $writer->count('other'), 'another database is not affected'); + touch($release); + $this->assertSame('COMPLETED', $this->finish($child)); + $writer->insert('u', ['v' => 'late']); + $this->assertSame(6, $writer->count('u')); + $this->assertSame(5, json_decode(file_get_contents("$dir/manifest.json"), true)['records']); + } + + /** @test a backup killed while it holds its locks releases them, and its copy is refused */ + public function aKilledBackupReleasesItsLocksAndIsRefused(): void + { + $db = $this->db(); + $db->insert('u', array_map(function ($n) { return ['v' => $n]; }, range(0, 4))); + $dir = $this->outsidePath('backup'); + $paused = $this->outsidePath('paused'); + $release = $this->outsidePath('release'); + $child = $this->pausedBackup($dir, $paused, $release); + $this->assertSame(\noneDB::MAINTENANCE_BUSY, \noneDB::registerKey($this->testDbDir, self::SECRET, 'another-registered-key-1')['code']); + posix_kill(proc_get_status($child[0])['pid'], 9); + $this->finish($child); + + $register = \noneDB::registerKey($this->testDbDir, self::SECRET, 'another-registered-key-1'); + $this->assertTrue($register['success'], (string)$register['error']); + $this->assertFileDoesNotExist("$dir/manifest.json"); + $this->assertSame(\noneDB::RESTORE_REFUSED, $this->code(function () use ($dir) { $this->db()->restore($dir, 'u'); })); + $this->assertSame(5, $this->db()->count('u')); + } +} diff --git a/tests/Regression/ConcurrencyRegressionTest.php b/tests/Concurrency/ConcurrencyRegressionTest.php similarity index 70% rename from tests/Regression/ConcurrencyRegressionTest.php rename to tests/Concurrency/ConcurrencyRegressionTest.php index ab181fc..779e93c 100644 --- a/tests/Regression/ConcurrencyRegressionTest.php +++ b/tests/Concurrency/ConcurrencyRegressionTest.php @@ -1,6 +1,8 @@ assertCount(360, array_unique($keys), "$label: keys are unique"); $this->assertCount(360, $pairs, "$label: no record lost or duplicated"); $this->assertSame(360, $check->count('c'), "$label: count() agrees"); + // Keys are assigned in order and never skipped when nothing failed: exactly 0..359 + $sorted = $keys; + sort($sorted); + $this->assertSame(range(0, 359), $sorted, "$label: keys 0..359"); + // Each writer inserts one record after the other: its records got increasing keys + $byWriter = []; + foreach ($all as $r) { + $byWriter[$r['w']][$r['i']] = $r['key']; + } + ksort($byWriter); + $this->assertSame(range(0, 5), array_keys($byWriter), "$label: writer 0 and the 5 parallel ones"); + foreach ($byWriter as $w => $keysOfWriter) { + ksort($keysOfWriter); + $this->assertSame(range(0, 59), array_keys($keysOfWriter), "$label: writer $w"); + $inOrder = array_values($keysOfWriter); + $sortedKeys = $inOrder; + sort($sortedKeys); + $this->assertSame($sortedKeys, $inOrder, "$label: writer $w got increasing keys"); + } } /** @test K-01 */ @@ -114,20 +135,25 @@ public function longLivedInstanceSeesWritesOfOtherProcesses(): void } /** @test Y-13 */ - public function migrationByAnotherProcessIsNotSplitBrain(): void + public function aSwitchToShardsByAnotherProcessIsNotSplitBrain(): void { $a = $this->db(['shardSize' => 20]); $a->insert('m', ['i' => -1]); $this->assertSame(1, count($a->find('m', 0))); // A has "not sharded" cached - // B pushes the database over the threshold: it migrates to shards + // B pushes the database over the threshold: it switches to shards $this->runChild('$db = ' . $this->childDb(['shardSize' => 20]) . '; $r = []; for ($i = 0; $i < 30; $i++) { $r[] = ["i" => $i]; } $db->insert("m", $r);'); - $this->assertTrue(file_exists($this->dbBasePath('m') . '.nonedb.meta')); + $this->assertTrue(file_exists($this->dbPath('m', 'shards'))); + $this->assertSame([], preg_grep('#/segments/main\.jsonl$#', $this->dbFiles('m')), 'set up: no non-sharded file left'); $a->insert('m', ['i' => 999]); - $this->assertFalse(file_exists($this->dbBasePath('m') . '.nonedb'), 'A must not recreate the non-sharded file'); + $this->assertSame([], preg_grep('#/segments/main\.jsonl$#', $this->dbFiles('m')), 'A must not recreate the non-sharded file'); \noneDB::clearStaticCache(); - $this->assertCount(32, $this->db()->find('m', 0)); + $all = $this->db()->find('m', 0); + $this->assertCount(32, $all); + // A's first record got key 0, B's 30 records 1..30, A's second insert key 31 + $this->assertSame(range(0, 31), array_column($all, 'key')); + $this->assertSame(array_merge([-1], range(0, 29), [999]), array_column($all, 'i')); } /** @test O-26 */ @@ -135,7 +161,7 @@ public function lockTimeoutIsHonoured(): void { $db = $this->db(); $db->insert('lk', ['a' => 1]); - $lock = $this->dbBasePath('lk') . '.nonedb.lock'; + $lock = $this->dbPath('lk', 'lock'); $holder = $this->childScript('$fp = fopen(' . var_export($lock, true) . ', "c+"); flock($fp, LOCK_EX); echo "locked\n"; fflush(STDOUT); sleep(4);'); $proc = proc_open([PHP_BINARY, $holder], [1 => ['pipe', 'w']], $pipes); // no shell: proc_terminate() reaches PHP $this->assertSame("locked", trim(fgets($pipes[1]))); @@ -146,10 +172,16 @@ public function lockTimeoutIsHonoured(): void $this->fail('expected a lock timeout'); } catch (\RuntimeException $e) { $this->assertStringContainsString('timed out', $e->getMessage()); + // docs/API.md: "Waiting longer than lockTimeout seconds throws RuntimeException with code noneDB::LOCK_FAILED" + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode()); } - $this->assertLessThan(3.5, microtime(true) - $start); + $elapsed = microtime(true) - $start; + $this->assertGreaterThanOrEqual(0.95, $elapsed, 'it waited for the lock timeout'); + $this->assertLessThan(3.5, $elapsed); fclose($pipes[1]); proc_terminate($proc); proc_close($proc); + \noneDB::clearStaticCache(); + $this->assertSame([['a' => 1, 'key' => 0]], $this->db()->find('lk', 0), 'nothing written without the lock'); } } diff --git a/tests/Integration/ConcurrencyTest.php b/tests/Concurrency/ConcurrencyTest.php similarity index 60% rename from tests/Integration/ConcurrencyTest.php rename to tests/Concurrency/ConcurrencyTest.php index abbf9e7..eee7bc7 100644 --- a/tests/Integration/ConcurrencyTest.php +++ b/tests/Concurrency/ConcurrencyTest.php @@ -1,6 +1,6 @@ noneDB->insert($dbName, ['index' => $i]); - $this->assertEquals(1, $result['n'], "Insert $i should succeed"); + $this->assertSame(['n' => 1], $result, "Insert $i should succeed"); } - // Verify all data + // Verify all data: record i got key i $all = $this->noneDB->find($dbName, 0); - $this->assertCount(50, $all); + $this->assertSame(range(0, 49), array_column($all, 'index')); + $this->assertSame(range(0, 49), array_column($all, 'key')); } /** @@ -43,10 +44,11 @@ public function rapidSequentialReads(): void ['id' => 3], ]); - // Perform rapid sequential reads + // Perform rapid sequential reads: always the same records + $expected = [['id' => 1, 'key' => 0], ['id' => 2, 'key' => 1], ['id' => 3, 'key' => 2]]; for ($i = 0; $i < 100; $i++) { $result = $this->noneDB->find($dbName, 0); - $this->assertCount(3, $result, "Read $i should return 3 records"); + $this->assertSame($expected, $result, "Read $i should return the 3 records"); } } @@ -64,8 +66,7 @@ public function insertThenImmediateRead(): void // Immediate read $result = $this->noneDB->find($dbName, ['index' => $i]); - $this->assertCount(1, $result, "Should find record immediately after insert (iteration $i)"); - $this->assertEquals($i, $result[0]['index']); + $this->assertSame([['index' => $i, 'key' => $i]], $result, "Should find record immediately after insert (iteration $i)"); } } @@ -88,8 +89,8 @@ public function updateThenImmediateRead(): void // Immediate read $result = $this->noneDB->find($dbName, ['counter' => $i]); - $this->assertCount(1, $result, "Should find updated record (iteration $i)"); - $this->assertEquals($i, $result[0]['counter']); + $this->assertSame([['counter' => $i, 'key' => 0]], $result, "Should find updated record (iteration $i)"); + $this->assertSame([], $this->noneDB->find($dbName, ['counter' => $i - 1]), 'the old version is gone'); } } @@ -106,7 +107,7 @@ public function clearStatCacheEffectiveness(): void // Read to populate stat cache $result1 = $this->noneDB->find($dbName, 0); - $this->assertEquals('initial', $result1[0]['value']); + $this->assertSame([['value' => 'initial', 'key' => 0]], $result1); // Update using API (direct file modification not supported in JSONL) $this->noneDB->update($dbName, [ @@ -117,12 +118,14 @@ public function clearStatCacheEffectiveness(): void // Read again - should get updated data $result2 = $this->noneDB->find($dbName, 0); - $this->assertEquals('modified', $result2[0]['value']); + $this->assertSame([['value' => 'modified', 'key' => 0]], $result2); } /** * @test - * v3.0: Multiple instances require flush/clear to sync JSONL index caches + * Instances see each other's writes without any manual cache clearing: shared static caches are + * validated against the database's lock state record, and with static caching disabled every instance keeps + * its own cache, validated the same way (docs/STORAGE_FORMAT.md §4.2) */ public function multipleInstancesConcurrent(): void { @@ -135,35 +138,35 @@ public function multipleInstancesConcurrent(): void 'autoCreateDB' => true ]; - // Create multiple instances with test config - $db1 = new \noneDB($testConfig); - $db2 = new \noneDB($testConfig); - $db3 = new \noneDB($testConfig); - - $reflector = new \ReflectionClass(\noneDB::class); - - // Insert from instance 1 - $db1->insert($dbName, ['from' => 'db1']); - $db1->flush($dbName); // Flush buffer to file - - // Read from instance 2 - fresh instance reads from file - $result2 = $db2->find($dbName, 0); - $this->assertCount(1, $result2); - - // Insert from instance 3 - $db3->insert($dbName, ['from' => 'db3']); - $db3->flush($dbName); // Flush buffer to file - - // Clear db1's index cache to force re-read - $cacheProperty = $reflector->getProperty('indexCache'); - if (PHP_VERSION_ID < 80100) { - $cacheProperty->setAccessible(true); // no effect since 8.1, deprecated in 8.5 + foreach (['shared cache' => true, 'separate caches' => false] as $case => $shared) { + \noneDB::clearStaticCache(); + if (!$shared) { + \noneDB::disableStaticCache(); + } + try { + $name = $dbName . ($shared ? '-a' : '-b'); + $db1 = new \noneDB($testConfig); + $db2 = new \noneDB($testConfig); + $db3 = new \noneDB($testConfig); + + $db1->insert($name, ['from' => 'db1']); + $this->assertSame([['from' => 'db1', 'key' => 0]], $db2->find($name, 0), $case); + $this->assertSame([['from' => 'db1', 'key' => 0]], $db1->find($name, 0), "$case: db1 has its index cached now"); + + $db3->insert($name, ['from' => 'db3']); + $expected = [['from' => 'db1', 'key' => 0], ['from' => 'db3', 'key' => 1]]; + $this->assertSame($expected, $db1->find($name, 0), "$case: db1 sees db3's insert"); + $this->assertSame($expected, $db2->find($name, 0), "$case: db2 sees db3's insert"); + + $db2->update($name, [['from' => 'db1'], ['set' => ['seen' => true]]]); + $db1->delete($name, ['from' => 'db3']); + $this->assertSame([['from' => 'db1', 'seen' => true, 'key' => 0]], $db3->find($name, 0), $case); + $db3->insert($name, ['from' => 'db3 again']); + $this->assertSame([0, 2], array_column($db1->find($name, 0), 'key'), "$case: the key of the deleted record is not reused"); + } finally { + \noneDB::enableStaticCache(); + } } - $cacheProperty->setValue($db1, []); - - // Read from instance 1 - fresh read with cleared cache - $result1 = $db1->find($dbName, 0); - $this->assertCount(2, $result1); } /** @@ -187,11 +190,8 @@ public function fileLockPreventsCorruption(): void // Verify final state $result = $this->noneDB->find($dbName, 0); - // Should have exactly one record - $this->assertCount(1, $result); - - // Should have the final value - $this->assertEquals(100, $result[0]['id']); + // Exactly one record, with the final value, still key 0 + $this->assertSame([['id' => 100, 'key' => 0]], $result); } /** @@ -230,8 +230,7 @@ public function emptyFileHandling(): void // Read from empty file $result = $this->noneDB->find($dbName, 0); - $this->assertIsArray($result); - $this->assertCount(0, $result); + $this->assertSame([], $result); } /** @@ -248,13 +247,12 @@ public function largeDataHandling(): void ]; $result = $this->noneDB->insert($dbName, $largeData); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); // Read back $found = $this->noneDB->find($dbName, 0); - $this->assertCount(1, $found); - $this->assertEquals(100000, strlen($found[0]['content'])); + $this->assertSame([$largeData + ['key' => 0]], $found); } /** @@ -269,14 +267,14 @@ public function writeReadWriteSequence(): void // Read $r1 = $this->noneDB->find($dbName, 0); - $this->assertCount(1, $r1); + $this->assertSame([['step' => 1, 'key' => 0]], $r1); // Write $this->noneDB->insert($dbName, ['step' => 2]); // Read $r2 = $this->noneDB->find($dbName, 0); - $this->assertCount(2, $r2); + $this->assertSame([['step' => 1, 'key' => 0], ['step' => 2, 'key' => 1]], $r2); // Write $this->noneDB->update($dbName, [ @@ -286,7 +284,8 @@ public function writeReadWriteSequence(): void // Final read $r3 = $this->noneDB->find($dbName, ['step' => 3]); - $this->assertCount(1, $r3); + $this->assertSame([['step' => 3, 'key' => 0]], $r3); + $this->assertSame([3, 2], array_column($this->noneDB->find($dbName, 0), 'step')); } /** @@ -306,8 +305,9 @@ public function getDBsDoesNotBlockOperations(): void $this->noneDB->find($dbName, ['index' => $i]); } - // Verify data integrity + // Verify data integrity: 1 initial + 10 loop inserts, keys 0..10 $all = $this->noneDB->find($dbName, 0); - $this->assertCount(11, $all); // 1 initial + 10 loop inserts + $this->assertSame(range(0, 10), array_column($all, 'key')); + $this->assertSame(range(0, 9), array_column(array_slice($all, 1), 'index')); } } diff --git a/tests/Concurrency/LockEpochTest.php b/tests/Concurrency/LockEpochTest.php new file mode 100644 index 0000000..503ea6c --- /dev/null +++ b/tests/Concurrency/LockEpochTest.php @@ -0,0 +1,231 @@ +dbPath($dbName, 'lock'))))[0]; + $parts = explode(':', $first, 2); + return $parts[1] ?? null; + } + + /** @test */ + public function sessionsThatWriteNothingKeepTheGeneration(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('noop', [['n' => 1], ['n' => 2]]); + $gen = $this->lockState('noop')['gen']; + + $db->checkDB('noop'); + $this->assertSame(0, $db->update('noop', [['n' => 99], ['set' => ['n' => 100]]])['n']); + $this->assertSame(0, $db->delete('noop', ['n' => 99])['n']); + $db->flush('noop'); + $this->assertFalse($db->createDB('noop')); + $db->find('noop', 0); + $db->count('noop'); + $this->assertSame($gen, $this->lockState('noop')['gen'], 'nothing was written'); + + $db->insert('noop', ['n' => 3]); + $this->assertGreaterThan($gen, $this->lockState('noop')['gen']); + } + + /** @test */ + public function writesWithoutTheExclusiveLockAreRefused(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('guard', ['n' => 1]); + $file = $this->dbPath('guard', 'data'); + $unlink = function (string $file) { + return $this->fsUnlink($file); + }; + try { + $unlink->call(\noneDB\Tests\Support\Testing::engine($db), $file); + $this->fail('no lock: refused'); + } catch (\LogicException $e) { + } + $withLock = function (callable $fn) { + return $this->withLock('guard', false, $fn); + }; + try { + $withLock->call(\noneDB\Tests\Support\Testing::engine($db), function () use ($unlink, $db, $file) { + $unlink->call(\noneDB\Tests\Support\Testing::engine($db), $file); + }); + $this->fail('shared lock: refused'); + } catch (\LogicException $e) { + } + $this->assertFileExists($file); + $this->assertSame(1, $db->count('guard')); + } + + /** @test */ + public function aRecreatedLockFileNeverValidatesAnOldCache(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('epo', [['n' => 0], ['n' => 1]]); + $db->insert('epo', ['n' => 2]); // a generation the child can reach again however a new lock file starts + $this->assertCount(3, $db->find('epo', 0)); // cached in this process + $gen = $this->lockState('epo')['gen']; + $this->assertGreaterThan(0, $gen); + + // Another process starts over with a new lock file and writes until the generation is the same again + $this->assertTrue(unlink($this->dbPath('epo', 'lock'))); + $out = $this->runChild(' + $db = ' . $this->childDb(['shardingEnabled' => false]) . '; + $lock = $argv[1]; + $gen = function () use ($lock) { return (int)explode(" ", trim((string)@file_get_contents($lock)))[0]; }; + for ($i = 0; $gen() < ' . ($gen - 1) . ' && $i < 50; $i++) { + $db->insert("epo", ["n" => 100 + $i]); + } + $db->delete("epo", ["n" => 0]); + echo $gen(); + ', [$this->dbPath('epo', 'lock')]); + $this->assertSame((string)$gen, $out, 'the child reached the same generation'); + + $ns = array_column($db->find('epo', 0), 'n'); + $this->assertNotContains(0, $ns, 'the cache from before the new lock file is not used'); + $this->assertContains(1, $ns); + } + + /** @test */ + public function replacingADataFileChangesTheEpochAppendingDoesNot(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('rep', [['n' => 1], ['n' => 2], ['n' => 3]]); + $epoch = $this->epoch('rep'); + $this->assertNotNull($epoch); + $db->insert('rep', ['n' => 4]); + $db->update('rep', [['n' => 4], ['set' => ['m' => 1]]]); + $this->assertSame($epoch, $this->epoch('rep'), 'appends keep the epoch'); + $db->compact('rep'); + $this->assertNotSame($epoch, $this->epoch('rep'), 'compaction replaced the data file'); + $epoch = $this->epoch('rep'); + $this->assertSame(4, $db->count('rep')); + $db->dropDB('rep'); + $this->assertNotSame($epoch, $this->epoch('rep'), 'drop removed the data files'); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$this->epoch('rep')); + } + + /** + * @test a new epoch comes before a data file is replaced or removed (STORAGE_FORMAT.md §5.2 INTENT for the + * change to the sharded layout, §6.2 for compaction, §8.2 for a drop) and for every new lock record (§4.2) + */ + public function theSwitchToShardsShardRemovalAndNewLockFilesChangeTheEpoch(): void + { + $db = $this->db(['shardSize' => 3, 'shardingEnabled' => false]); + $db->insert('mig', array_map(function ($n) { return ['n' => $n]; }, range(0, 7))); + $epoch = $this->epoch('mig'); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('mig')); + $this->assertNotSame($epoch, $this->epoch('mig'), 'the main data file was replaced by shards (generation switch)'); + + // Shard 1 (keys 3-5) emptied: compaction removes it + foreach ([3, 4, 5] as $n) { + $db->delete('mig', ['n' => $n]); + } + $epoch = $this->epoch('mig'); + $result = $db->compact('mig'); + $this->assertSame(2, $result['newShardCount']); + $this->assertFileDoesNotExist($this->dbPath('mig', 'data', 1)); + $this->assertNotSame($epoch, $this->epoch('mig')); + $this->assertSame([0, 1, 2, 6, 7], array_column($db->find('mig', 0), 'n')); + + // A new lock file (deleted while nothing runs: documented as harmless) gets a new epoch + $epoch = $this->epoch('mig'); + $gen = $this->lockState('mig')['gen']; + $this->removeLockFile('mig'); + $this->assertSame([0, 1, 2, 6, 7], array_column($this->db(['shardSize' => 3])->find('mig', 0), 'n')); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$this->epoch('mig')); + $this->assertNotSame($epoch, $this->epoch('mig')); + $this->assertLessThan($gen, $this->lockState('mig')['gen'], 'the generation starts over: only the epoch tells the caches apart'); + $this->assertSame([], $this->lockState('mig')['flags']); + } + + /** + * @test a record whose epoch isn't 8 lowercase hex digits (or that has no ":") does not match the grammar: + * it is damaged (STORAGE_FORMAT.md §4.2), treated as "J" by the next exclusive session (full recovery, + * §7.3) with a new epoch; the journal is cleared at the end of that session + */ + public function anInvalidEpochIsReplaced(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('bad', [['n' => 1], ['n' => 2]]); + \noneDB::clearStaticCache(); + foreach (['5:xyz', '5:0123456789', '5:GHIJKLMN', '5:', '7'] as $record) { + $this->writeLockState('bad', $record); + $this->assertDoesNotMatchRegularExpression('/^[0-9]+:[0-9a-f]{8}$/', trim((string)file_get_contents($this->dbPath('bad', 'lock'))), $record . ': set up'); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2], array_column($this->db(['shardingEnabled' => false])->find('bad', 0), 'n'), $record); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$this->epoch('bad'), $record); + $this->assertMatchesRegularExpression('/^[0-9]+:[0-9a-f]{8} *$/', (string)file_get_contents($this->dbPath('bad', 'lock')), $record . ': a valid record, nothing pending'); + $this->assertSame([], $this->lockState('bad')['flags'], $record); + $this->assertSame(1, $this->lockState('bad')['gen'], $record . ': a damaged record starts again at generation 1 (§4.2)'); + } + } + + /** @test bumpEpoch() without the database's exclusive lock is a bug: LogicException, nothing written */ + public function anEpochChangeNeedsTheExclusiveLock(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ep', ['n' => 1]); + $record = (string)file_get_contents($this->dbPath('ep', 'lock')); + $bump = function () { + $this->bumpEpoch('ep'); + }; + $shared = function (callable $fn) { + return $this->withLock('ep', false, $fn); + }; + foreach (['no lock' => function () use ($bump, $db) { $bump->call(\noneDB\Tests\Support\Testing::engine($db)); }, + 'shared lock' => function () use ($bump, $shared, $db) { $shared->call(\noneDB\Tests\Support\Testing::engine($db), function () use ($bump, $db) { $bump->call(\noneDB\Tests\Support\Testing::engine($db)); }); }] as $label => $attempt) { + try { + $attempt(); + $this->fail($label . ': refused'); + } catch (\LogicException $e) { + $this->assertStringContainsString("'ep'", $e->getMessage(), $label); + } + } + $this->assertSame($record, file_get_contents($this->dbPath('ep', 'lock'))); + } + + /** + * @test another generation at lock time with the same epoch: the key indexes are kept and only the new + * lines are replayed; another epoch: every cache of the database is dropped (STORAGE_FORMAT.md §4.2) and + * everything is loaded again. Results are the other process's in both cases. + */ + public function cachesAreRefreshedOnAnotherGenerationAndDroppedOnAnotherEpoch(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->createSpatialIndex('xp', 'loc'); + $db->insert('xp', [['n' => 1, 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['n' => 2]]); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([1, 2], array_column($db->find('xp', 0), 'n')); + $this->assertSame([1], array_column($db->withinBBox('xp', 'loc', 0, 0, 5, 5), 'n')); + $stats = new \noneDB\Tests\InternalValue('indexStats'); + $loads = $stats->getValue()['loads']; + $epoch = $this->epoch('xp'); + + // Another process appends (same epoch) + $this->runChild('$d = ' . $this->childDb($config) . '; $d->insert("xp", ["n" => 3, "loc" => ["type" => "Point", "coordinates" => [2, 2]]]);'); + $this->assertSame($epoch, $this->epoch('xp')); + $this->assertSame([1, 2, 3], array_column($db->find('xp', 0), 'n')); + $this->assertSame([1, 3], array_column($db->withinBBox('xp', 'loc', 0, 0, 5, 5), 'n')); + $this->assertSame($loads, $stats->getValue()['loads'], 'the key index was kept and only replayed'); + + // Another process replaces the data file (new epoch) + $this->runChild('$d = ' . $this->childDb($config) . '; $d->delete("xp", ["n" => 2]); $d->compact("xp");'); + $this->assertNotSame($epoch, $this->epoch('xp')); + $this->assertSame([1, 3], array_column($db->find('xp', 0), 'n')); + $this->assertSame([1, 3], array_column($db->withinBBox('xp', 'loc', 0, 0, 5, 5), 'n')); + $this->assertSame($loads + 1, $stats->getValue()['loads'], 'the key index was loaded again'); + } +} diff --git a/tests/Concurrency/LockStateRegressionTest.php b/tests/Concurrency/LockStateRegressionTest.php new file mode 100644 index 0000000..dd0bcc4 --- /dev/null +++ b/tests/Concurrency/LockStateRegressionTest.php @@ -0,0 +1,351 @@ +db(); + $db->insert('st', ['a' => 1]); + $state = $this->lockState('st'); + $this->assertSame([], $state['flags'], 'a new database: nothing pending'); + $this->assertSame(128, filesize($this->dbPath('st', 'lock')), 'fixed size record'); + + // Buffered records: flag set while they wait, cleared once they are flushed (STORAGE_FORMAT.md §4.2: + // "{gen}:{epoch}[ b][ c][ d][ m][ g=N>X][ j=...|J]") + $buffered = $this->db(['bufferEnabled' => true]); + $buffered->insert('st', ['a' => 2]); + $this->assertSame(['b'], $this->lockState('st')['flags']); + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8} b +$/', $this->rawRecord('st')); + $this->assertSame([1, 2], array_column($this->db()->find('st', 0), 'a'), 'a reader without buffering flushes them'); + $this->assertSame([], $this->lockState('st')['flags']); + + // Field index: the journal entry exists only during the write + $db->createFieldIndex('st', 'a'); + $db->insert('st', ['a' => 3]); + $this->assertSame([], $this->lockState('st')['flags']); + + $gen = $this->lockState('st')['gen']; + $this->assertTrue($db->dropDB('st')['success']); + $this->assertFileExists($this->dbPath('st', 'lock'), 'lock file survives a drop (§8.2)'); + $this->assertSame([], $this->lockState('st')['flags'], 'd cleared once the drop finished (§8.2 step 4)'); + $this->assertGreaterThan($gen, $this->lockState('st')['gen']); + // Dropped: nothing to read (autoCreateDB: [] rather than false), a check creates it again, empty + $this->assertSame([], $this->db()->find('st', 0)); + $this->assertSame(0, $db->count('st')); + $this->assertTrue($db->checkDB('st')); + $this->assertSame([], $db->find('st', 0)); + } + + /** The raw 128-byte record of a database's lock file */ + private function rawRecord(string $dbName): string + { + clearstatcache(); + return (string)file_get_contents($this->dbPath($dbName, 'lock')); + } + + /** Name of the database (of $names) a file belongs to */ + private function dbOfFile(string $file, array $names): ?string + { + foreach ($names as $name) { + if (strpos($file, $this->dbPath($name, 'home')) === 0) { + return $name; + } + } + return null; + } + + /** + * @test STORAGE_FORMAT.md §4.2: "{gen}:{epoch}[ flags]" space padded to 128 bytes, overwritten in place; + * an empty record is a new record ("the next exclusive session writes 1:{new epoch}"); the generation is + * bumped by the first file write of an exclusive session + */ + public function stateRecordIsAFixedSizeRecordOfGenerationEpochAndFlags(): void + { + $db = $this->db(['shardingEnabled' => false]); + $this->assertTrue($db->createDB('fmt')); + $record = $this->rawRecord('fmt'); + $this->assertSame(128, strlen($record)); + // A new lock file starts empty (a new record); createDB() is one write session: generation 1, and the + // creation flag "c" is cleared at its end (§8.1 step 6) + $this->assertMatchesRegularExpression('/^1:[0-9a-f]{8} {' . (128 - strlen('1:00000000')) . '}$/', $record); + $epoch = substr($record, 2, 8); + clearstatcache(); + $inode = fileinode($this->dbPath('fmt', 'lock')); + + $db->insert('fmt', ['a' => 1]); + $db->insert('fmt', [['a' => 2], ['a' => 3]]); + $this->assertSame(str_pad('3:' . $epoch, 128), $this->rawRecord('fmt'), 'two more sessions, same epoch'); + clearstatcache(); + $this->assertSame($inode, fileinode($this->dbPath('fmt', 'lock')), 'overwritten in place, never replaced'); + + // Writes that fail validation write nothing: no bump + $this->assertSame(0, $db->insert('fmt', 'not an array')['n']); + $this->assertSame(0, $db->insert('fmt', ['key' => 5])['n']); + $this->assertSame(str_pad('3:' . $epoch, 128), $this->rawRecord('fmt')); + } + + /** + * @test the journal is set before a sharded segment's data changes and cleared at the end of the session; + * non-sharded writes need none (STORAGE_FORMAT.md §7.3: "j (or J for all segments) is set before a sharded + * segment's data changes. It is cleared at the end of the session"; secondary indexes check themselves + * against the data, §6.5) + */ + public function journalIsSetDuringShardedWritesOnly(): void + { + $seen = []; + $this->setFaultHook(function ($point, $context) use (&$seen) { + if ($point === 'append:after' && preg_match('#/segments/(main|s\d+)\.jsonl$#', $context['path']) + && ($name = $this->dbOfFile($context['path'], ['plain', 'shard'])) !== null) { + $seen[] = trim($this->rawRecord($name)); + } + }); + $plain = $this->db(['shardingEnabled' => false]); + $plain->createFieldIndex('plain', 'a'); + $plain->insert('plain', ['a' => 1]); + $plain->update('plain', [['a' => 1], ['set' => ['a' => 2]]]); + $this->assertNotEmpty($seen); + foreach ($seen as $state) { + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8}$/', $state, 'no journal for a non-sharded write'); + } + + $sharded = $this->db(['shardSize' => 3]); + $sharded->insert('shard', array_map(function ($n) { return ['a' => $n]; }, range(0, 3))); // switched: shards 0, 1 + $seen = []; + $sharded->insert('shard', ['a' => 4]); // key 4: shard 1 + $this->assertSame(1, count($seen)); + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8} j=s1$/', $seen[0]); + $seen = []; + $sharded->insert('shard', [['a' => 5], ['a' => 6], ['a' => 7]]); // keys 5 (shard 1), 6 and 7 (shard 2) + $this->assertMatchesRegularExpression('/ j=s1,s2$/', end($seen)); + $this->setFaultHook(null); + $this->assertSame([], $this->lockState('shard')['flags'], 'cleared once the session ended'); + $this->assertSame(range(0, 7), array_column($sharded->find('shard', 0), 'a')); + } + + /** @test a journal that doesn't fit into the 128-byte record becomes "J" (all segments) */ + public function aJournalTooLongForTheRecordBecomesJ(): void + { + $db = $this->db(['shardSize' => 2]); + $db->insert('many', [['n' => 0], ['n' => 1]]); // switched: shard 0 + $states = []; + $this->setFaultHook(function ($point) use (&$states) { + if ($point === 'append:after') { + clearstatcache(); + $states[] = (string)file_get_contents($this->dbPath('many', 'lock')); + } + }); + // keys 2-121: shards 1-60; "j=s1,s2,...,s60" alone is 232 bytes + $db->insert('many', array_map(function ($n) { return ['n' => $n]; }, range(2, 121))); + $this->setFaultHook(null); + $this->assertNotEmpty($states); + foreach ($states as $state) { + $this->assertSame(128, strlen($state), 'never longer than the record'); + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8} (j=s[0-9]+(,s[0-9]+)*|J) *$/', $state); + } + $this->assertSame('J', substr(trim(end($states)), -1)); + $this->assertSame([], $this->lockState('many')['flags']); + $this->assertSame(range(0, 121), array_column($db->find('many', 0), 'key')); + $this->assertSame(122, $db->count('many')); + } + + /** + * @test STORAGE_FORMAT.md §7.3: a journal found at lock time makes the next exclusive session recount the + * shard counters (shards.json) from the data; "J" means every segment + */ + public function aPendingJournalIsRecoveredAtLockTime(): void + { + foreach (['J', 'j=s1'] as $journal) { + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 3]); + $db->insert('jr', array_map(function ($n) { return ['n' => $n]; }, range(0, 6))); // shards 0, 1, 2 + $metaPath = $this->dbPath('jr', 'shards'); + $meta = json_decode((string)file_get_contents($metaPath), true); + $this->assertSame([3, 3, 1], array_map('intval', array_column($meta['shards'], 'count')), 'set up'); + // A writer died between its data append and its meta update + $meta['totalRecords'] = 50; + foreach ($meta['shards'] as $i => $shard) { + if ((int)$shard['id'] === 1) { + $meta['shards'][$i]['count'] = 40; + } + } + file_put_contents($metaPath, json_encode($meta)); + \noneDB::clearStaticCache(); + [$state] = explode(' ', trim($this->rawRecord('jr'))); + $this->writeLockState('jr', $state . ' ' . $journal); + + $db = $this->db(['shardSize' => 3]); + $this->assertSame(7, $db->count('jr'), $journal); + $this->assertSame([], $this->lockState('jr')['flags'], $journal . ': recovered, journal cleared'); + $meta = json_decode((string)file_get_contents($metaPath), true); + $this->assertSame(7, (int)$meta['totalRecords'], $journal); + $this->assertSame([3, 3, 1], array_map('intval', array_column($meta['shards'], 'count')), $journal); + $this->assertSame(range(0, 6), array_column($db->find('jr', 0), 'n'), $journal); + } + } + + /** + * @test STORAGE_FORMAT.md §4.2: "A record that is empty, all NUL bytes, or all spaces is a new record. The + * next exclusive session writes 1:{new epoch}" (whatever the lost record said is recovered from the data) + */ + public function aNewRecordIsWrittenAsGenerationOneWithANewEpoch(): void + { + $records = ['empty file' => '', 'NUL bytes' => str_repeat("\0", 128), 'spaces' => str_repeat(' ', 128)]; + foreach (['single' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 3]] as $layout => $config) { + foreach ($records as $label => $record) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('nr', array_map(function ($n) { return ['n' => $n]; }, range(0, 6))); + $db->insert('nr', ['n' => 7]); + $epoch = $this->lockState('nr')['epoch']; + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$epoch, 'set up'); + \noneDB::clearStaticCache(); // closes the kept lock handles + file_put_contents($this->dbPath('nr', 'lock'), $record); + + $this->assertSame(range(0, 7), array_column($this->db($config)->find('nr', 0), 'n'), "$layout, $label"); + $raw = $this->rawRecord('nr'); + $this->assertSame(128, strlen($raw), "$layout, $label"); + $this->assertMatchesRegularExpression('/^1:[0-9a-f]{8} {118}$/', $raw, "$layout, $label: 1:{new epoch}, nothing pending"); + $this->assertNotSame($epoch, $this->lockState('nr')['epoch'], "$layout, $label: a new epoch"); + $this->assertSame(8, $this->db($config)->count('nr'), "$layout, $label"); + } + } + } + + /** @test */ + public function generationMovesOncePerWriteAndNeverOnReads(): void + { + $db = $this->db(); + $db->insert('g', [['a' => 1], ['a' => 2]]); + $db->createFieldIndex('g', 'a'); + $gen = $this->lockState('g')['gen']; + + $db->find('g', 0); + $db->find('g', ['a' => 1]); + $db->count('g'); + $db->query('g')->where(['a' => ['$gt' => 0]])->get(); + $this->assertSame($gen, $this->lockState('g')['gen'], 'reads leave the generation alone'); + + $db->insert('g', ['a' => 3]); + $this->assertSame($gen + 1, $this->lockState('g')['gen'], 'one write session, one bump'); + $db->update('g', [['a' => 3], ['set' => ['b' => 1]]]); + $this->assertSame($gen + 2, $this->lockState('g')['gen']); + $db->delete('g', ['a' => 3]); + $this->assertSame($gen + 3, $this->lockState('g')['gen']); + + // Another process sees each change (its caches are dropped on the generation change) + $out = $this->runChild('$d = ' . $this->childDb() . '; echo json_encode(array_column($d->find("g", 0), "a"));'); + $this->assertSame('[1,2]', $out); + } + + /** @test the path is locked, not a deleted file a kept handle still points to */ + public function replacedLockFileIsNotTrusted(): void + { + $db = $this->db(['lockTimeout' => 1]); + $db->insert('rl', ['a' => 1]); // the lock handle stays open in this process + $lock = $this->dbPath('rl', 'lock'); + $this->assertTrue(unlink($lock)); // e.g. a cleanup script + + $holder = $this->childScript('$fp = fopen(' . var_export($lock, true) . ', "c+"); flock($fp, LOCK_EX); echo "locked\n"; fflush(STDOUT); sleep(3);'); + $proc = proc_open([PHP_BINARY, $holder], [1 => ['pipe', 'w']], $pipes); // no shell: proc_terminate() reaches PHP + $this->assertSame('locked', trim(fgets($pipes[1]))); + try { + $db->insert('rl', ['a' => 2]); + $this->fail('the insert must wait for the process holding the new lock file'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('timed out', $e->getMessage()); + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode()); + } finally { + fclose($pipes[1]); + proc_terminate($proc); + proc_close($proc); + } + $db->insert('rl', ['a' => 3]); + $this->assertSame([1, 3], array_column($db->find('rl', 0), 'a')); + $this->assertSame([], $this->lockState('rl')['flags'], 'state record rewritten in the new file'); + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8} *$/', $this->rawRecord('rl')); + } + + /** @test a forked child must not share the parent's lock handle (flock locks belong to the open file) */ + public function forkedProcessesStillExcludeEachOther(): void + { + if (!function_exists('pcntl_fork')) { + $this->markTestSkipped('needs pcntl'); + } + $out = $this->runChild(' + $cfg = ' . var_export(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir], true) . '; + $db = new \noneDB($cfg); + $db->insert("fk", ["w" => "init"]); // lock handle opened before the fork + $pid = pcntl_fork(); + $who = $pid === 0 ? "c" : "p"; + for ($i = 0; $i < 300; $i++) { + $db->insert("fk", ["w" => $who, "i" => $i]); + $db->update("fk", [["w" => "init"], ["set" => ["n" => $i]]]); + } + if ($pid === 0) { exit(0); } + pcntl_waitpid($pid, $status); + \noneDB::clearStaticCache(); + $all = (new \noneDB($cfg))->find("fk", 0); + $c = count(array_filter($all, function ($r) { return $r["w"] === "c"; })); + echo count($all), " ", count(array_unique(array_column($all, "key"))), " ", $c; + '); + $this->assertSame('601 601 300', $out); + } + + /** + * @test kept lock handles are close-on-exec: a program this process starts must not inherit one + * (an inherited copy would keep a lock held after this process died in the middle of a write) + */ + public function keptLockHandlesAreNotInherited(): void + { + if (DIRECTORY_SEPARATOR === '\\') { + $this->markTestSkipped('POSIX file descriptors'); + } + $db = $this->db(); + $db->insert('ih', ['a' => 1]); // the lock handle stays open in this process + $lock = $this->dbPath('ih', 'lock'); + $out = $this->runChild('$ino = stat(' . var_export($lock, true) . ')["ino"]; $n = 0; + for ($i = 3; $i < 256; $i++) { + $f = @fopen("php://fd/$i", "r"); + if ($f) { $s = @fstat($f); if ($s && $s["ino"] === $ino) { $n++; } fclose($f); } + } + echo "inherited:", $n;'); + $this->assertSame('inherited:0', $out); + } + + /** @test */ + public function keptLockHandlesAreLimited(): void + { + $db = $this->db(); + for ($i = 0; $i < 100; $i++) { + $db->insert('h' . $i, ['i' => $i]); + } + $handles = new \noneDB\Tests\InternalValue('lockHandles'); + $this->assertLessThanOrEqual(64, count($handles->getValue())); + for ($i = 0; $i < 100; $i++) { + $this->assertSame([$i], array_column($db->find('h' . $i, 0), 'i')); + } + \noneDB::clearStaticCache(); + $this->assertSame([], $handles->getValue(), 'clearStaticCache closes them'); + } + + /** @test */ + public function readingAMissingDatabaseCreatesNothing(): void + { + $db = $this->db(); + $before = glob($this->testDbDir . '*'); + $this->assertSame([], $db->find('nothere', 0)); + $this->assertSame(0, $db->count('nothere')); + $this->assertSame([], $db->query('nothere')->get()); + $this->assertSame($before, glob($this->testDbDir . '*')); + } +} diff --git a/tests/Crash/BackupRestoreCrashTest.php b/tests/Crash/BackupRestoreCrashTest.php new file mode 100644 index 0000000..1e0cd94 --- /dev/null +++ b/tests/Crash/BackupRestoreCrashTest.php @@ -0,0 +1,258 @@ +markTestSkipped('needs POSIX signals and proc_open'); + } + } + + protected function tearDown(): void + { + foreach ($this->outside as $path) { + $this->removeDirectoryTree($path); + } + $this->outside = []; + parent::tearDown(); + } + + private function backupDir(): string + { + $dir = sys_get_temp_dir() . '/nonedb_backup_' . bin2hex(random_bytes(6)); + $this->outside[] = $dir; + return $dir; + } + + /** [v, key] of every record, by key */ + private function pairs(\noneDB $db, string $name): array + { + $records = $db->find($name, 0); + usort($records, function ($a, $b) { return $a['key'] <=> $b['key']; }); + return array_map(function ($r) { return [$r['v'], $r['key']]; }, $records); + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + private function generations(string $db): array + { + $out = array_map('intval', array_values(array_filter(scandir($this->dbPath($db, 'home') . 'generations') ?: [], 'ctype_digit'))); + sort($out); + return $out; + } + + private function snapshot(string $dir): array + { + $out = []; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $full = $path . '/' . $entry; + if (is_dir($full)) { + $out[$rel . $entry] = 'dir'; + $walk($full, $rel . $entry . '/'); + } else { + $out[$rel . $entry] = hash_file('sha256', $full); + } + } + }; + if (is_dir($dir)) { + $walk(rtrim($dir, '/'), ''); + } + ksort($out); + return $out; + } + + /** Run $call in a child (no shell) that is killed (SIGKILL) at the $nth occurrence of fault point $point */ + private function killedAt(string $call, string $point, int $nth, array $config): string + { + $script = $this->childScript(' + $db = ' . $this->childDb($config) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { + static $seen = 0; + if ($p === ' . var_export($point, true) . ' && ++$seen === ' . $nth . ') { posix_kill(getmypid(), 9); } + }); + ' . $call . '; + echo "COMPLETED"; + '); + $proc = proc_open([PHP_BINARY, $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $output = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($proc); + return trim($output); + } + + private function seed(\noneDB $db, int $from, int $to): void + { + $db->insert('u', array_map(function ($n) { return ['v' => $n, 'c' => 'c' . ($n % 2)]; }, range($from, $to))); + } + + /** @test §10.3 step 5, present database: §5.2 decides */ + public function aRestoreKilledAtEveryStepOpensTheOldOrTheRestoredDatabase(): void + { + $config = ['shardSize' => 4]; + $points = [['restore:copied', 1], ['restore:copied', 4], ['restore:verified', 1], ['generation:prepared', 1], + ['generation:intent', 1], ['generation:published', 1], ['generation:cleaned', 1]]; + foreach ($points as [$point, $nth]) { + $label = "$point #$nth"; + $this->cleanTestDirectory(); + $db = $this->db($config); + $this->seed($db, 0, 5); + $db->createFieldIndex('u', 'c'); + $dir = $this->backupDir(); + $db->backup('u', $dir); + $restored = $this->pairs($db, 'u'); + $this->seed($db, 6, 9); + $db->delete('u', ['v' => 0]); + $old = $this->pairs($db, 'u'); + \noneDB::clearStaticCache(); + + $this->assertSame('', $this->killedAt('$db->restore(' . var_export($dir, true) . ', "u")', $point, $nth, $config), $label); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $published = in_array($point, ['generation:published', 'generation:cleaned'], true); + $expected = $published ? $restored : $old; + $this->assertSame($expected, $this->pairs($db, 'u'), "$label: the old or the restored database, never a mix"); + $odd = array_values(array_filter($expected, function ($p) { return $p[0] % 2 === 1; })); + $this->assertSame(array_column($odd, 1), array_column($db->find('u', ['c' => 'c1']), 'key'), "$label: its field index"); + $this->assertSame([], array_values(array_filter($this->lockState('u')['flags'], function ($f) { return $f[0] === 'g'; })), "$label: recovered"); + + $result = $db->restore($dir, 'u'); + $this->assertSame(10, $result['nextKey'], "$label: keys 0..9 were handed out"); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame($restored, $this->pairs($db, 'u'), "$label: a second run completes"); + $this->assertCount(1, $this->generations('u'), $label); + $this->assertTrue($db->checkIntegrity('u')['ok'], $label); + } + } + + /** @test §10.3 step 5, absent database: §8.3 removes an interrupted creation */ + public function aRestoreIntoAnAbsentDatabaseKilledAtEveryStepLeavesItAbsentOrRestored(): void + { + $config = ['shardSize' => 4]; + $points = [['create:flagged', 1], ['restore:copied', 1], ['restore:verified', 1], ['create:generation', 1], + ['create:current', 1], ['create:format', 1]]; + foreach ($points as [$point, $nth]) { + $label = "$point #$nth"; + $this->cleanTestDirectory(); + $db = $this->db($config); + $this->seed($db, 0, 5); + $dir = $this->backupDir(); + $db->backup('u', $dir); + $restored = $this->pairs($db, 'u'); + $this->assertTrue($db->dropDB('u')['success']); + \noneDB::clearStaticCache(); + + $this->assertSame('', $this->killedAt('$db->restore(' . var_export($dir, true) . ', "u")', $point, $nth, $config), $label); + \noneDB::clearStaticCache(); + $db = $this->db($config); + if ($point === 'create:format') { + $this->assertSame($restored, $this->pairs($db, 'u'), "$label: FORMAT is the commit point"); + $this->assertSame(['u'], $db->getDBs(), $label); + } else { + $this->assertSame([], $db->find('u', 0), "$label: absent"); + $this->assertSame([], $db->getDBs(), $label); + $this->assertSame(['LOCK'], $this->dbFiles('u'), "$label: the interrupted creation is removed"); + } + $this->assertSame([], $this->lockState('u')['flags'], "$label: no c left"); + + $db->restore($dir, 'u'); + $this->assertSame($restored, $this->pairs($this->db($config), 'u'), "$label: a second run completes"); + } + } + + /** @test §10.3 step 5, damaged database: damaged until CURRENT names X */ + public function aRestoreOverADamagedDatabaseKilledAtEveryStep(): void + { + $config = ['shardingEnabled' => false]; + $points = [['restore:copied', 1], ['restore:verified', 1], ['restore:intent', 1], ['restore:published', 1]]; + foreach (['FORMAT valid' => false, 'FORMAT invalid' => true] as $case => $formatToo) { + foreach ($points as [$point, $nth]) { + $label = "$case, $point #$nth"; + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('u', [['v' => 0], ['v' => 1], ['v' => 2]]); + $dir = $this->backupDir(); + $db->backup('u', $dir); + $restored = $this->pairs($db, 'u'); + $db->insert('u', [['v' => 3], ['v' => 4]]); + file_put_contents($this->dbPath('u', 'current'), 'x'); + if ($formatToo) { + file_put_contents($this->dbPath('u', 'format'), 'x'); + } + \noneDB::clearStaticCache(); + + $this->assertSame('', $this->killedAt('$db->restore(' . var_export($dir, true) . ', "u")', $point, $nth, $config), $label); + \noneDB::clearStaticCache(); + $db = $this->db($config); + if ($point === 'restore:published') { + $this->assertSame($restored, $this->pairs($db, 'u'), "$label: CURRENT names X: the commit point"); + if ($formatToo) { + $this->assertContains('format-repaired', array_column(array_map(function ($l) { return json_decode($l, true); }, + $this->dataLines($this->dbPath('u', 'recovery'))), 'action'), "$label: FORMAT repaired from X's metadata (§8.3)"); + } + $expectedTarget = 'present'; // the second run finds a present database + } else { + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () use ($db) { $db->find('u', 0); }), "$label: still damaged"); + $this->assertContains(1, $this->generations('u'), "$label: the damaged generation is kept"); + $expectedTarget = 'damaged'; + } + $result = $db->restore($dir, 'u'); + $this->assertSame($expectedTarget, $result['target'], $label); + $this->assertSame(5, $result['nextKey'], "$label: keys 0..4 were handed out"); + $this->assertSame($restored, $this->pairs($this->db($config), 'u'), "$label: a second run completes"); + $this->assertContains(1, $this->generations('u'), "$label: never removed without proof"); + } + } + } + + /** @test §10.2: the manifest is written last */ + public function anInterruptedBackupHasNoManifestAndIsRefused(): void + { + foreach ([['backup:copied', 1], ['backup:copied', 3], ['backup:verified', 1]] as [$point, $nth]) { + $label = "$point #$nth"; + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 4]); + $this->seed($db, 0, 5); + $db->find('u', 0); + $dir = $this->backupDir(); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame('', $this->killedAt('$db->backup("u", ' . var_export($dir, true) . ')', $point, $nth, ['shardSize' => 4]), $label); + $this->assertSame($before, $this->snapshot($this->testDbDir), "$label: the database is untouched"); + $this->assertFileDoesNotExist("$dir/manifest.json", $label); + $db = $this->db(['shardSize' => 4]); + $this->assertSame(\noneDB::RESTORE_REFUSED, $this->code(function () use ($db, $dir) { $db->restore($dir, 'u'); }), $label); + $this->assertSame($before, $this->snapshot($this->testDbDir), "$label: nothing restored"); + // The killed backup's locks went with it + $db->insert('u', ['v' => 6, 'c' => 'c0']); + $this->assertSame(7, $db->count('u'), $label); + } + } +} diff --git a/tests/Regression/CrashRecoveryTest.php b/tests/Crash/CrashRecoveryTest.php similarity index 91% rename from tests/Regression/CrashRecoveryTest.php rename to tests/Crash/CrashRecoveryTest.php index 35339e8..5b7f32d 100644 --- a/tests/Regression/CrashRecoveryTest.php +++ b/tests/Crash/CrashRecoveryTest.php @@ -1,6 +1,8 @@ assertSame($expected, $got, 'spatial index for box ' . json_encode($box)); } $this->assertSame(count($all), $db->count($name), 'count() agrees with the records'); + // The recovered indexes are exactly what the data says (rebuilt from scratch: same records) + \noneDB::clearStaticCache(); + foreach (glob($this->dbPath($name, 'gen') . 'segments/*.kix') as $index) { + unlink($index); + } + $this->assertSame($all, $this->db($config)->find($name, 0), 'indexes rebuilt from the data give the same records'); + \noneDB::clearStaticCache(); + $db = $this->db($config); $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState($name)['flags']), 'journal cleared after recovery'); $this->assertNotContains('b', $this->lockState($name)['flags'], 'buffer flag cleared after the replay'); - $this->assertSame([], glob($this->dbBasePath($name) . '*.flushing*'), 'interrupted flushes replayed'); + $this->assertSame([], glob($this->dbPath($name, 'gen') . 'buffer/flushing.*'), 'interrupted flushes replayed'); // Still writable $db->insert($name, ['n' => -2 - $seed, 'c' => 'after', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); @@ -164,7 +174,7 @@ public function killedWriterNonSharded(): void } /** @test */ - public function killedWriterShardedIncludingMigration(): void + public function killedWriterShardedIncludingTheSwitchToShards(): void { mt_srand(12); foreach ([20, 60, 150, 250, 99, 33, 400] as $i => $acks) { diff --git a/tests/Crash/DatabaseLifecycleTest.php b/tests/Crash/DatabaseLifecycleTest.php new file mode 100644 index 0000000..4648bc5 --- /dev/null +++ b/tests/Crash/DatabaseLifecycleTest.php @@ -0,0 +1,351 @@ +getCode(); + } + return null; + } + + private function killAt(string $point, string $body, array $config = []): string + { + return $this->runChild(' + $db = ' . $this->childDb($config) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === ' . var_export($point, true) . ') { exit(0); } }); + ' . $body . ' + echo "not reached"; + '); + } + + /** @test */ + public function aNewDatabaseHasFormatCurrentAndGenerationOne(): void + { + $this->db(['shardSize' => 7])->insert('fresh', ['a' => 1]); + $format = $this->record($this->dbPath('fresh', 'format')); + $this->assertNotNull($format, 'FORMAT is a valid record (§4.1)'); + $this->assertSame(['magic', 'kind', 'storageFormat', 'databaseId', 'name', 'shardSize', 'createdAt', 'formatOptions', 'checksum'], array_keys($format)); + $this->assertSame('database', $format['kind']); + $this->assertSame(4, $format['storageFormat']); + $this->assertSame('fresh', $format['name']); + $this->assertSame(7, $format['shardSize'], 'fixed at creation'); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', $format['databaseId']); + $current = $this->record($this->dbPath('fresh', 'current')); + $this->assertSame(['magic' => 'NONEDB4', 'kind' => 'current', 'databaseId' => $format['databaseId'], 'generation' => 1, 'checksum' => $current['checksum']], $current); + $metadata = $this->record($this->dbPath('fresh', 'gen') . 'metadata'); + $this->assertSame('generation', $metadata['kind']); + $this->assertSame(1, $metadata['generation']); + $this->assertSame('single', $metadata['layout']); + foreach (['storageFormat', 'databaseId', 'name', 'shardSize', 'createdAt'] as $member) { + $this->assertSame($format[$member], $metadata[$member], "identity copy: $member (§5.1)"); + } + $this->assertSame([], $this->lockState('fresh')['flags'], 'creation finished: no c flag'); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', $this->lockState('fresh')['epoch']); + } + + /** @test */ + public function creationInterruptedAtEveryStepLeavesNoDatabaseOrACompleteOne(): void + { + foreach (['create:flagged' => false, 'create:generation' => false, 'create:current' => false, 'create:format' => true] as $point => $complete) { + $this->cleanTestDirectory(); + $this->db()->insert('other', ['x' => 1]); // the store exists + \noneDB::clearStaticCache(); + $out = $this->killAt($point, '$db->insert("c", ["a" => 1]);'); + $this->assertSame('', $out, $point); + $this->assertContains('c', $this->lockState('c')['flags'], "$point: c set before anything else (§8.1)"); + \noneDB::clearStaticCache(); + $db = $this->db(); + // No data was written during creation: the interrupted insert is not there, the database is + // either absent (cleaned up) or complete and empty + $this->assertSame([], $db->find('c', 0), $point); + $this->assertSame($complete ? ['c', 'other'] : ['other'], $db->getDBs(), $point); + $db->insert('c', ['a' => 2]); + $this->assertSame([['a' => 2, 'key' => 0]], $db->find('c', 0), "$point: usable, key 0"); + $this->assertNotContains('c', $this->lockState('c')['flags'], "$point: c cleared"); + $this->assertTrue($db->checkIntegrity('c')['ok'], $point); + } + } + + /** + * @test a creation killed before its state record carries "c" (right after mkdir, or after LOCK was + * created): the directory holds nothing or an empty LOCK, an absent location (§8.1 steps 1-3, §8.3) + */ + public function aDirectoryLeftBeforeTheCreationFlagIsAnAbsentDatabase(): void + { + foreach (['empty directory' => null, 'empty LOCK' => '', 'NUL LOCK' => str_repeat("\0", 128)] as $case => $lock) { + $this->cleanTestDirectory(); + $this->db()->insert('other', ['x' => 1]); // the store exists + \noneDB::clearStaticCache(); + mkdir($this->dbPath('half', 'home')); + if ($lock !== null) { + file_put_contents($this->dbPath('half', 'lock'), $lock); + } + $db = $this->db(); + $this->assertSame([], $db->find('half', 0), "$case: read as absent, never FORMAT_DAMAGED"); + $this->assertSame(0, $db->count('half'), $case); + $this->assertSame(['other'], $db->getDBs(), $case); + $this->assertFileDoesNotExist($this->dbPath('half', 'format'), "$case: the read created nothing"); + $db->insert('half', ['a' => 1]); + $this->assertSame([['a' => 1, 'key' => 0]], $db->find('half', 0), "$case: created from key 0"); + $this->assertSame(['half', 'other'], $db->getDBs(), $case); + $this->assertTrue($db->checkIntegrity('half')['ok'], $case); + } + } + + /** @test */ + public function dropKeepsOnlyTheLockAndAllowsRecreation(): void + { + $db = $this->db(['shardSize' => 3]); + $db->createFieldIndex('dr', 'g'); + $db->insert('dr', array_map(function ($n) { return ['n' => $n, 'g' => $n % 2]; }, range(0, 7))); + $result = $db->dropDB('dr'); + $this->assertTrue($result['success']); + $this->assertSame(['LOCK'], $this->dbFiles('dr'), '§8.2: only LOCK stays'); + $this->assertSame([], $db->find('dr', 0)); + $this->assertSame([], $db->getDBs()); + $this->assertSame(['success' => false, 'files' => 0, 'error' => 'Database does not exist'], $db->dropDB('dr')); + $db->insert('dr', ['n' => 'new']); + $this->assertSame([['n' => 'new', 'key' => 0]], $db->find('dr', 0), 'recreated from key 0'); + } + + /** @test */ + public function dropInterruptedAtEveryStepIsFinished(): void + { + foreach (['drop:flagged', 'drop:format', 'drop:cleaned'] as $point) { + $this->cleanTestDirectory(); + $this->db()->insert('d', ['a' => 1]); + \noneDB::clearStaticCache(); + $this->assertSame('', $this->killAt($point, '$db->dropDB("d");'), $point); + $this->assertContains('d', $this->lockState('d')['flags'], "$point: d set first"); + \noneDB::clearStaticCache(); + $db = $this->db(); + $this->assertSame([], $db->find('d', 0), "$point: the drop is finished, never half a database"); + $this->assertSame(['LOCK'], $this->dbFiles('d'), $point); + $this->assertNotContains('d', $this->lockState('d')['flags'], $point); + } + } + + /** @test */ + public function aMissingFormatIsRepairedOnlyWithProof(): void + { + $db = $this->db(); + $db->insert('f', [['a' => 1], ['a' => 2]]); + $format = file_get_contents($this->dbPath('f', 'format')); + unlink($this->dbPath('f', 'format')); + \noneDB::clearStaticCache(); + // CURRENT and its generation's metadata are valid and agree: FORMAT comes back from the metadata + $this->assertSame([1, 2], array_column($this->db()->find('f', 0), 'a'), '§8.3 repair with proof'); + $this->assertSame($format, file_get_contents($this->dbPath('f', 'format')), 'the same identity, byte for byte'); + $log = file_get_contents($this->dbPath('f', 'recovery')); + $this->assertStringContainsString('"format-repaired"', $log); + } + + /** @test */ + public function aDamagedDirectoryWithoutProofIsNeverEmpty(): void + { + $db = $this->db(); + $db->insert('g', [['a' => 1]]); + foreach (['no FORMAT, CURRENT invalid' => function () { + unlink($this->dbPath('g', 'format')); + file_put_contents($this->dbPath('g', 'current'), 'garbage'); + }, + 'FORMAT invalid, metadata invalid' => function () { + file_put_contents($this->dbPath('g', 'format'), "{}\n"); + file_put_contents($this->dbPath('g', 'gen') . 'metadata', "{}\n"); + }] as $case => $damage) { + $this->cleanTestDirectory(); + $this->db()->insert('g', [['a' => 1]]); + $damage(); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $db = $this->db(); + foreach (['find' => function () use ($db) { $db->find('g', 0); }, + 'count' => function () use ($db) { $db->count('g'); }, + 'insert' => function () use ($db) { $db->insert('g', ['a' => 9]); }, + 'createDB' => function () use ($db) { $db->createDB('g'); }] as $op => $fn) { + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code($fn), "$case: $op is never answered as an empty database"); + } + $this->assertSame($before, $this->snapshot($this->testDbDir), "$case: data kept, nothing changed"); + $this->assertNotContains('g', $db->getDBs(), $case); + } + } + + /** @test */ + public function aLostLockRecordIsANewRecordAndTheDatabaseIsRecovered(): void + { + $db = $this->db(['shardSize' => 2]); + $db->insert('l', array_map(function ($n) { return ['n' => $n]; }, range(0, 4))); + $this->removeLockFile('l'); + $db = $this->db(['shardSize' => 2]); + $this->assertSame(range(0, 4), array_column($db->find('l', 0), 'n')); + $state = $this->lockState('l'); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$state['epoch'], '§4.2: an epoch is written'); + $this->assertSame([], $state['flags'], 'the recovery (J) is done'); + $db->insert('l', ['n' => 5]); + $this->assertSame(range(0, 5), array_column($db->find('l', 0), 'n')); + } + + /** @test */ + public function aDamagedLockRecordIsRecoveredAsJ(): void + { + $db = $this->db(['shardSize' => 2]); + $db->insert('dmg', array_map(function ($n) { return ['n' => $n]; }, range(0, 4))); + $this->writeLockState('dmg', '17 v'); // not this format (no epoch): damaged + $db = $this->db(['shardSize' => 2]); + $this->assertSame(range(0, 4), array_column($db->find('dmg', 0), 'n')); + $state = $this->lockState('dmg'); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$state['epoch']); + $this->assertSame([], $state['flags'], 'no flag survives, J was recovered'); + } + + private function listings(): int + { + $property = new \noneDB\Tests\InternalValue('directoryListings'); + return $property->getValue(); + } + + /** + * @test Only maintenance steps list a directory: compaction (stale temp files) and creating a shard + * (orphan files); both are kept out of this test (no compaction threshold reached, room in the last shard) + */ + public function crudOnExistingDatabasesNeverListsADirectory(): void + { + $config = ['shardSize' => 50, 'autoCompactThreshold' => 5]; + $db = $this->db($config); + $db->createFieldIndex('sh', 'g'); + $db->insert('sh', array_map(function ($n) { return ['n' => $n, 'g' => $n % 7]; }, range(0, 139))); // 3 shards, room in s2 + $db->insert('one', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + $this->assertTrue($db->getShardInfo('sh')['sharded']); + $crud = function (\noneDB $db, int $round) { + foreach (['sh', 'one'] as $name) { + $db->find($name, ['key' => 3]); + $db->find($name, ['n' => 4]); + $db->count($name); + $db->insert($name, ['n' => 1000 + $round, 'g' => 1]); // shard 2 has room: no new shard + $db->update($name, [['key' => 5], ['set' => ['u' => $round]]]); + $db->delete($name, ['key' => 6 + $round]); + $db->query($name)->where(['g' => 3])->sort('n', 'desc')->limit(2)->get(); + } + $db->find('sh', ['g' => 2]); + }; + $crud($db, 0); // warm the caches + $before = $this->listings(); + $crud($db, 1); + $this->assertSame($before, $this->listings(), 'warm CRUD lists no directory (§5)'); + \noneDB::clearStaticCache(); + $before = $this->listings(); + $crud($this->db($config), 2); + $this->assertSame($before, $this->listings(), 'a new process: store, FORMAT, CURRENT and shards.json are read, no directory is listed'); + $this->writeLockState('sh', $this->lockState('sh')['gen'] . ':' . substr(md5('other'), 0, 8)); // another process replaced a file + $before = $this->listings(); + $crud($db, 3); + $this->assertSame($before, $this->listings(), 'the first operations after an epoch change list nothing either'); + $this->assertSame(140 + 4 - 4, $db->count('sh'), 'every operation worked'); + $this->assertSame(3, $db->getShardInfo('sh')['shards'], 'no shard was created (that checks for orphan files once)'); + } + + /** + * @test A drop that can't remove FORMAT did not happen (§8.2): the flags stay as they were, so the buffered records + * of a state record with `b` stay records. Without `b` a new process read none of them. + */ + public function aDropThatFailsKeepsTheBufferedRecords(): void + { + if (PHP_OS_FAMILY === 'Windows' || (function_exists('posix_geteuid') && posix_geteuid() === 0)) { + $this->markTestSkipped('a directory that cannot be written is not made with chmod here'); + } + $db = $this->db(['bufferEnabled' => true]); + $db->insert('fd', ['x' => 1]); + $db->insert('fd', ['x' => 2]); + $db->insert('fd', ['x' => 3]); + $this->assertContains('b', $this->lockState('fd')['flags'], 'precondition: records in the buffer'); + $home = $this->dbPath('fd', 'home'); + $epoch = $this->lockState('fd')['epoch']; + chmod($home, 0555); + try { + $result = $db->dropDB('fd'); + } finally { + chmod($home, 0755); + } + $this->assertFalse($result['success']); + $state = $this->lockState('fd'); + $this->assertContains('b', $state['flags'], 'the state record still says the buffer may hold records'); + $this->assertNotContains('d', $state['flags'], 'the drop was withdrawn'); + $this->assertNotSame($epoch, $state['epoch'], 'the flag was written with a new epoch: the epoch stays new'); + $out = $this->runChild('echo json_encode((' . $this->childDb(['bufferEnabled' => true]) . ')->find("fd", []));'); + $this->assertSame([['x' => 1, 'key' => 0], ['x' => 2, 'key' => 1], ['x' => 3, 'key' => 2]], json_decode($out, true), $out); + } + + /** + * @test A damaged directory is never absent (§8.3), for the calls that write nothing too: they say FORMAT_DAMAGED + * as every other call does and change nothing, with or without the proof a repair needs. A read then repairs what + * it can prove, and fails on the rest. checkIntegrity() answered `ok`, `exists` false. + */ + public function theCallsThatWriteNothingSayADatabaseIsDamaged(): void + { + foreach (['with proof' => false, 'without proof' => true] as $case => $breakCurrent) { + $this->cleanTestDirectory(); + $this->db()->insert('dm', [['x' => 1], ['x' => 2]]); + unlink($this->dbPath('dm', 'format')); + if ($breakCurrent) { + file_put_contents($this->dbPath('dm', 'current'), 'junk'); + } + $before = $this->snapshot($this->testDbDir); + $out = $this->runChild('$db = ' . $this->childDb() . '; $codes = [];' + . ' foreach (["checkIntegrity" => function () use ($db) { return $db->checkIntegrity("dm"); },' + . ' "explain" => function () use ($db) { return $db->query("dm")->explain(); },' + . ' "getMetrics" => function () use ($db) { return $db->getMetrics("dm"); }] as $call => $run) {' + . ' try { $run(); $codes[$call] = "answered"; } catch (\RuntimeException $e) { $codes[$call] = $e->getCode(); } }' + . ' echo json_encode($codes);'); + $this->assertSame(['checkIntegrity' => 3305, 'explain' => 3305, 'getMetrics' => 3305], json_decode($out, true), "$case: $out"); + $this->assertSame($before, $this->snapshot($this->testDbDir), "$case: nothing changed"); + + $read = $this->runChild('try { echo json_encode((' . $this->childDb() . ')->find("dm", [])); } catch (\RuntimeException $e) { echo $e->getCode(); }'); + $this->assertSame($breakCurrent ? '3305' : '[{"x":1,"key":0},{"x":2,"key":1}]', $read, $case); + } + } +} diff --git a/tests/Crash/DerivedFileDamageTest.php b/tests/Crash/DerivedFileDamageTest.php new file mode 100644 index 0000000..5dcd37b --- /dev/null +++ b/tests/Crash/DerivedFileDamageTest.php @@ -0,0 +1,404 @@ +db(['shardSize' => 500] + $config); + $rows = []; + for ($i = 0; $i < 1500; $i++) { + $rows[] = ['g' => 'g' . ($i % 37), 'u' => 'value-' . $i . str_repeat('-', 30), + 'p' => ['type' => 'Point', 'coordinates' => [20 + ($i % 90) / 10, 30 + ($i % 70) / 10]]]; + } + $db->insert('dm', $rows); + $db->createFieldIndex('dm', 'g'); + $db->createFieldIndex('dm', 'u'); + $db->createSpatialIndex('dm', 'p'); + $db->find('dm', ['g' => 'g5']); + \noneDB::clearStaticCache(); + } + + /** The derived files of the database */ + private function derived(): array + { + return array_values(array_filter(array_map(function ($file) { + return $this->dbPath('dm', 'home') . $file; + }, $this->dbFiles('dm')), function ($file) { + return preg_match('/\.(kix|fx|gfx|rx|grx)$/', $file) === 1; + })); + } + + /** Everything asked of the database, as a new process asks it */ + private function answers(): string + { + return $this->runChild('$db = ' . $this->childDb(['shardSize' => 500]) . '; $out = [];' + . ' $out[] = $db->checkIntegrity("dm")["records"];' // before a read that replaces a damaged file + . ' $sorted = function ($r) { $k = array_column($r, "key"); sort($k); return $k; };' + . ' $out[] = $db->find("dm", ["key" => 777])[0]["u"];' + . ' $out[] = $sorted($db->find("dm", ["g" => "g5"]));' + . ' $out[] = $sorted($db->find("dm", ["u" => "value-777" . str_repeat("-", 30)]));' + . ' $out[] = $sorted($db->query("dm")->where(["u" => ["\$gte" => "value-90", "\$lte" => "value-91"]])->get());' + . ' $out[] = $sorted($db->query("dm")->withinDistance("p", 21.0, 31.0, 30000)->get());' + . ' $out[] = $db->count("dm");' + . ' echo json_encode($out);'); + } + + /** @test §6.5: one digit of any header changed, its crc doesn't fit: the file is invalid, the answers and the next key stay */ + public function aHeaderThatDoesNotCheckMakesItsFileInvalid(): void + { + $this->store(); + $expected = $this->answers(); + $this->assertStringStartsWith('[1500,"value-777', $expected); + $files = $this->derived(); + $kinds = array_unique(array_map(function ($file) { return pathinfo($file, PATHINFO_EXTENSION); }, $files)); + sort($kinds); + $this->assertSame(['fx', 'gfx', 'grx', 'kix', 'rx'], $kinds, 'precondition: every kind'); + foreach ($files as $file) { + $intact = (string)file_get_contents($file); + $end = strpos($intact, "\n"); + preg_match_all('/":(\d+)[,}]/', substr($intact, 0, $end), $numbers, PREG_OFFSET_CAPTURE); + foreach ($numbers[1] as $n => [$digits, $at]) { + if ($n % 3 === 0) { + $changed = $intact; + $changed[$at] = $digits[0] === '1' ? '2' : '1'; + file_put_contents($file, $changed); + \noneDB::clearStaticCache(); + $this->assertSame($expected, $this->answers(), basename($file) . ' at ' . $at); + } + } + file_put_contents($file, $intact); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 500]); + $db->insert('dm', ['g' => 'new']); + $this->assertSame([1500], array_column($db->find('dm', ['g' => 'new']), 'key'), 'the next key'); + } + + /** @test A record the data file doesn't hold is not asked of it: an index that says 4 GB is repaired, the process stays */ + public function aRecordLongerThanItsFileIsNotRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('dm', array_map(function ($i) { return ['n' => $i]; }, range(0, 199))); + $kix = $this->dbPath('dm', 'kix'); + $checkpoint = $this->readKeyIndexFile($kix); + \noneDB::clearStaticCache(); + $entries = $checkpoint['entries']; + $entries[120][1] = 4000000000; + $this->writeKeyIndexFile($kix, $checkpoint['header'], $entries); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . '; echo count($db->find("dm", [])), " ", $db->find("dm", ["key" => 120])[0]["n"];'); + $this->assertSame('200 120', $out); + $this->writeKeyIndexFile($kix, $checkpoint['header'], $entries); + \noneDB::clearStaticCache(); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . '; echo json_encode(array_column($db->find("dm", ["key" => [118, 120]]), "n"));'); + $this->assertSame('[118,120]', $out, 'many keys'); + } + + /** + * @test §6.5.1 An .fx whose header or directory points outside the file: the file is invalid, however the crcs fit + * @runInSeparateProcess + */ + public function aPageOutsideTheFileIsNotRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('dm', array_map(function ($i) { return ['u' => sprintf('value-%06d', $i)]; }, range(0, 29999))); + $this->assertTrue($db->createFieldIndex('dm', 'u')['success']); + $path = $this->dbPath('dm', 'fx', 'u'); + $intact = (string)file_get_contents($path); + $line = strpos($intact, "\n"); + $header = $this->openHeaderLine($path, substr($intact, 0, $line)); + $body = substr($intact, $line + 1); + [$offset, $length] = $header['pg']['root']; + $sealed = function (array $changed) use ($path, $body): string { + return $this->sealHeaderLine(json_encode($changed)) . "\n" . $body; + }; + $reseal = function (string $page): string { return $page . pack('V', crc32($page)); }; + $root = substr($body, $offset, $length - 4); + $n = unpack('V', $root)[1]; + $cases = [ + 'a root past the file' => $sealed(['pg' => ['root' => [$offset, 3000000000]] + $header['pg']] + $header), + 'a root of no bytes' => $sealed(['pg' => ['root' => [$offset + $length, 0]] + $header['pg']] + $header), + 'a root that is not a pair' => $sealed(['pg' => ['root' => [$offset]] + $header['pg']] + $header), + 'more levels than a tree can have' => $sealed(['pg' => ['levels' => 64] + $header['pg']] + $header), + 'levels below none' => $sealed(['pg' => ['levels' => -1] + $header['pg']] + $header), + 'a header whose h is no string' => $sealed(['h' => []] + $header), + 'a header that covers less than nothing' => $sealed(['s' => -1] + $header), + 'a directory that names itself' => $this->sealHeaderLine(json_encode(['pg' => ['levels' => 1 << 40] + $header['pg']] + $header)) . "\n" . substr_replace($body, $reseal(substr_replace($root, pack('V2', $offset, $length), 4 + $n, 8)), $offset, $length), + 'a child past the file' => substr($intact, 0, $line + 1) . substr_replace($body, $reseal(substr_replace($root, pack('V', 3000000000), 4 + $n + 4, 4)), $offset, $length), + 'a child of no bytes' => substr($intact, 0, $line + 1) . substr_replace($body, $reseal(substr_replace($root, pack('V', 0), 4 + $n + 4, 4)), $offset, $length), + ]; + foreach ($cases as $case => $bytes) { + file_put_contents($path, $bytes); + \noneDB::clearStaticCache(); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . '; echo json_encode(array_column($db->find("dm", ["u" => "value-000001"]), "key"));'); + $this->assertSame('[1]', $out, $case); + } + } + + /** + * @test §6.5 version rule: a .grx of a version the engine doesn't know is not used. Its entries fit their shards and + * their boxes lie elsewhere: read, they would skip every shard + */ + public function aGlobalSpatialFileOfAnotherVersionIsNotUsed(): void + { + $this->store(); + $expected = $this->answers(); + $path = $this->dbPath('dm', 'grx', 'p'); + $raw = (string)file_get_contents($path); + $end = (int)strpos($raw, "\n"); + $header = $this->openHeaderLine($path, substr($raw, 0, $end)); + foreach (array_keys($header['sh']) as $shard) { + $header['sh'][$shard]['b'] = [-1500000000, -800000000, -1400000000, -700000000]; + } + foreach ([1 => false, 2 => true] as $version => $same) { + $header['v'] = $version; + file_put_contents($path, $this->sealHeaderLine(json_encode($header)) . substr($raw, $end)); + \noneDB::clearStaticCache(); + $same ? $this->assertSame($expected, $this->answers()) : $this->assertNotSame($expected, $this->answers(), 'precondition: read, the file skips the shards'); + } + } + + /** + * @test §6.5: numbers of a header that its crc fits and no file can hold: keys past the integers, bytes before the + * file. The file is invalid, the answers stay + */ + public function aHeaderWithImpossibleNumbersIsInvalid(): void + { + $this->store(); + $expected = $this->answers(); + foreach ([ + ['kix', [1], 'base', PHP_INT_MAX], + ['kix', [1], 'base', PHP_INT_MAX - 500], + ['fx', ['g', 1], 's', -1], + ['rx', ['p', 1], 's', -1], + ] as [$role, $args, $member, $value]) { + $path = $this->dbPath('dm', $role, ...$args); + $raw = (string)file_get_contents($path); + $end = $role === 'kix' ? 511 : (int)strpos($raw, "\n"); + $header = $this->openHeaderLine($path, rtrim(substr($raw, 0, $end), ' ')); + $header[$member] = $value; + $line = $this->sealHeaderLine(json_encode($header)); + file_put_contents($path, ($role === 'kix' ? str_pad($line, 511) : $line) . substr($raw, $end)); + \noneDB::clearStaticCache(); + $this->assertSame($expected, $this->answers(), "$role $member $value"); + file_put_contents($path, $raw); + } + } + + /** + * @test §6.5: a member of a global index's entry that has another type, or a place before the file, makes it no entry: + * that shard is read, no error of PHP comes out. The crc fits: the file is hand-made. Its lock epoch is not the lock's, + * so the entries are read + */ + public function aGlobalEntryOfOtherTypesIsNoEntry(): void + { + $this->store(); + $expected = $this->answers(); + $values = [[1], 7, -1, 'x', 1.5, true, 'absent']; + foreach (['gfx' => $this->dbPath('dm', 'gfx', 'g'), 'grx' => $this->dbPath('dm', 'grx', 'p')] as $kind => $path) { + $raw = (string)file_get_contents($path); + $end = (int)strpos($raw, "\n"); + $header = $this->openHeaderLine($path, substr($raw, 0, $end)); + $this->assertCount(3, $header['sh'], 'precondition: an entry per shard'); + foreach ($kind === 'grx' ? ['s', 'e', 'h', 'b'] : ['s', 'e', 'h'] as $member) { + foreach ($values as $value) { + $changed = $header; + $changed['le'] = null; + foreach (array_keys($changed['sh']) as $shard) { + if ($value === 'absent') { + unset($changed['sh'][$shard][$member]); + } else { + $changed['sh'][$shard][$member] = $value; + } + } + file_put_contents($path, $this->sealHeaderLine(json_encode($changed)) . substr($raw, $end)); + \noneDB::clearStaticCache(); + $this->assertSame($expected, $this->answers(), "$kind $member " . json_encode($value)); + } + } + file_put_contents($path, $raw); + } + } + + /** One segment of 300 points on a grid with its R-tree, written by a process that has ended: [path, header, nodes, entries] */ + private function tree(): array + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('rt', array_map(function ($i) { + return ['p' => ['type' => 'Point', 'coordinates' => [20 + ($i % 30) / 10, 30 + intdiv($i, 30) / 10]]]; + }, range(0, 299))); + $this->assertTrue($db->createSpatialIndex('rt', 'p')['success']); + \noneDB::clearStaticCache(); + $path = $this->dbPath('rt', 'rx', 'p'); + [$line, $nodes, $entries] = explode("\n", (string)file_get_contents($path), 3); + return [$path, $this->openHeaderLine($path, $line), json_decode($nodes, true), $entries]; + } + + /** The keys of the grid's first five columns of its first five rows, as a new process reads them */ + private function pointsInTheBox(): array + { + \noneDB::clearStaticCache(); + $keys = array_column($this->db(['shardingEnabled' => false])->query('rt')->withinBBox('p', 19.95, 29.95, 20.45, 30.45)->get(), 'key'); + sort($keys); + return $keys; + } + + /** The box's keys by the model of the grid */ + private function modelOfTheBox(): array + { + $keys = []; + for ($row = 0; $row < 5; $row++) { + for ($column = 0; $column < 5; $column++) { + $keys[] = 30 * $row + $column; + } + } + return $keys; + } + + /** @test §6.5: the first coordinate of leaf entries changed in an .rx body: the body's crc doesn't fit, no record goes missing */ + public function aDamagedRTreeBodyChangesNoAnswer(): void + { + [$path, , , $entries] = $this->tree(); + $this->assertSame(0, strlen($entries) % 40, 'precondition: entries of five 64-bit integers'); + $raw = (string)file_get_contents($path); + $damaged = $entries; + for ($at = 0; $at < strlen($entries); $at += 40) { + if (unpack('P', $entries, $at + 8)[1] === 200000000) { + $damaged = substr_replace($damaged, pack('P', 900000000), $at + 8, 8); + } + } + $this->assertNotSame($entries, $damaged, 'precondition: the entries of the box\'s first column changed'); + file_put_contents($path, substr($raw, 0, strlen($raw) - strlen($entries)) . $damaged); + + $this->assertSame($this->modelOfTheBox(), $this->pointsInTheBox()); + $this->assertTrue($this->db(['shardingEnabled' => false])->checkIntegrity('rt')['ok']); + } + + /** + * @test §6.5: an .rx whose crcs fit but which is no tree the code can walk is invalid: rebuilt from the data, no + * error of PHP, no walk without an end. The read, a write of more records than a node holds (a split) and their delete + * @dataProvider notTrees + */ + public function aFileThatIsNoTreeIsRefused(callable $change): void + { + [$path, $header, $nodes, $entries] = $this->tree(); + $root = $header['root']; + $this->assertGreaterThan(0, $nodes[$root][0], 'precondition: the root is above the leaves'); + $change($header, $nodes, $root, $nodes[$root][2][0]); + $body = json_encode($nodes) . "\n" . $entries; + $header['bc'] = crc32($body); + $made = $this->sealHeaderLine(json_encode($header)) . "\n" . $body; + file_put_contents($path, $made); + + $model = $this->modelOfTheBox(); + $this->assertSame($model, $this->pointsInTheBox()); + $this->assertNotSame($made, (string)file_get_contents($path), 'the file was refused and made again'); + + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('rt', array_fill(0, 40, ['new' => true, 'p' => ['type' => 'Point', 'coordinates' => [20.1, 30.1]]])); + $this->assertSame(array_merge($model, range(300, 339)), $this->pointsInTheBox()); + $this->assertSame(40, $this->db(['shardingEnabled' => false])->delete('rt', ['new' => true])['n']); + $this->assertSame($model, $this->pointsInTheBox()); + } + + public function notTrees(): array + { + $members = []; + foreach ([0, 1, 2, 3] as $i) { + $members["box member $i"] = [function (&$h, &$n, $root, $leaf) use ($i) { $n[$leaf][1][$i] = 'x'; }]; + } + return $members + [ + 'nodes that are a string' => [function (&$h, &$n, $root, $leaf) { $n = str_repeat('x', $root + 1); }], + 'a node that is no array' => [function (&$h, &$n, $root, $leaf) { $n[$leaf] = 5; }], + 'a leaf without a number of entries' => [function (&$h, &$n, $root, $leaf) { unset($n[$leaf][2]); }], + 'a leaf of more entries than the body holds' => [function (&$h, &$n, $root, $leaf) { $n[$leaf][2]++; }], + 'a leaf of fewer' => [function (&$h, &$n, $root, $leaf) { $n[$leaf][2]--; }], + 'a number of entries that is no integer' => [function (&$h, &$n, $root, $leaf) { $n[$leaf][2] = 1.5; }], + 'a number of entries that is an array' => [function (&$h, &$n, $root, $leaf) { $n[$leaf][2] = [1]; }], + 'a number of entries below zero' => [function (&$h, &$n, $root, $leaf) { $other = $n[$root][2][1]; $n[$other][2] += $n[$leaf][2] + 1; $n[$leaf][2] = -1; }], + 'a level that is no integer' => [function (&$h, &$n, $root, $leaf) { $n[$root][0] = 'x'; }], + 'a root below the leaves' => [function (&$h, &$n, $root, $leaf) { $n[$root] = [-1, $n[$root][1], []]; }], + 'children that are no array' => [function (&$h, &$n, $root, $leaf) { $n[$root][2] = 7; }], + 'a box of three' => [function (&$h, &$n, $root, $leaf) { array_pop($n[$leaf][1]); }], + 'no box on a node with entries' => [function (&$h, &$n, $root, $leaf) { $n[$leaf][1] = null; }], + 'a child that is missing' => [function (&$h, &$n, $root, $leaf) { $n[$root][2][0] = $h['next'] + 7; }], + 'a child that is no integer' => [function (&$h, &$n, $root, $leaf) { $n[$root][2][0] = 'x'; }], + 'a child that is an array' => [function (&$h, &$n, $root, $leaf) { $n[$root][2][0] = [1]; }], + 'a child at its parent\'s level' => [function (&$h, &$n, $root, $leaf) { $n[$leaf] = [$n[$root][0], $n[$leaf][1], [$root]]; }], + 'a child of itself' => [function (&$h, &$n, $root, $leaf) { $n[$root][2][0] = $root; }], + 'a child named twice' => [function (&$h, &$n, $root, $leaf) { $n[$root][2][1] = $leaf; }], + 'a node id at the next id' => [function (&$h, &$n, $root, $leaf) { $n[$h['next']] = [0, null, []]; }], + 'a root that is no node' => [function (&$h, &$n, $root, $leaf) { $h['root'] = $h['next'] + 5; }], + 'a root that is no integer' => [function (&$h, &$n, $root, $leaf) { $h['root'] = [$root]; }], + 'a next id that is no integer' => [function (&$h, &$n, $root, $leaf) { $h['next'] = 'x'; }], + 'a node size of no integer' => [function (&$h, &$n, $root, $leaf) { $h['ns'] = 'x'; }], + 'a node size of nothing' => [function (&$h, &$n, $root, $leaf) { $h['ns'] = 0; }], + 'a next id among the nodes' => [function (&$h, &$n, $root, $leaf) { $h['next'] = $leaf; }], + 'a version the engine does not know' => [function (&$h, &$n, $root, $leaf) { $h['v'] = 2; }], + 'a highest key that is no integer' => [function (&$h, &$n, $root, $leaf) { $h['mk'] = []; }], + 'a count that is no integer' => [function (&$h, &$n, $root, $leaf) { $h['cnt'] = []; }], + ]; + } + + /** + * @test Bytes of derived files changed at random, in the header (its crc left, or made to fit, as a hand-made file) + * and in pages sealed again: every answer stays while a crc stands in the way, and in no case does more than a + * noneDB error come out. Seeds: NONEDB_FUZZ_SEEDS. + */ + public function damagedDerivedFilesNeverChangeAnAnswer(): void + { + $this->store(); + $expected = $this->answers(); + $source = rtrim($this->testDbDir, '/'); + $copy = sys_get_temp_dir() . '/nonedb_damage_' . bin2hex(random_bytes(6)); + exec('cp -R ' . escapeshellarg($source) . ' ' . escapeshellarg($copy)); + $junk = [null, -1, 0, 1, PHP_INT_MAX, 'x', '', [], [1, 2], 1.5, true, ['a' => 1]]; + $change = function (array &$header) use (&$change, $junk): void { + $key = array_rand($header); + if (is_array($header[$key]) && $header[$key] !== [] && mt_rand(0, 1)) { + $change($header[$key]); + } else { + $header[$key] = $junk[array_rand($junk)]; + } + }; + try { + foreach (explode(',', getenv('NONEDB_FUZZ_SEEDS') ?: '1,2') as $seed) { + mt_srand((int)$seed); + for ($round = 0; $round < 30; $round++) { + \noneDB::clearStaticCache(); + exec('rm -rf ' . escapeshellarg($source) . ' && cp -R ' . escapeshellarg($copy) . ' ' . escapeshellarg($source)); + $files = $this->derived(); + $file = $files[array_rand($files)]; + $raw = (string)file_get_contents($file); + $kix = substr($file, -4) === '.kix'; + $end = $kix ? 511 : strpos($raw, "\n"); + $handMade = (bool)mt_rand(0, 1); + $header = $this->openHeaderLine($file, rtrim(substr($raw, 0, $end), ' ')); + $change($header); + $line = json_encode($header); + $line = $handMade ? $this->sealHeaderLine($line) : substr($line, 0, -1) . ',"crc":1}'; + file_put_contents($file, ($kix ? str_pad($line, 511) : $line) . substr($raw, $end)); + $label = "seed $seed round $round " . basename($file) . ($handMade ? ' hand-made' : ' damaged'); + $answers = $this->answers(); + $this->assertStringNotContainsString('Fatal', $answers, $label); + $this->assertStringNotContainsString('Warning', $answers, $label); + if (!$handMade) { + $this->assertSame($expected, $answers, $label); + } + } + } + } finally { + exec('rm -rf ' . escapeshellarg($copy)); + } + } +} diff --git a/tests/Crash/DurabilityContractTest.php b/tests/Crash/DurabilityContractTest.php new file mode 100644 index 0000000..6f88906 --- /dev/null +++ b/tests/Crash/DurabilityContractTest.php @@ -0,0 +1,833 @@ +markTestSkipped('durable mode needs directory syncs'); + } + Testing::set('osFamily', 'Linux'); + } + + protected function tearDown(): void + { + foreach (['osFamily', 'syncFn', 'flushFn', 'writeFn', 'renameFn'] as $hook) { + Testing::set($hook, null); + } + foreach ($this->outside as $dir) { + $this->removeDirectoryTree($dir); + } + $this->outside = []; + parent::tearDown(); + } + + /** A path outside dbDir (not created), resolved as backup() resolves it */ + private function outsideDir(): string + { + $dir = realpath(sys_get_temp_dir()) . '/nonedb_backup_' . bin2hex(random_bytes(6)); + $this->outside[] = $dir; + return $dir; + } + + // ------------------------------------------------------------------ helpers + + private function durable(array $config = []): \noneDB + { + return $this->db(array_merge(['durability' => 'durable'], $config)); + } + + private function rel(string $path): string + { + $dir = rtrim($this->testDbDir, '/'); + if ($path === $dir) { + return '.'; + } + return strncmp($path, $dir . '/', strlen($dir) + 1) === 0 ? substr($path, strlen($dir) + 1) : $path; + } + + /** + * Sync hook: $fail(kind, relative path, absolute path) says whether this sync fails; the others succeed + * without touching the disk (the order and the outcome are what is tested here) + */ + private function syncs(callable $fail): void + { + $this->calls = []; + Testing::set('syncFn', function ($handle, string $kind, $path) use ($fail) { + $ok = !$fail($kind, $this->rel((string)$path), (string)$path); + $this->calls[] = [$kind, $this->rel((string)$path), $ok]; + return $ok; + }); + } + + /** Fail the $n-th sync of $kind whose relative path matches $pattern (fnmatch) */ + private function failNth(string $kind, string $pattern, int $n = 1): void + { + $seen = 0; + $this->syncs(function ($k, $rel) use ($kind, $pattern, $n, &$seen) { + return $k === $kind && fnmatch($pattern, $rel) && ++$seen === $n; + }); + } + + /** Record the syncs without failing any */ + private function recordSyncs(): void + { + $this->syncs(function () { return false; }); + } + + private function failing(int $code, callable $fn): \RuntimeException + { + try { + $fn(); + } catch (\RuntimeException $e) { + $this->assertSame($code, $e->getCode(), get_class($e) . ': ' . $e->getMessage()); + return $e; + } + $this->fail('expected a RuntimeException with code ' . $code); + } + + private function last(\noneDB $db): array + { + $last = $db->getDurability()['lastOperation']; + $this->assertIsArray($last); + return $last; + } + + /** + * Records of a database after a restart (static caches cleared, hooks off, a new instance); + * checkIntegrity() must find nothing to report unless $integrity is false + */ + private function afterRestart(string $name, array $config = [], bool $integrity = true): array + { + Testing::set('syncFn', null); + Testing::set('flushFn', null); + Testing::set('writeFn', null); + Testing::set('renameFn', null); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $records = $db->find($name, 0); + if ($integrity) { + $report = $db->checkIntegrity($name); + $this->assertTrue($report['ok'], json_encode($report)); + } + return $this->plain($records); + } + + private function bytes(string $file): ?string + { + clearstatcache(true, $file); + return is_file($file) ? (string)file_get_contents($file) : null; + } + + private function lockRecordHas(string $path, string $token): bool + { + $raw = (string)@file_get_contents($path); + return in_array($token, explode(' ', trim($raw)), true) || strpos($raw, ' ' . $token) !== false; + } + + // ------------------------------------------------------------------ §3 row 1: append to an existing file + + /** @test a failed file sync of an append: rolled back, DURABILITY_FAILED, nothing applied */ + public function anAppendWhoseSyncFailsIsRolledBack(): void + { + foreach (['single' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 2], 'buffered' => ['shardingEnabled' => false, 'bufferEnabled' => true]] as $case => $config) { + $this->cleanTestDirectory(); + $db = $this->durable($config); + $model = [['v' => 'a'], ['v' => 'b'], ['v' => 'c']]; + $db->insert('ap', $model); + $file = $case === 'sharded' ? '*/segments/s1.jsonl' : ($case === 'buffered' ? '*/buffer/pending.jsonl' : '*/segments/main.jsonl'); + $data = glob($this->dbPath('ap', 'gen') . ($case === 'buffered' ? 'buffer/pending.jsonl' : 'segments/*.jsonl')); + $before = array_map([$this, 'bytes'], $data); + + $this->failNth('file', $file); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('ap', ['v' => 'd']); }); + $last = $this->last($db); + $this->assertSame(['insert', 'ap', 'rolled_back'], [$last['op'], $last['db'], $last['outcome']], $case); + $this->assertContains('file', $last['required'], $case); + $this->assertNotContains('file', $last['completed'], $case); + $this->assertSame($before, array_map([$this, 'bytes'], $data), "$case: the appended bytes were cut back"); + $this->assertSame($model, $this->afterRestart('ap', $config), $case); + + // A failure is never carried over: the next write commits with every required sync done + $db = $this->durable($config); + $db->insert('ap', ['v' => 'e']); + $last = $this->last($db); + $this->assertSame('committed', $last['outcome'], $case); + $this->assertSame($last['required'], $last['completed'], $case); + $model[] = ['v' => 'e']; + $this->assertSame($model, $this->afterRestart('ap', $config), $case); + } + } + + /** @test §4: the cut of a failed append is synced; when that sync fails the outcome is unknown and "J" is set */ + public function anUndoWhoseSyncFailsIsUnknown(): void + { + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('un', [['v' => 1], ['v' => 2]]); + // 1st sync of the data file: the append (fails); 2nd: its cut (fails too) + $this->syncs(function ($kind, $rel) { return $kind === 'file' && fnmatch('*/segments/main.jsonl', $rel); }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->insert('un', ['v' => 3]); }); + $this->assertSame('unknown', $this->last($db)['outcome']); + $this->assertSame(2, count(array_filter($this->calls, function ($c) { return $c[0] === 'file' && fnmatch('*/segments/main.jsonl', $c[1]); })), + 'the append and its cut were each synced once'); + $this->assertContains('J', $this->lockState('un')['flags'], 'the next session recovers the database'); + // The cut itself happened in this process: the record is not there; the next access recovers + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('un')); + $this->assertNotContains('J', $this->lockState('un')['flags']); + } + + /** @test §4: an append whose cut fails (any mode) is unknown, never a plain failure */ + public function anAppendThatCannotBeCutBackIsUnknown(): void + { + foreach (['performance', 'durable'] as $mode) { + $this->cleanTestDirectory(); + $db = $this->db(['durability' => $mode, 'shardingEnabled' => false]); + $db->insert('cut', ['v' => 1]); + Testing::set('flushFn', function () { return false; }); + $this->setFaultHook(function ($point) { + if ($point === 'append:undo' || $point === 'rollback:truncate') { + throw new \RuntimeException('disk gone'); + } + }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->insert('cut', ['v' => 2]); }); + $this->setFaultHook(null); + $this->assertSame('unknown', $this->last($db)['outcome'], $mode); + // Nothing was cut: the complete line is a record (roll forward), for every reader + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('cut'), $mode); + } + } + + // ------------------------------------------------------------------ §3 row 2: append that creates the file + + /** @test the directory sync of an append that created its file is part of the commit */ + public function aCreatingAppendWhoseDirectorySyncFailsIsRolledBack(): void + { + // A new buffer file + $db = $this->durable(['shardingEnabled' => false, 'bufferEnabled' => true]); + $db->createDB('cb'); + $this->failNth('dir', '*/buffer'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('cb', ['v' => 1]); }); + $last = $this->last($db); + $this->assertSame('rolled_back', $last['outcome']); + $this->assertSame(['file', 'dir'], $last['required']); + $this->assertSame(['file'], $last['completed']); + $this->assertSame('', (string)$this->bytes($this->dbPath('cb', 'buffer')), 'cut back to empty'); + $this->assertSame([], $this->afterRestart('cb', ['bufferEnabled' => true])); + + // A new shard file + $this->cleanTestDirectory(); + $db = $this->durable(['shardSize' => 2]); + $db->insert('cs', [['v' => 0], ['v' => 1]]); + $this->failNth('dir', '*/segments'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('cs', ['v' => 2]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame('', (string)$this->bytes($this->dbPath('cs', 'data', 1))); + $this->assertSame([['v' => 0], ['v' => 1]], $this->afterRestart('cs')); + $db = $this->durable(['shardSize' => 2]); + $db->insert('cs', ['v' => 2]); + $this->assertSame([['v' => 0], ['v' => 1], ['v' => 2]], $this->afterRestart('cs')); + } + + // ------------------------------------------------------------------ §3 row 3: state-record flags + + /** @test "b", "j", "c" and a new epoch are synced before the change they protect; a failure changes nothing */ + public function aFlagWhoseSyncFailsStopsTheWriteBeforeTheData(): void + { + // b: the first buffered insert after a flush sets it + $db = $this->durable(['shardingEnabled' => false, 'bufferEnabled' => true]); + $db->insert('fb', ['v' => 1]); + $db->flush('fb'); + $this->assertNotContains('b', $this->lockState('fb')['flags']); + $buffer = $this->bytes($this->dbPath('fb', 'buffer')); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && $this->lockRecordHas($path, 'b'); }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('fb', ['v' => 2]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame($buffer, $this->bytes($this->dbPath('fb', 'buffer')), 'nothing appended to the buffer'); + $this->assertSame([['v' => 1]], $this->afterRestart('fb', ['bufferEnabled' => true])); + + // j: a sharded write + $this->cleanTestDirectory(); + $db = $this->durable(['shardSize' => 2]); + $db->insert('fj', [['v' => 0], ['v' => 1], ['v' => 2]]); + $data = [$this->bytes($this->dbPath('fj', 'data', 0)), $this->bytes($this->dbPath('fj', 'data', 1))]; + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && strpos((string)file_get_contents($path), ' j=') !== false; }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('fj', ['v' => 3]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame($data, [$this->bytes($this->dbPath('fj', 'data', 0)), $this->bytes($this->dbPath('fj', 'data', 1))]); + $this->assertSame([['v' => 0], ['v' => 1], ['v' => 2]], $this->afterRestart('fj')); + + // c: creation + $this->cleanTestDirectory(); + $db = $this->durable(); + $db->createDB('other'); // the store exists + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && $this->lockRecordHas($path, 'c'); }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->createDB('fc'); }); + $this->assertSame(['createDB', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertFileDoesNotExist($this->dbPath('fc', 'format')); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $this->assertSame(['other'], $this->db()->getDBs(false), 'the half-created database is absent'); + $this->assertSame([], $this->db()->find('fc', 0)); + + // epoch: compaction replaces the data file only after its new epoch is durable + $this->cleanTestDirectory(); + $db = $this->durable(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); + $db->insert('fe', [['v' => 1], ['v' => 2]]); + $db->update('fe', [['v' => 1], ['set' => ['w' => 1]]]); + $data = $this->bytes($this->dbPath('fe', 'data')); + $epoch = $this->lockState('fe')['epoch']; + $this->syncs(function ($kind, $rel, $path) use ($epoch) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && strpos((string)file_get_contents($path), ':' . $epoch) === false; }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->compact('fe'); }); + $this->assertSame(['compact', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame($data, $this->bytes($this->dbPath('fe', 'data')), 'the data file was not replaced'); + $this->assertSame([['v' => 1, 'w' => 1], ['v' => 2]], $this->afterRestart('fe')); + } + + /** @test "d": its sync failing withdraws the drop (durably); if that fails too the outcome is unknown */ + public function aDropWhoseFlagCannotBeSyncedLeavesTheDatabase(): void + { + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('fd', [['v' => 1], ['v' => 2]]); + // Only the record with "d" fails: the withdrawal (without "d") is synced + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && $this->lockRecordHas($path, 'd'); }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->dropDB('fd'); }); + $this->assertSame(['dropDB', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertNotContains('d', $this->lockState('fd')['flags']); + $this->assertFileExists($this->dbPath('fd', 'format')); + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('fd')); + + // Every record sync fails: the drop flag can't be withdrawn durably + $db = $this->durable(['shardingEnabled' => false]); + $this->syncs(function ($kind, $rel) { return $kind === 'file' && fnmatch('*/LOCK', $rel); }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->dropDB('fd'); }); + $this->assertSame('unknown', $this->last($db)['outcome']); + $this->assertNotContains('d', $this->lockState('fd')['flags'], 'withdrawn in this process'); + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('fd')); + } + + /** @test "g": a generation switch whose intent can't be synced keeps the old generation */ + public function aGenerationSwitchWhoseIntentCannotBeSyncedKeepsTheOldGeneration(): void + { + $db = $this->durable(['shardingEnabled' => false]); + $model = [['v' => 1], ['v' => 2], ['v' => 3]]; + $db->insert('fg', $model); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && strpos((string)file_get_contents($path), ' g=') !== false; }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->convertToSharded('fg'); }); + $this->assertSame(['convertToSharded', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame($model, $this->afterRestart('fg')); + $info = $this->db()->getShardInfo('fg'); + $this->assertFalse($info['sharded']); + $this->assertSame(1, json_decode((string)file_get_contents($this->dbPath('fg', 'current')), true)['generation']); + $this->assertDirectoryDoesNotExist($this->dbPath('fg', 'home') . 'generations/2', 'the prepared generation was removed by recovery'); + $this->assertTrue($this->durable()->convertToSharded('fg')['success']); + $this->assertSame($model, $this->afterRestart('fg')); + } + + // ------------------------------------------------------------------ §3 row 4: atomic replacement + + /** @test a failed sync of the temp file: nothing replaced; of the directory after the rename: unknown */ + public function anAtomicReplacementFailsBeforeOrAfterItsRename(): void + { + // shards.json (the key reservation of a sharded write) + $db = $this->durable(['shardSize' => 2]); + $model = [['v' => 0], ['v' => 1], ['v' => 2]]; + $db->insert('ar', $model); + $shards = $this->bytes($this->dbPath('ar', 'shards')); + $this->failNth('file', '*/shards.json'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('ar', ['v' => 3]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame($shards, $this->bytes($this->dbPath('ar', 'shards')), 'not replaced'); + $this->assertSame([], glob($this->dbPath('ar', 'gen') . '*.tmp*'), 'no temp file left'); + $this->assertSame($model, $this->afterRestart('ar')); + + $db = $this->durable(['shardSize' => 2]); + $genDir = $this->rel(rtrim($this->dbPath('ar', 'gen'), '/')); + $this->syncs(function ($kind, $rel) use ($genDir) { return $kind === 'dir' && $rel === $genDir; }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->insert('ar', ['v' => 3]); }); + $this->assertSame('unknown', $this->last($db)['outcome']); + $this->assertContains('J', $this->lockState('ar')['flags']); + // The reservation was replaced, the data append never ran + $this->assertSame($model, $this->afterRestart('ar')); + + // FORMAT: the commit point of a creation + $this->cleanTestDirectory(); + $db = $this->durable(); + $db->createDB('other'); + $this->failNth('file', '*/FORMAT'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->createDB('af'); }); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $this->assertSame(['other'], $this->db()->getDBs(false)); + + $db = $this->durable(); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'dir' && fnmatch('*.ndb4', $rel) && is_file($path . '/FORMAT') && strpos($rel, $this->locatorOf('af')) === 0; }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->createDB('af'); }); + $this->assertSame(['createDB', 'unknown'], [$this->last($db)['op'], $this->last($db)['outcome']]); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $dbs = $this->db()->getDBs(false); + sort($dbs); + $this->assertSame(['af', 'other'], $dbs, 'in this process FORMAT is there: the creation completes'); + $this->assertSame([], $this->db()->find('af', 0)); + $this->assertNotContains('c', $this->lockState('af')['flags'], 'its next access finished the creation'); + + // CURRENT: the commit point of a generation switch + $this->cleanTestDirectory(); + $db = $this->durable(['shardingEnabled' => false]); + $model = [['v' => 1], ['v' => 2]]; + $db->insert('ac', $model); + $this->syncs(function ($kind, $rel, $path) { + return $kind === 'dir' && fnmatch('*.ndb4', $rel) && is_file($path . '/CURRENT') + && (json_decode((string)file_get_contents($path . '/CURRENT'), true)['generation'] ?? 0) === 2; + }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->convertToSharded('ac'); }); + $this->assertSame('unknown', $this->last($db)['outcome']); + $this->assertSame($model, $this->afterRestart('ac')); + $this->assertTrue($this->db()->getShardInfo('ac')['sharded'], 'CURRENT names generation 2 in this process'); + $this->assertDirectoryDoesNotExist($this->dbPath('ac', 'home') . 'generations/1'); + + // indexes.json: createFieldIndex() commits with it + $this->cleanTestDirectory(); + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('ai', [['v' => 1], ['v' => 2]]); + $this->failNth('file', '*/indexes.json'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->createFieldIndex('ai', 'v'); }); + $this->assertSame(['createFieldIndex', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('ai')); + $this->assertFalse($this->db()->hasFieldIndex('ai', 'v')); + } + + /** @test §4: a replacement whose rename fails leaves the target and no temp file; the outcome is rolled back */ + public function aReplacementWhoseRenameFailsChangesNothing(): void + { + $db = $this->durable(['shardSize' => 2]); + $model = [['v' => 0], ['v' => 1], ['v' => 2]]; + $db->insert('rn', $model); + $shards = $this->bytes($this->dbPath('rn', 'shards')); + Testing::set('renameFn', function ($from, $to) { return substr($to, -12) === '/shards.json' ? false : rename($from, $to); }); + try { + $db->insert('rn', ['v' => 3]); + $this->fail('must throw'); + } catch (\RuntimeException $e) { + $this->assertNotContains($e->getCode(), [\noneDB::OUTCOME_UNKNOWN, \noneDB::DURABILITY_FAILED], 'the original error'); + } + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame($shards, $this->bytes($this->dbPath('rn', 'shards'))); + $this->assertSame([], glob($this->dbPath('rn', 'gen') . '*.tmp*')); + $this->assertSame($model, $this->afterRestart('rn')); + } + + // ------------------------------------------------------------------ §3 rows 5–7: compaction, generation, create/drop + + /** @test compaction: temp file sync (both modes), directory sync after the rename (durable mode) */ + public function compactionFailsBeforeOrAfterTheReplacement(): void + { + foreach (['performance', 'durable'] as $mode) { + $this->cleanTestDirectory(); + $config = ['shardingEnabled' => false, 'autoCompactThreshold' => 1000]; // garbage stays until compact() + $db = $this->db(['durability' => $mode] + $config); + $db->insert('cp', [['v' => 1], ['v' => 2], ['v' => 3]]); + $db->update('cp', [['v' => 2], ['set' => ['w' => 2]]]); + $db->delete('cp', ['v' => 3]); + $model = [['v' => 1], ['v' => 2, 'w' => 2]]; + $data = $this->bytes($this->dbPath('cp', 'data')); + + $this->failNth('file', '*/segments/main.jsonl'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->compact('cp'); }); + $this->assertSame(['compact', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']], $mode); + $this->assertSame($data, $this->bytes($this->dbPath('cp', 'data')), $mode); + $this->assertSame([], glob($this->dbPath('cp', 'gen') . 'segments/*.tmp*'), $mode); + $this->assertSame($model, $this->afterRestart('cp', $config), $mode); + $this->assertSame($data, $this->bytes($this->dbPath('cp', 'data')), "$mode: still not compacted"); + + $db = $this->db(['durability' => $mode] + $config); + $this->failNth('dir', '*/segments'); + if ($mode === 'durable') { + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->compact('cp'); }); + $this->assertSame('unknown', $this->last($db)['outcome']); + } else { + $db->compact('cp'); // performance mode makes no power-loss claim: the directory sync is best effort + $this->assertSame('committed', $this->last($db)['outcome']); + } + $this->assertNotSame($data, $this->bytes($this->dbPath('cp', 'data')), "$mode: replaced in this process"); + $this->assertSame($model, $this->afterRestart('cp', $config), $mode); + } + } + + /** @test generation switch: a failed sync while the new generation is prepared keeps the old one */ + public function aGenerationThatCannotBePreparedIsDropped(): void + { + $db = $this->durable(['shardingEnabled' => false, 'shardSize' => 2]); + $model = [['v' => 1], ['v' => 2], ['v' => 3]]; + $db->insert('gp', $model); + $this->failNth('file', '*/generations/2/segments/s1.jsonl'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->convertToSharded('gp'); }); + $this->assertSame(['convertToSharded', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame($model, $this->afterRestart('gp')); + $this->assertFalse($this->db()->getShardInfo('gp')['sharded']); + $this->assertNotContains('g', array_map(function ($f) { return $f[0]; }, $this->lockState('gp')['flags'])); + $this->assertTrue($this->durable()->convertToSharded('gp')['success']); + $this->assertSame($model, $this->afterRestart('gp')); + $this->assertTrue($this->db()->getShardInfo('gp')['sharded']); + } + + /** @test creation: a failed sync before FORMAT leaves no database; drop: FORMAT's removal commits */ + public function creationAndDropCommitWithFormat(): void + { + $db = $this->durable(); + $db->createDB('keep'); + foreach (['*/generations/1/metadata', '*/CURRENT'] as $pattern) { + $this->failNth('file', $pattern); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('cr', ['v' => 1]); }); + $this->assertSame(['insert', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']], $pattern); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $this->assertSame(['keep'], $this->db()->getDBs(false), $pattern); + $this->assertSame([], $this->db()->find('cr', 0), $pattern); + $this->assertSame(['LOCK'], $this->dbFiles('cr'), "$pattern: the next access removed the half creation"); + $this->assertNotContains('c', $this->lockState('cr')['flags'], $pattern); + $db = $this->durable(); + } + + $db->insert('dr', [['v' => 1]]); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'dir' && fnmatch('*.ndb4', $rel) && !is_file($path . '/FORMAT') && is_file($path . '/LOCK') && strpos($rel, $this->locatorOf('dr')) === 0; }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->dropDB('dr'); }); + $this->assertSame(['dropDB', 'unknown'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertContains('d', $this->lockState('dr')['flags'], 'the next session finishes the drop'); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $this->assertSame([], $this->db()->find('dr', 0)); + $this->assertSame(['LOCK'], $this->dbFiles('dr')); + $this->assertNotContains('d', $this->lockState('dr')['flags']); + } + + // ------------------------------------------------------------------ §3 row 8: quarantine + + /** @test the quarantine copy is durable before the source is cut; a failed sync leaves the source as it was */ + public function aQuarantineThatCannotBeSyncedCutsNothing(): void + { + foreach (['file' => '*/recovery/q.tmp*', 'dir' => '*/recovery'] as $kind => $pattern) { + $this->cleanTestDirectory(); + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('qt', [['v' => 1], ['v' => 2]]); + $file = $this->dbPath('qt', 'data'); + file_put_contents($file, '{"v":3,"ke', FILE_APPEND); // a torn tail (a crashed append) + $torn = $this->bytes($file); + \noneDB::clearStaticCache(); + $db = $this->durable(['shardingEnabled' => false]); + $this->failNth($kind, $pattern); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->insert('qt', ['v' => 4]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome'], $kind); + $this->assertSame($torn, $this->bytes($file), "$kind: the torn tail is still there"); + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('qt', [], false), $kind); + $this->assertSame([['segment' => 'main', 'offset' => strlen($torn) - 10, 'length' => 10]], $this->db()->checkIntegrity('qt')['tornTail'], $kind); + + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('qt', ['v' => 4]); + $this->assertSame([['v' => 1], ['v' => 2], ['v' => 4]], $this->afterRestart('qt'), $kind); + $this->assertCount(1, glob($this->dbPath('qt', 'quarantine') . 'q.main.*.bin'), $kind); + } + } + + // ------------------------------------------------------------------ §3 row 9: maintenance + + /** @test the maintenance journal's COMMITTED record: a failed directory sync after its rename is unknown */ + public function aMaintenanceCommitThatCannotBeSyncedIsUnknown(): void + { + $this->db()->insert('mt', ['v' => 1]); + $other = 'second_registered_key_0123'; + $this->syncs(function ($kind, $rel, $path) { + return $kind === 'dir' && $rel === '.nonedb-maintenance' && is_file($path . '/journal.json') + && strpos((string)file_get_contents($path . '/journal.json'), '"COMMITTED"') !== false; + }); + $result = \noneDB::registerKey($this->testDbDir, self::SECRET, $other, ['durability' => 'durable']); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::OUTCOME_UNKNOWN, $result['code'], (string)$result['error']); + Testing::set('syncFn', null); + // The journal is open: every process is kept out until the maintenance is run again + $code = null; + try { + $this->db()->find('mt', 0); + } catch (\RuntimeException $e) { + $code = $e->getCode(); + } + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $code); + $this->assertTrue(\noneDB::registerKey($this->testDbDir, self::SECRET, $other, ['durability' => 'durable'])['success']); + $this->assertSame([['v' => 1]], $this->afterRestart('mt')); + $this->assertSame([], (new \noneDB(['secretKey' => $other, 'dbDir' => $this->testDbDir]))->find('mt', 0)); + } + + /** @test a rotation whose "m" can't be synced fails before its commit; abort restores everything */ + public function aRotationWhoseMarkCannotBeSyncedCanBeAborted(): void + { + $this->db()->insert('rt', [['v' => 1], ['v' => 2]]); + $new = 'rotated_secret_key_456789'; + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*.ndb4/LOCK', $rel) && $this->lockRecordHas($path, 'm'); }); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new, ['durability' => 'durable']); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::DURABILITY_FAILED, $result['code'], (string)$result['error']); + Testing::set('syncFn', null); + $abort = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new, ['abort' => true, 'durability' => 'durable']); + $this->assertTrue($abort['success'], (string)$abort['error']); + $this->assertSame([['v' => 1], ['v' => 2]], $this->afterRestart('rt')); + } + + /** @test a rotation's renames are durable before its commit: when that sync fails, nothing is committed */ + public function aRotationWhoseRenamesCannotBeSyncedDoesNotCommit(): void + { + $new = 'rotated_secret_key_456789'; + foreach ([true, false] as $abort) { + $this->cleanTestDirectory(); + $this->db()->insert('rr', [['v' => 1], ['v' => 2]]); + $this->syncs(function ($kind, $rel) { + $journal = (string)@file_get_contents($this->testDbDir . '.nonedb-maintenance/journal.json'); + return $kind === 'dir' && $rel === '.' && strpos($journal, '"STARTED"') !== false; + }); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new, ['durability' => 'durable']); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::DURABILITY_FAILED, $result['code'], (string)$result['error']); + $this->assertStringContainsString('"STARTED"', (string)file_get_contents($this->testDbDir . '.nonedb-maintenance/journal.json'), 'not committed'); + Testing::set('syncFn', null); + $again = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new, ['abort' => $abort, 'durability' => 'durable']); + $this->assertTrue($again['success'], (string)$again['error']); + \noneDB::clearStaticCache(); + $key = $abort ? self::SECRET : $new; + $this->assertSame([['v' => 1], ['v' => 2]], $this->plain((new \noneDB(['secretKey' => $key, 'dbDir' => $this->testDbDir]))->find('rr', 0)), + $abort ? 'aborted: the old key' : 'run again: finished'); + } + } + + // ------------------------------------------------------------------ §4: the rollback scope + + /** @test a scope's rollback cuts are synced; a cut whose sync fails makes the outcome unknown */ + public function aRollbackWhoseCutCannotBeSyncedIsUnknown(): void + { + $db = $this->durable(['shardSize' => 2]); + $db->insert('rs', [['v' => 0, 'g' => 1], ['v' => 1, 'g' => 1], ['v' => 2, 'g' => 1], ['v' => 3, 'g' => 1]]); + $model = $this->plain($db->find('rs', 0)); + // One update over both shards: s0 appended and synced, s1's sync fails (cut, synced), then the + // rollback cuts s0 and its sync fails + $s0 = 0; + $s1 = 0; + $this->syncs(function ($kind, $rel) use (&$s0, &$s1) { + if ($kind !== 'file') { + return false; + } + if (fnmatch('*/segments/s0.jsonl', $rel)) { + return ++$s0 === 2; + } + return fnmatch('*/segments/s1.jsonl', $rel) && ++$s1 === 1; + }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db) { $db->update('rs', [['g' => 1], ['set' => ['h' => 1]]]); }); + $this->assertSame([2, 2], [$s0, $s1], 's0: append + rollback cut; s1: append + its cut'); + $this->assertSame('unknown', $this->last($db)['outcome']); + $this->assertContains('J', $this->lockState('rs')['flags']); + $this->assertSame($model, $this->afterRestart('rs')); + + // The same with every cut synced: rolled back + $db = $this->durable(['shardSize' => 2]); + $this->failNth('file', '*/segments/s1.jsonl'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db) { $db->update('rs', [['g' => 1], ['set' => ['h' => 1]]]); }); + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame($model, $this->afterRestart('rs')); + } + + /** @test a failure that is not a sync keeps its own error, also in durable mode */ + public function otherFailuresKeepTheirError(): void + { + $db = $this->durable(['shardingEnabled' => false]); + $db->insert('ot', ['v' => 1]); + Testing::set('flushFn', function () { return false; }); + try { + $db->insert('ot', ['v' => 2]); + $this->fail('a failed fflush must fail the write'); + } catch (\RuntimeException $e) { + $this->assertNotContains($e->getCode(), [\noneDB::OUTCOME_UNKNOWN, \noneDB::DURABILITY_FAILED]); + } + $this->assertSame('rolled_back', $this->last($db)['outcome']); + $this->assertSame([['v' => 1]], $this->afterRestart('ot')); + } + + // ------------------------------------------------------------------ what is synced + + /** @test derived files (checkpoints, field and spatial indexes) are never synced (§3) */ + public function derivedFilesAreNeverSynced(): void + { + $db = $this->durable(['shardSize' => 50]); + $db->createFieldIndex('dv', 'g'); + $db->createSpatialIndex('dv', 'loc'); + $this->recordSyncs(); + for ($i = 0; $i < 6; $i++) { + $batch = []; + for ($j = 0; $j < 40; $j++) { + $batch[] = ['g' => $j % 3, 'loc' => ['type' => 'Point', 'coordinates' => [$j / 10, $i / 10]]]; + } + $db->insert('dv', $batch); + } + $db->find('dv', ['g' => 1]); + $db->withinBBox('dv', 'loc', 0, 0, 1, 1); + $paths = array_column($this->calls, 1); + $this->assertNotEmpty($paths); + foreach ($paths as $path) { + $this->assertDoesNotMatchRegularExpression('/\.(kix|fx|gfx|rx|grx)$/', $path); + } + $this->assertNotEmpty(glob($this->dbPath('dv', 'gen') . 'segments/*.kix'), 'checkpoints were written'); + } + + // ------------------------------------------------------------------ §3: backup and restore + + /** @test backup: every copy and every directory is synced before the manifest; a failed sync leaves no backup */ + public function aBackupIsDurableBeforeItsManifest(): void + { + $db = $this->durable(['shardSize' => 4]); + $db->insert('bk', array_map(function ($n) { return ['v' => $n]; }, range(0, 5))); + $dir = $this->outsideDir(); + $this->recordSyncs(); + $db->backup('bk', $dir); + $manifest = json_decode(file_get_contents("$dir/manifest.json"), true); + $order = []; + foreach ($this->calls as [$kind, $path]) { + if (strncmp($path, $dir, strlen($dir)) === 0) { + $order[] = [$kind, substr($path, strlen($dir))]; + } + } + $files = array_values(array_map(function ($c) { return $c[1]; }, array_filter($order, function ($c) { return $c[0] === 'file'; }))); + $this->assertEqualsCanonicalizing(array_map(function ($e) { return '/' . $e['path']; }, $manifest['files']), array_slice($files, 0, -1), + 'every copy is synced'); + $this->assertMatchesRegularExpression('#^/manifest\.json\.tmp[0-9a-f]{8}$#', end($files), 'the manifest last'); + $manifestAt = array_search(['file', end($files)], $order, true); + $dirsBefore = array_map(function ($c) { return $c[1]; }, array_filter(array_slice($order, 0, $manifestAt), function ($c) { return $c[0] === 'dir'; })); + foreach ($manifest['files'] as $entry) { + $parent = dirname('/' . $entry['path']); + $this->assertContains($parent === '/' ? '' : $parent, $dirsBefore, "{$entry['path']}: its directory synced before the manifest"); + } + $this->assertSame(['dir', ''], end($order), 'the backup directory after the manifest\'s rename'); + $lastDirSync = count(array_filter($order, function ($c) { return $c === ['dir', '']; })); + + foreach ([['file', '/generations/*/segments/s1.jsonl', 1], ['dir', '/generations/*/segments', 1], ['file', '/manifest.json.tmp*', 1], + ['dir', '', $lastDirSync]] as [$kind, $pattern, $nth]) { + $failed = $this->outsideDir(); + $seen = 0; + $this->syncs(function ($k, $rel) use ($kind, $pattern, $nth, $failed, &$seen) { + return $k === $kind && fnmatch($failed . $pattern, $rel) && ++$seen === $nth; + }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $failed) { $db->backup('bk', $failed); }); + $this->assertFileDoesNotExist($failed, "$kind $pattern #$nth: the backup is removed"); + } + } + + /** @test restore: PREPARE's copies synced (a failure keeps the database), INTENT synced, CURRENT the commit point */ + public function aRestoreCommitsWithCurrentAndKeepsTheDatabaseWhenASyncFails(): void + { + $config = ['shardSize' => 4]; + $db = $this->durable($config); + $db->insert('rs', array_map(function ($n) { return ['v' => $n]; }, range(0, 5))); + $dir = $this->outsideDir(); + $db->backup('rs', $dir); + $backup = $this->plain($db->find('rs', 0)); + $db->insert('rs', [['v' => 6], ['v' => 7]]); + $model = $this->plain($db->find('rs', 0)); + $x = json_decode((string)file_get_contents($this->dbPath('rs', 'current')), true)['generation'] + 1; + + $this->failNth('file', "*/generations/$x/segments/s0.jsonl"); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame(['restore', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertDirectoryDoesNotExist($this->dbPath('rs', 'home') . "generations/$x", 'a failed PREPARE removes X'); + $this->assertSame($model, $this->afterRestart('rs', $config)); + + $db = $this->durable($config); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'file' && fnmatch('*/LOCK', $rel) && strpos((string)file_get_contents($path), ' g=') !== false; }); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame(['restore', 'rolled_back'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame($model, $this->afterRestart('rs', $config)); + + $db = $this->durable($config); + $this->syncs(function ($kind, $rel, $path) use ($x) { + return $kind === 'dir' && fnmatch('*.ndb4', $rel) && is_file($path . '/CURRENT') + && (json_decode((string)file_get_contents($path . '/CURRENT'), true)['generation'] ?? 0) === $x; + }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame(['restore', 'unknown'], [$this->last($db)['op'], $this->last($db)['outcome']]); + $this->assertSame($backup, $this->afterRestart('rs', $config), 'CURRENT names X: restored'); + + // Absent: a failure in PREPARE undoes the creation at once; FORMAT is the commit point + $db = $this->durable($config); + $this->assertTrue($db->dropDB('rs')['success']); + $this->failNth('file', '*/generations/1/segments/s1.jsonl'); // s0 is copied already + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame(['LOCK'], $this->dbFiles('rs'), 'nothing of the creation is left'); + $this->assertDirectoryDoesNotExist($this->dbPath('rs', 'home') . 'generations'); + $this->assertNotContains('c', $this->lockState('rs')['flags'], 'undone at once, not by the next access'); + $this->assertSame([], $this->afterRestart('rs', $config)); + $db = $this->durable($config); + $this->failNth('file', '*/FORMAT'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame([], $this->afterRestart('rs', $config)); + $this->assertSame(['LOCK'], $this->dbFiles('rs'), 'the next access removed the half creation'); + + // Damaged: CURRENT is the commit point + $db = $this->durable($config); + $db->restore($dir, 'rs'); + file_put_contents($this->dbPath('rs', 'current'), 'x'); + \noneDB::clearStaticCache(); + $db = $this->durable($config); + $this->failNth('file', '*/CURRENT'); + $this->failing(\noneDB::DURABILITY_FAILED, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + Testing::set('syncFn', null); + \noneDB::clearStaticCache(); + $this->failing(\noneDB::FORMAT_DAMAGED, function () use ($config) { $this->db($config)->find('rs', 0); }); + $db = $this->durable($config); + $this->syncs(function ($kind, $rel, $path) { return $kind === 'dir' && fnmatch('*.ndb4', $rel) && is_file($path . '/CURRENT') && strpos($rel, $this->locatorOf('rs')) === 0; }); + $this->failing(\noneDB::OUTCOME_UNKNOWN, function () use ($db, $dir) { $db->restore($dir, 'rs'); }); + $this->assertSame($backup, $this->afterRestart('rs', $config, false), 'CURRENT names X: restored'); + } + + /** @test performance mode syncs only data-file replacements (and quarantine copies) */ + public function performanceModeSyncsOnlyReplacements(): void + { + $db = $this->db(['shardSize' => 3, 'autoCompactThreshold' => 1000]); // compaction replaces files: tested below + $db->insert('pm', [['v' => 1], ['v' => 2], ['v' => 3], ['v' => 4]]); // sharded: a generation switch (replacements) + $this->recordSyncs(); + $db->insert('pm', [['v' => 5], ['v' => 6], ['v' => 7]]); // and a new shard + $db->update('pm', [['v' => 1], ['set' => ['w' => 1]]]); + $db->delete('pm', ['v' => 2]); + $db->createFieldIndex('pm', 'v'); + $buffered = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false]); + $buffered->insert('pb', ['v' => 1]); + $buffered->flush('pb'); + $db->dropDB('pb'); + $this->assertSame([], $this->calls, 'no sync in performance mode'); + $this->assertSame('committed', $this->last($db)['outcome']); + $this->assertSame([], $this->last($db)['required']); + + $single = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); + $single->insert('pc', [['v' => 1], ['v' => 2]]); + $single->update('pc', [['v' => 1], ['set' => ['w' => 1]]]); + $this->assertSame([], $this->calls); + $single->compact('pc'); + $this->assertSame([['file', $this->locatorOf('pc') . '.ndb4/generations/1/segments/main.jsonl', true], + ['dir', $this->locatorOf('pc') . '.ndb4/generations/1/segments', true]], $this->calls, + 'the replaced data file and its directory'); + $this->assertSame([], $this->last($single)['required'], 'none required in performance mode'); + } +} diff --git a/tests/Crash/DurabilityLinuxTest.php b/tests/Crash/DurabilityLinuxTest.php new file mode 100644 index 0000000..7548a01 --- /dev/null +++ b/tests/Crash/DurabilityLinuxTest.php @@ -0,0 +1,117 @@ +markTestSkipped('durable mode is supported on Linux only (UNSUPPORTED_DURABILITY elsewhere: DurabilityConfigTest)'); + } + } + + /** The last operation committed with every required sync done ($required: the kinds §3 gives it, null: any) */ + private function committed(\noneDB $db, string $op, ?array $required): void + { + $last = $db->getDurability()['lastOperation']; + $this->assertSame([$op, 'committed'], [$last['op'], $last['outcome']], json_encode($last)); + $this->assertSame($last['required'], $last['completed'], $op); + if ($required !== null) { + $this->assertSame($required, $last['required'], $op); + } + } + + /** @test every write operation commits with its syncs done, on the real file system */ + public function everyOperationCommitsWithItsSyncs(): void + { + $db = $this->db(['durability' => 'durable', 'shardSize' => 4, 'autoCompactThreshold' => 1000]); + $this->assertSame(['os' => 'Linux', 'durableSupported' => true, 'notes' => []], $db->getDurability()['platform']); + $before = $db->getDurability()['syncs']; + $errorBefore = $db->getDurability()['lastSyncError']; // process-wide: an earlier test may have left one + + $db->createDB('lx'); + $this->committed($db, 'createDB', ['file', 'dir']); // records and new directories + $db->insert('lx', [['v' => 1], ['v' => 2], ['v' => 3]]); + $this->committed($db, 'insert', ['file', 'dir']); // the first append to the empty data file + $db->insert('lx', ['v' => 4]); + $this->committed($db, 'insert', ['file']); // an append to an existing file + $db->insert('lx', [['v' => 5], ['v' => 6]]); // past shardSize: the switch to shards follows it + $this->committed($db, 'insert', null); + $db->insert('lx', [['v' => 7], ['v' => 8]]); // sharded: j, the key reservation (a replacement) + $this->committed($db, 'insert', ['file', 'dir']); + $db->update('lx', [['v' => 1], ['set' => ['w' => 1]]]); + $this->committed($db, 'update', null); + $db->delete('lx', ['v' => 2]); + $this->committed($db, 'delete', null); + $db->compact('lx'); + $this->committed($db, 'compact', ['file', 'dir']); // replaced data files + $db->createFieldIndex('lx', 'v'); + $this->committed($db, 'createFieldIndex', ['file', 'dir']); // indexes.json + + $single = $this->db(['durability' => 'durable', 'shardingEnabled' => false, 'shardSize' => 2]); + $single->insert('ls', [['v' => 1], ['v' => 2], ['v' => 3]]); + $single->insert('ls', ['v' => 4]); + $this->committed($single, 'insert', ['file']); + $single->convertToSharded('ls'); + $this->committed($single, 'convertToSharded', ['file', 'dir']); + + $buffered = $this->db(['durability' => 'durable', 'shardingEnabled' => false, 'bufferEnabled' => true]); + $buffered->insert('lb', ['v' => 1]); + $buffered->insert('lb', ['v' => 2]); + $this->committed($buffered, 'insert', null); + $buffered->flush('lb'); + $this->committed($buffered, 'flush', null); + + $db->dropDB('lb'); + $this->committed($db, 'dropDB', ['file', 'dir']); // "d" and FORMAT's removal + + $backup = realpath(sys_get_temp_dir()) . '/nonedb_backup_' . bin2hex(random_bytes(6)); + try { + $syncs = $db->getDurability()['syncs']; + $db->backup('lx', $backup); + $this->assertGreaterThan($syncs['file'], $db->getDurability()['syncs']['file'], 'backup: every copy synced'); + $db->insert('lx', ['v' => 9]); + $db->restore($backup, 'lx'); + $this->committed($db, 'restore', ['file', 'dir']); // the copies, INTENT, CURRENT + } finally { + $this->removeDirectoryTree($backup); + } + + $after = $db->getDurability()['syncs']; + $this->assertGreaterThan($before['file'], $after['file']); + $this->assertGreaterThan($before['dir'], $after['dir']); + $this->assertSame($before['failed'], $after['failed']); + $this->assertSame($errorBefore, $db->getDurability()['lastSyncError'], 'no sync failed'); + + \noneDB::clearStaticCache(); + $check = $this->db(['shardSize' => 4]); + $this->assertSame([['v' => 1, 'w' => 1], ['v' => 3], ['v' => 4], ['v' => 5], ['v' => 6], ['v' => 7], ['v' => 8]], $this->plain($check->find('lx', 0))); + // (restored from the backup taken before v 9) + $this->assertTrue($check->getShardInfo('lx')['sharded']); + $this->assertSame([['v' => 1], ['v' => 2], ['v' => 3], ['v' => 4]], $this->plain($check->find('ls', 0))); + $this->assertTrue($check->getShardInfo('ls')['sharded']); + $this->assertSame([], $check->find('lb', 0)); + $this->assertTrue($check->checkIntegrity('lx')['ok']); + $this->assertTrue($check->checkIntegrity('ls')['ok']); + } + + /** @test key maintenance in durable mode on the real file system */ + public function maintenanceCommitsDurably(): void + { + $this->db(['durability' => 'durable'])->insert('lm', ['v' => 1]); + $new = 'rotated_linux_key_0123456789'; + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new, ['durability' => 'durable']); + $this->assertTrue($result['success'], (string)$result['error']); + \noneDB::clearStaticCache(); + $this->assertSame([['v' => 1]], $this->plain((new \noneDB(['secretKey' => $new, 'dbDir' => $this->testDbDir]))->find('lm', 0))); + } +} diff --git a/tests/Crash/FaultMatrixTest.php b/tests/Crash/FaultMatrixTest.php new file mode 100644 index 0000000..463fd81 --- /dev/null +++ b/tests/Crash/FaultMatrixTest.php @@ -0,0 +1,486 @@ +markTestSkipped('needs POSIX signals and proc_open'); + } + } + + // ------------------------------------------------------------------ the model + + private static function seed(int $n): array + { + $records = []; + for ($i = 0; $i < $n; $i++) { + $records[] = ['n' => $i, 'g' => $i % 3]; + } + return $records; + } + + /** + * Operations: config, the calls that build the state before ([op, args]...), the faulted call, and its + * kill points ([point, nth occurrence]) + */ + private function operations(): array + { + $batch = [['n' => 100, 'g' => 9], ['n' => 101, 'g' => 9], ['n' => 102, 'g' => 9], ['n' => 103, 'g' => 9], ['n' => 104, 'g' => 9]]; + return [ + 'insert' => [ + 'config' => ['shardingEnabled' => false], + 'before' => [['insert', self::seed(40)]], + 'call' => ['insert', [['n' => 100, 'g' => 9]]], + 'kill' => [['append:after', 1], ['index:written', 1], ['scope:commit', 1], ['checkpoint:written', 1], ['atomic:renaming', 1]], + ], + 'insert-new-shards' => [ + 'config' => ['shardSize' => 4], + 'before' => [['insert', self::seed(6)]], + 'call' => ['insert', $batch], // keys 6..10: the end of shard 1, shard 2 (new) + 'kill' => [['shard:created', 1], ['append:after', 1], ['append:after', 2], ['index:written', 2], ['atomic:renaming', 1], ['scope:commit', 1], ['checkpoint:written', 1]], + ], + 'update-across-shards' => [ + 'config' => ['shardSize' => 4, 'autoCompactThreshold' => 1000], + 'before' => [['insert', self::seed(12)]], + 'call' => ['update', ['g' => 1], ['u' => 1]], // keys 1, 4, 7, 10 in shards 0, 1, 1, 2 + 'kill' => [['append:after', 1], ['append:after', 2], ['append:after', 3], ['scope:commit', 1], ['checkpoint:written', 1]], + ], + 'delete-across-shards' => [ + 'config' => ['shardSize' => 4, 'autoCompactThreshold' => 1000], + 'before' => [['insert', self::seed(12)]], + 'call' => ['delete', ['g' => 2]], // keys 2, 5, 8, 11 + 'kill' => [['append:after', 1], ['append:after', 2], ['scope:commit', 1]], + ], + 'buffered-insert-and-flush' => [ + 'config' => ['shardingEnabled' => false, 'bufferEnabled' => true], + 'before' => [['insert', self::seed(5)], ['flush']], + 'call' => ['insert-flush', [['n' => 200, 'g' => 9], ['n' => 201, 'g' => 9]]], + 'kill' => [['append:after', 1], ['append:after', 2], ['scope:commit', 1], ['scope:commit', 2], ['checkpoint:written', 1]], + ], + 'compact' => [ + 'config' => ['shardingEnabled' => false, 'autoCompactThreshold' => 1000], + 'before' => [['insert', self::seed(20)], ['update', ['g' => 0], ['w' => 1]], ['delete', ['g' => 1]]], + 'call' => ['compact'], + 'kill' => [['compact:replace', 1], ['atomic:renaming', 1], ['compact:replaced', 1], ['checkpoint:written', 1]], + ], + 'compact-sharded' => [ + 'config' => ['shardSize' => 5, 'autoCompactThreshold' => 1000], + 'before' => [['insert', self::seed(15)], ['delete', ['g' => 1]]], + 'call' => ['compact'], + 'kill' => [['compact:replace', 1], ['compact:replaced', 1], ['compact:replace', 2], ['compact:replaced', 3]], + ], + 'convertToSharded' => [ + 'config' => ['shardingEnabled' => false, 'shardSize' => 4], + 'before' => [['insert', self::seed(10)], ['delete', ['n' => 9]]], + 'call' => ['convertToSharded'], + 'kill' => [['switch:shard-written', 1], ['switch:shard-written', 3], ['atomic:renaming', 2], ['generation:prepared', 1], + ['generation:intent', 1], ['generation:published', 1], ['generation:cleaned', 1], ['checkpoint:written', 1]], + ], + 'create-field-index' => [ + 'config' => ['shardSize' => 5], + 'before' => [['insert', self::seed(12)]], + 'call' => ['createFieldIndex', 'g'], + 'kill' => [['atomic:renaming', 1], ['atomic:renaming', 3]], + ], + ]; + } + + /** Apply a call to the model: [key => record], keys given in order from $next */ + private static function apply(array $model, int &$next, array $call): array + { + switch ($call[0]) { + case 'insert': + case 'insert-flush': + foreach ($call[1] as $record) { + $model[$next++] = $record; + } + return $model; + case 'update': + foreach ($model as $key => $record) { + if (array_intersect_assoc($call[1], $record) == $call[1]) { + $model[$key] = array_merge($record, $call[2]); + } + } + return $model; + case 'delete': + foreach ($model as $key => $record) { + if (array_intersect_assoc($call[1], $record) == $call[1]) { + unset($model[$key]); + } + } + return $model; + } + return $model; // flush, compact, convertToSharded, createFieldIndex: the records stay as they are + } + + /** The states an interrupted call may leave: before plus each prefix of its changes in key order */ + private static function candidates(array $before, int $next, array $call): array + { + $after = self::apply($before, $next, $call); + $changed = []; + foreach (array_unique(array_merge(array_keys($before), array_keys($after))) as $key) { + if (($before[$key] ?? null) !== ($after[$key] ?? null)) { + $changed[] = $key; + } + } + sort($changed); + $states = []; + for ($k = 0; $k <= count($changed); $k++) { + $state = $before; + foreach (array_slice($changed, 0, $k) as $key) { + if (array_key_exists($key, $after)) { + $state[$key] = $after[$key]; + } else { + unset($state[$key]); + } + } + ksort($state); + $states[] = array_values($state); + } + return $states; + } + + private static function code(array $call): string + { + $db = var_export(self::DB, true); + switch ($call[0]) { + case 'insert': + return '$db->insert(' . $db . ', ' . var_export($call[1], true) . ');'; + case 'insert-flush': + return '$db->insert(' . $db . ', ' . var_export($call[1], true) . '); $db->flush(' . $db . ');'; + case 'update': + return '$db->update(' . $db . ', [' . var_export($call[1], true) . ', ["set" => ' . var_export($call[2], true) . ']]);'; + case 'delete': + return '$db->delete(' . $db . ', ' . var_export($call[1], true) . ');'; + case 'flush': + return '$db->flush(' . $db . ');'; + case 'compact': + return '$db->compact(' . $db . ');'; + case 'convertToSharded': + return '$db->convertToSharded(' . $db . ');'; + case 'createFieldIndex': + return '$db->createFieldIndex(' . $db . ', ' . var_export($call[1], true) . ');'; + } + throw new \LogicException($call[0]); + } + + /** Run a call through the API */ + private static function perform(\noneDB $db, array $call): void + { + switch ($call[0]) { + case 'insert': + $db->insert(self::DB, $call[1]); + return; + case 'update': + $db->update(self::DB, [$call[1], ['set' => $call[2]]]); + return; + case 'delete': + $db->delete(self::DB, $call[1]); + return; + case 'flush': + $db->flush(self::DB); + return; + } + throw new \LogicException($call[0]); + } + + /** Build the state before the call (through the API) and return [model, next key] */ + private function prepare(array $op): array + { + $this->cleanTestDirectory(); + $db = $this->db($op['config']); + $model = []; + $next = 0; + foreach ($op['before'] as $call) { + self::perform($db, $call); + $model = self::apply($model, $next, $call); + } + unset($db); + \noneDB::clearStaticCache(); + return [$model, $next]; + } + + /** + * Run $body in a child process after $hooks, with the operation's config plus $config. Started without a + * shell (CLAUDE.md): a child that kills itself is reported by nobody, its output is what it printed. + */ + private function child(array $op, string $hooks, string $body, array $config = []): string + { + return $this->runDirect(' + ' . $hooks . ' + try { + $db = ' . $this->childDb($op['config'] + $config) . '; + ' . $body . ' + echo "COMPLETED"; + } catch (\Throwable $e) { + echo "CODE:" . $e->getCode() . ":" . get_class($e) . ":" . $e->getMessage(); + } + '); + } + + private function runDirect(string $body): string + { + $script = $this->childScript($body); + $proc = proc_open([PHP_BINARY, $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $out = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($proc); + return trim($out); + } + + /** + * The state after the fault, opened again: one of $allowed; then a write works, takes a new key and + * leaves the database clean + */ + private function verify(string $label, array $op, array $allowed, int $next, string $output, bool $leftover = false): void + { + \noneDB::clearStaticCache(); + $db = $this->db($op['config']); + $records = $db->find(self::DB, 0); + $actual = $this->plain($records); + $this->assertContains($actual, $allowed, "$label: the records are a state the operation may leave\n$output"); + $keys = array_column($records, 'key'); + $db->insert(self::DB, ['probe' => $label]); + $probe = $db->find(self::DB, ['probe' => $label]); + $this->assertCount(1, $probe, $label); + $this->assertGreaterThanOrEqual($next, $probe[0]['key'], "$label: no key handed out before is used again"); + $this->assertNotContains($probe[0]['key'], $keys, $label); + $this->assertSame(count($actual) + 1, $db->count(self::DB), $label); + $report = $db->checkIntegrity(self::DB); + if (!$report['ok'] && $leftover) { + // STORAGE_FORMAT §5.2: a generation prepared before INTENT is reported, never removed without + // proof (the next switch removes it first); nothing else may be wrong + $this->assertSame([['segment' => null, 'index' => 'generation', 'detail' => 'generation 2 is not the current one (1)']], + $report['indexMismatches'], "$label: " . json_encode($report)); + $report['indexMismatches'] = []; + $this->assertSame([], array_filter([$report['badLines'], $report['tornTail'], $report['pendingRecovery']]), "$label: " . json_encode($report)); + return; + } + $this->assertTrue($report['ok'], "$label: " . json_encode($report)); + } + + // ------------------------------------------------------------------ SIGKILL at every step + + /** @test SIGKILL at every step of every write operation */ + public function killedAtEveryStep(): void + { + $points = []; + foreach ($this->operations() as $name => $op) { + foreach ($op['kill'] as [$point, $nth]) { + [$before, $next] = $this->prepare($op); + $allowed = self::candidates($before, $next, $op['call']); + $hook = 'noneDB\Tests\Support\Testing::setFaultHook(function ($p) { + static $seen = 0; + if ($p === ' . var_export($point, true) . ' && ++$seen === ' . $nth . ') { posix_kill(getmypid(), 9); } + });'; + $output = $this->child($op, $hook, self::code($op['call'])); + $label = "$name killed at $point #$nth"; + if (strpos($output, 'COMPLETED') !== false) { + // The step doesn't occur in this operation: it ran to the end + $allowed = [end($allowed)]; + } else { + $this->assertSame('', $output, "$label: killed, nothing printed"); + $points[$point] = true; + } + // Killed while generation 2 was prepared (before INTENT): it is left for the next switch + $prepare = $op['call'][0] === 'convertToSharded' && in_array($point, ['switch:shard-written', 'atomic:renaming', 'generation:prepared'], true); + $this->verify($label, $op, $allowed, $next, $output, $prepare); + } + } + // Every step of the matrix was reached by some operation + foreach (['append:after', 'index:written', 'scope:commit', 'checkpoint:written', 'atomic:renaming', 'shard:created', + 'compact:replace', 'compact:replaced', 'switch:shard-written', 'generation:prepared', 'generation:intent', + 'generation:published', 'generation:cleaned'] as $point) { + $this->assertArrayHasKey($point, $points, "no operation was killed at $point"); + } + } + + /** The temp files of atomic replacements in the generation's segments directory */ + private function segmentTemps(): array + { + return array_values(preg_grep('/\.tmp[0-9a-f]{8}$/', scandir($this->dbPath(self::DB, 'gen') . 'segments') ?: [])); + } + + /** + * @test STORAGE_FORMAT.md §2, §5.3: a compaction killed between writing the new data file's temp and renaming it + * leaves that temp (never read: the records are the old ones); the next compaction of the segment removes it + */ + public function aCompactionKilledBeforeItsRenameLeavesATempTheNextCompactionRemoves(): void + { + foreach (['compact', 'compact-sharded'] as $name) { + $op = $this->operations()[$name]; + [$before, $next] = $this->prepare($op); + $hook = 'noneDB\Tests\Support\Testing::setFaultHook(function ($p, $c) { + if ($p === "atomic:renaming" && substr((string)$c["path"], -6) === ".jsonl") { posix_kill(getmypid(), 9); } + });'; + $this->assertSame('', $this->child($op, $hook, self::code($op['call'])), "$name: killed"); + $temps = $this->segmentTemps(); + $this->assertCount(1, $temps, "$name: the temp of the data file being replaced"); + $this->assertMatchesRegularExpression('/^(main|s[0-9]+)\.jsonl\.tmp[0-9a-f]{8}$/', $temps[0], $name); + \noneDB::clearStaticCache(); + $db = $this->db($op['config']); + $this->assertSame(array_values($before), $this->plain($db->find(self::DB, 0)), "$name: the old records, the temp never read"); + $db->compact(self::DB); + $this->assertSame([], $this->segmentTemps(), "$name: removed by the next compaction"); + $this->assertSame(array_values($before), $this->plain($db->find(self::DB, 0)), "$name: compacted"); + $this->assertTrue($db->checkIntegrity(self::DB)['ok'], $name); + } + } + + // ------------------------------------------------------------------ I/O errors + + /** @return array fault => [hooks (child code), config added, code expected or null for any non-unknown failure, appends only] */ + private function faults(): array + { + $dataOnly = 'substr((string)$path, -6) === ".jsonl"'; + return [ + 'short write' => ['noneDB\Tests\Support\Testing::set("writeFn", function ($fp, $bytes, $path) { + static $calls = 0; + if (!(' . $dataOnly . ')) { return fwrite($fp, $bytes); } + return ++$calls === 1 ? fwrite($fp, substr($bytes, 0, max(1, intdiv(strlen($bytes), 2)))) : false; + });', [], null], + 'disk full' => ['noneDB\Tests\Support\Testing::set("writeFn", function ($fp, $bytes, $path) { + return ' . $dataOnly . ' ? false : fwrite($fp, $bytes); + });', [], null], + 'fflush fails' => ['noneDB\Tests\Support\Testing::set("flushFn", function ($fp, $path) { return false; });', [], null], + 'fsync fails' => ['noneDB\Tests\Support\Testing::set("osFamily", "Linux"); + noneDB\Tests\Support\Testing::set("syncFn", function ($fp, $kind, $path) { + static $failed = false; + if ($kind === "file" && ' . $dataOnly . ' && !$failed) { $failed = true; return false; } + return true; + });', ['durability' => 'durable'], \noneDB::DURABILITY_FAILED], + ]; + } + + /** @test an I/O error rolls the operation back: exactly the state before, the original error */ + public function anIoErrorRollsTheOperationBack(): void + { + foreach ($this->operations() as $name => $op) { + foreach ($this->faults() as $fault => [$hooks, $config, $code]) { + [$before, $next] = $this->prepare($op); + $output = $this->child($op, $hooks, self::code($op['call']), $config); + $label = "$name / $fault"; + $allowed = self::candidates($before, $next, $op['call']); + if (strpos($output, 'COMPLETED') !== false) { + // Only an operation that writes no data file goes through without the fault + $this->assertSame('createFieldIndex', $op['call'][0], "$label: the fault must hit\n$output"); + $this->verify($label, $op, [end($allowed)], $next, $output); + continue; + } + $this->assertMatchesRegularExpression('/^CODE:(\d+):RuntimeException:/', $output, $label); + $thrown = (int)explode(':', $output)[1]; + $this->assertNotSame(\noneDB::OUTCOME_UNKNOWN, $thrown, "$label: the undo worked, the outcome is known\n$output"); + if ($code !== null) { + $this->assertSame($code, $thrown, $label); + } + $this->verify($label, $op, [$allowed[0]], $next, $output); + } + } + } + + /** @test a second I/O error while the rollback cuts back: OUTCOME_UNKNOWN, and every reader sees one consistent state */ + public function aSecondErrorDuringTheRollbackIsUnknown(): void + { + foreach ($this->operations() as $name => $op) { + if (!in_array($op['call'][0], ['insert', 'update', 'delete', 'insert-flush'], true)) { + continue; // no appends to cut back + } + [$before, $next] = $this->prepare($op); + $hooks = 'noneDB\Tests\Support\Testing::set("flushFn", function ($fp, $path) { return substr((string)$path, -6) !== ".jsonl"; }); + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { + if ($p === "append:undo" || $p === "rollback:truncate") { throw new \RuntimeException("I/O error"); } + });'; + $output = $this->child($op, $hooks, self::code($op['call'])); + $label = "$name / error during the rollback"; + $this->assertStringStartsWith('CODE:' . \noneDB::OUTCOME_UNKNOWN . ':', $output, $label); + $this->assertContains('J', $this->lockState(self::DB)['flags'], "$label: the next session recovers"); + $this->verify($label, $op, self::candidates($before, $next, $op['call']), $next, $output); + } + } + + /** @test a data file or directory the process may not write: the operation fails and changes nothing */ + public function aPermissionErrorChangesNothing(): void + { + if (function_exists('posix_geteuid') && posix_geteuid() === 0) { + $this->markTestSkipped('root ignores file permissions'); + } + foreach ($this->operations() as $name => $op) { + [$before, $next] = $this->prepare($op); + $gen = $this->dbPath(self::DB, 'gen'); + $targets = array_merge(glob($gen . 'segments/*.jsonl') ?: [], glob($gen . 'buffer/*.jsonl') ?: []); + $dirs = array_values(array_filter([$gen . 'segments', $gen . 'buffer', $gen], 'is_dir')); + foreach ($targets as $file) { + chmod($file, 0444); + } + foreach ($dirs as $dir) { + chmod($dir, 0555); + } + try { + $output = $this->child($op, '', self::code($op['call'])); + } finally { + foreach ($dirs as $dir) { + chmod($dir, 0750); + } + foreach ($targets as $file) { + chmod($file, 0640); + } + } + $label = "$name / permission denied"; + $allowed = self::candidates($before, $next, $op['call']); + if (strpos($output, 'COMPLETED') !== false) { + // Work outside the protected directories (a new generation, index files) may succeed: + // only operations that change no record + $this->assertCount(1, $allowed, "$label: the records can't have changed\n$output"); + $this->verify($label, $op, $allowed, $next, $output); + continue; + } + $this->assertMatchesRegularExpression('/^CODE:\d+:RuntimeException:/', $output, $label); + $this->assertNotSame((string)\noneDB::OUTCOME_UNKNOWN, explode(':', $output)[1], "$label\n$output"); + $this->verify($label, $op, [$allowed[0]], $next, $output); + } + } + + /** @test a lock held by another process beyond lockTimeout: LOCK_FAILED, nothing changed */ + public function aLockTimeoutChangesNothing(): void + { + foreach ($this->operations() as $name => $op) { + [$before, $next] = $this->prepare($op); + $holder = proc_open([PHP_BINARY, '-r', '$fp = fopen($argv[1], "c+b"); flock($fp, LOCK_EX); echo "held\n"; fflush(STDOUT); sleep(3);', + $this->dbPath(self::DB, 'lock')], [1 => ['pipe', 'w']], $pipes); + $this->assertSame("held\n", fgets($pipes[1])); + $op['config']['lockTimeout'] = 0.2; + $output = $this->child($op, '', self::code($op['call'])); + proc_terminate($holder, 9); + fclose($pipes[1]); + proc_close($holder); + $label = "$name / lock timeout"; + $this->assertStringStartsWith('CODE:' . \noneDB::LOCK_FAILED . ':', $output, $label); + $this->verify($label, $op, [self::candidates($before, $next, $op['call'])[0]], $next, $output); + } + } +} diff --git a/tests/Crash/GenerationSwitchTest.php b/tests/Crash/GenerationSwitchTest.php new file mode 100644 index 0000000..5749ea0 --- /dev/null +++ b/tests/Crash/GenerationSwitchTest.php @@ -0,0 +1,237 @@ + sharded layout + * change interrupted at every step, recovery, invalid CURRENT, another process's stale caches. + */ +class GenerationSwitchTest extends RegressionTestCase +{ + private function snapshot(string $dir): array + { + $out = []; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $full = $path . '/' . $entry; + if (is_dir($full)) { + $out[$rel . $entry] = 'dir'; + $walk($full, $rel . $entry . '/'); + } else { + $out[$rel . $entry] = hash_file('sha256', $full); + } + } + }; + if (is_dir($dir)) { + $walk(rtrim($dir, '/'), ''); + } + ksort($out); + return $out; + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + private function generations(string $db): array + { + $out = array_values(array_filter(scandir($this->dbPath($db, 'home') . 'generations') ?: [], 'ctype_digit')); + sort($out); + return array_map('intval', $out); + } + + private function sealed(array $record): string + { + $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION; + $json = json_encode($record, $flags); + $record['checksum'] = hash('sha256', $json . "\n"); + return json_encode($record, $flags) . "\n"; + } + + /** Child: 3 records in the single layout, then 2 more reach shardSize 4: the switch runs after the commit */ + private function interruptSwitchAt(string $point): string + { + $db = $this->db(['shardSize' => 4]); + $db->createFieldIndex('m', 'c'); + $db->insert('m', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2)]; }, range(0, 2))); + \noneDB::clearStaticCache(); + return $this->runChild(' + $db = ' . $this->childDb(['shardSize' => 4]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === ' . var_export($point, true) . ') { exit(0); } }); + $db->insert("m", [["n" => 3, "c" => "c1"], ["n" => 4, "c" => "c0"]]); + echo "not reached"; + '); + } + + /** @test */ + public function theSwitchInterruptedAtEveryStepOpensOneConsistentGeneration(): void + { + $expected = ['generation:prepared' => [1, false], 'generation:intent' => [1, false], 'generation:published' => [2, true], 'generation:cleaned' => [2, true]]; + foreach ($expected as $point => [$generation, $sharded]) { + $this->cleanTestDirectory(); + $this->assertSame('', $this->interruptSwitchAt($point), $point); + $state = $this->lockState('m'); + if ($point === 'generation:prepared') { + $this->assertNotContains('g=1>2', $state['flags'], "$point: INTENT not written yet"); + } else { + $this->assertContains('g=1>2', $state['flags'], "$point: INTENT written before PUBLISH (§5.2)"); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + // The insert committed before the switch started: every record, once, in either layout + $this->assertSame(range(0, 4), array_column($db->find('m', 0), 'n'), $point); + $this->assertSame([1, 3], array_column($db->find('m', ['c' => 'c1']), 'n'), "$point: field index"); + $info = $db->getShardInfo('m'); + $this->assertSame($sharded, (bool)($info['sharded'] ?? false), "$point: layout of the committed generation"); + $this->assertNotContains('g=1>2', $this->lockState('m')['flags'], "$point: recovered, g cleared"); + if ($point !== 'generation:prepared') { + $this->assertSame([$generation], $this->generations('m'), "$point: the other generation is removed"); + } + // Still writable; a further switch (if still single) cleans an unreferenced leftover first + $db->insert('m', ['n' => 5, 'c' => 'c1']); + $this->assertSame(range(0, 5), array_column($db->find('m', 0), 'n'), $point); + $this->assertSame([1, 3, 5], array_column($db->find('m', ['c' => 'c1']), 'n'), $point); + $this->assertCount(1, $this->generations('m'), "$point: one generation in the end"); + $this->assertTrue($db->checkIntegrity('m')['ok'], $point); + } + } + + /** + * @test STORAGE_FORMAT §5.2 recovery: an obsolete generation that can't be removed is logged as + * generation-cleanup-failed; g is cleared (CURRENT decides) and checkIntegrity() reports what is left + */ + public function anObsoleteGenerationThatCannotBeRemovedIsLoggedAndReported(): void + { + if (function_exists('posix_geteuid') && posix_geteuid() === 0) { + $this->markTestSkipped('root ignores file permissions'); + } + $this->assertSame('', $this->interruptSwitchAt('generation:published')); + $this->assertContains('g=1>2', $this->lockState('m')['flags']); + $parent = $this->dbPath('m', 'home') . 'generations'; + chmod($parent, 0555); // generation 1's entry can't be removed + try { + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + $this->assertSame(range(0, 4), array_column($db->find('m', 0), 'n'), 'generation 2 is committed'); + } finally { + chmod($parent, 0750); + } + $this->assertNotContains('g=1>2', $this->lockState('m')['flags'], 'cleared: not retried by every operation'); + $this->assertSame([1, 2], $this->generations('m')); + $actions = array_column($db->getRecoveryLog('m'), 'action'); + $this->assertContains('generation-cleanup-failed', $actions); + $this->assertNotContains('generation-recovered', $actions); + $report = $db->checkIntegrity('m'); + $this->assertFalse($report['ok']); + $this->assertContains('generation', array_column($report['indexMismatches'], 'index')); + } + + /** @test */ + public function anUnreferencedPreparedGenerationIsReportedNotRemoved(): void + { + $this->assertSame('', $this->interruptSwitchAt('generation:prepared')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 400]); // no switch on the next writes + $this->assertSame([1, 2], $this->generations('m'), 'never removed without proof (§4.2/§5.2)'); + $report = $db->checkIntegrity('m'); + $this->assertFalse($report['ok']); + $this->assertContains('generation', array_column($report['indexMismatches'], 'index')); + $this->assertSame(range(0, 4), array_column($db->find('m', 0), 'n')); + } + + /** @test */ + public function anInvalidCurrentDuringASwitchRemovesNothing(): void + { + $cases = [ + 'empty' => function (array $current) { return ''; }, + 'garbage' => function (array $current) { return "\0\0garbage"; }, + 'checksum' => function (array $current) { return str_replace('"generation":2', '"generation":1', $this->sealed($current)) ; }, + 'other database' => function (array $current) { $current['databaseId'] = '00000000-0000-4000-8000-000000000000'; return $this->sealed($current); }, + 'other generation' => function (array $current) { $current['generation'] = 9; return $this->sealed($current); }, + ]; + foreach ($cases as $case => $make) { + $this->cleanTestDirectory(); + $this->assertSame('', $this->interruptSwitchAt('generation:published'), $case); + $raw = file_get_contents($this->dbPath('m', 'current')); + $current = json_decode($raw, true); + unset($current['checksum']); + file_put_contents($this->dbPath('m', 'current'), $make($current)); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $db = $this->db(['shardSize' => 4]); + foreach (['find' => function () use ($db) { $db->find('m', 0); }, + 'insert' => function () use ($db) { $db->insert('m', ['n' => 9]); }, + 'count' => function () use ($db) { $db->count('m'); }] as $op => $fn) { + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code($fn), "$case: $op (§5.2 recovery step 4)"); + } + $this->assertSame($before, $this->snapshot($this->testDbDir), "$case: both generations kept, nothing changed"); + $this->assertSame([1, 2], $this->generations('m'), $case); + } + } + + /** @test */ + public function invalidMetadataOfThePublishedGenerationIsDamage(): void + { + $this->assertSame('', $this->interruptSwitchAt('generation:published')); + file_put_contents($this->dbPath('m', 'gen') . 'metadata', "{}\n"); // CURRENT names 2, its metadata is invalid + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db(['shardSize' => 4])->find('m', 0); })); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + + /** @test */ + public function anInvalidCurrentWithoutSwitchIsDamage(): void + { + $this->db()->insert('c', [['a' => 1]]); + file_put_contents($this->dbPath('c', 'current'), 'x'); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db()->find('c', 0); })); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db()->insert('c', ['a' => 2]); })); + $this->assertSame($before, $this->snapshot($this->testDbDir), 'fail closed, nothing removed'); + } + + /** @test */ + public function aProcessWithCachesOfTheOldGenerationReadsTheNewOne(): void + { + $db = $this->db(['shardSize' => 4]); + $db->insert('st', array_map(function ($n) { return ['n' => $n]; }, range(0, 2))); + $go = $this->testDbDir . '../gen-go'; + $done = $this->testDbDir . '../gen-done'; + @unlink($go); + @unlink($done); + $proc = proc_open([PHP_BINARY, $this->childScript(' + $db = ' . $this->childDb(['shardSize' => 4]) . '; + $first = array_column($db->find("st", 0), "n"); // caches of generation 1 + file_put_contents(' . var_export($done, true) . ', "1"); + $deadline = microtime(true) + 20; + while (!is_file(' . var_export($go, true) . ') && microtime(true) < $deadline) { usleep(10000); } + $second = array_column($db->find("st", 0), "n"); // after the switch to generation 2 + echo json_encode([$first, $second, $db->getShardInfo("st")["sharded"] ?? false]); + ')], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $deadline = microtime(true) + 20; + while (!is_file($done) && microtime(true) < $deadline) { + usleep(10000); + } + $db->insert('st', [['n' => 3], ['n' => 4]]); // switch to the sharded layout + $this->assertSame([2], $this->generations('st')); + file_put_contents($go, '1'); + $out = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + proc_close($proc); + @unlink($go); + @unlink($done); + $this->assertSame([[0, 1, 2], [0, 1, 2, 3, 4], true], json_decode($out, true), $out); + } +} diff --git a/tests/Crash/KeyMaintenanceTest.php b/tests/Crash/KeyMaintenanceTest.php new file mode 100644 index 0000000..854ed48 --- /dev/null +++ b/tests/Crash/KeyMaintenanceTest.php @@ -0,0 +1,641 @@ +testDbDir . $file); + if ($raw === false || !preg_match('/,"checksum":"([0-9a-f]{64})"\}\n\z/', $raw, $m, PREG_OFFSET_CAPTURE) + || hash('sha256', substr($raw, 0, $m[0][1]) . "}\n") !== $m[1][0]) { + return null; + } + return json_decode($raw, true); + } + + private function keyCheck(string $key): string + { + return substr(hash_hmac('sha256', 'nonedb4/key', $key), 0, 32); + } + + private function locator(string $name, string $key): string + { + return substr(hash_hmac('sha256', 'nonedb4/db/' . $name, $key), 0, 32); + } + + private function with(string $key): \noneDB + { + return new \noneDB(['secretKey' => $key, 'dbDir' => $this->testDbDir, 'shardSize' => 3]); + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + private function seed(): void + { + $db = $this->with(self::SECRET); + $db->createFieldIndex('users', 'city'); + $db->insert('users', [['n' => 1, 'city' => 'A'], ['n' => 2, 'city' => 'B'], ['n' => 3, 'city' => 'A'], ['n' => 4, 'city' => 'C']]); + $db->insert('orders', [['o' => 'x'], ['o' => 'y']]); + \noneDB::clearStaticCache(); + } + + /** Expected end state of a rotation from SECRET to NEW */ + private function assertRotated(string $context): void + { + \noneDB::clearStaticCache(); + $store = $this->record('STORE'); + $this->assertNotNull($store, $context); + $this->assertSame($store, $this->record('STORE.2'), "$context: copies identical"); + $this->assertSame([$this->keyCheck(self::NEW)], $store['keys'], "$context: old key out, new key in"); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json', "$context: journal closed"); + foreach (['users', 'orders'] as $name) { + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->locator($name, self::SECRET) . '.ndb4', "$context: $name old dir"); + $this->assertDirectoryExists($this->testDbDir . $this->locator($name, self::NEW) . '.ndb4', "$context: $name new dir"); + } + $new = $this->with(self::NEW); + $this->assertSame([1, 2, 3, 4], array_column($new->find('users', 0), 'n'), $context); + $this->assertSame([1, 3], array_column($new->find('users', ['city' => 'A']), 'n'), "$context: indexes moved with the data"); + $this->assertSame(['x', 'y'], array_column($new->find('orders', 0), 'o'), $context); + $this->assertSame(['orders', 'users'], $new->getDBs(), $context); + foreach (['users', 'orders'] as $name) { + $this->assertNotContains('m', $this->lockStateOfDir($this->locator($name, self::NEW))['flags'], "$context: $name m cleared"); + } + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $this->code(function () { $this->with(self::SECRET)->find('users', 0); }), "$context: old key refused"); + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->locator('users', self::SECRET) . '.ndb4', "$context: the old key created nothing"); + } + + private function lockStateOfDir(string $locator): array + { + $tokens = explode(' ', trim((string)file_get_contents($this->testDbDir . $locator . '.ndb4/LOCK'), " \0")); + $state = (string)array_shift($tokens); + return ['state' => $state, 'flags' => array_values(array_filter($tokens, 'strlen'))]; + } + + /** @test */ + public function rotationMovesEveryDatabaseToTheNewKey(): void + { + $this->seed(); + $insideBefore = $this->snapshot($this->testDbDir . $this->locator('users', self::SECRET) . '.ndb4'); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($result['success'], (string)$result['error']); + $this->assertSame(['orders', 'users'], (function ($d) { sort($d); return $d; })($result['databases'])); + $this->assertSame($this->record('STORE'), $result['storeManifest'], 'storeManifest = the STORE record, checksum included (§9.4)'); + // Right after the rotation, before any read (readers may refresh derived files for the new epoch) + $insideAfter = $this->snapshot($this->testDbDir . $this->locator('users', self::NEW) . '.ndb4'); + unset($insideBefore['LOCK'], $insideAfter['LOCK']); + $this->assertSame($insideBefore, $insideAfter, 'contents unchanged: FORMAT and metadata hold no key (§4.1)'); + $this->assertRotated('rotation'); + $this->assertSame(2, $this->record('STORE')['seq']); + } + + /** @test */ + public function rotationInterruptedAtEveryStepIsFinishedByARerun(): void + { + $points = [['maintenance:journal', 1], ['rotate:marked', 1], ['rotate:renamed', 1], ['rotate:renamed', 2], + ['maintenance:journal', 2], ['rotate:committed', 1], ['store:write', 1], ['store:write', 2], + ['maintenance:store-written', 1], ['maintenance:close', 1]]; + foreach ($points as [$point, $hit]) { + $context = "$point #$hit"; + $this->cleanTestDirectory(); + $this->seed(); + $out = $this->runChild(' + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) use (&$n) { if ($p === ' . var_export($point, true) . ' && ++$n === ' . $hit . ') { exit(0); } }); + $r = noneDB::migrateKey(' . var_export($this->testDbDir, true) . ', ' . var_export(self::SECRET, true) . ', ' . var_export(self::NEW, true) . '); + echo "not reached ", json_encode($r); + '); + $this->assertSame('', $out, $context); + \noneDB::clearStaticCache(); + $journalOpen = is_file($this->testDbDir . '.nonedb-maintenance/journal.json'); + if ($journalOpen) { + // §9.1: while the journal exists no store is opened, whatever the key + foreach ([self::SECRET, self::NEW] as $key) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $this->code(function () use ($key) { $this->with($key)->find('users', 0); }), "$context: open refused"); + \noneDB::clearStaticCache(); + } + } + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($result['success'], "$context: rerun " . $result['error']); + $this->assertRotated($context); + } + } + + /** @test */ + public function anUncommittedRotationCanBeAborted(): void + { + $this->seed(); + $before = $this->snapshot($this->testDbDir); + $out = $this->runChild(' + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === "rotate:renamed") { exit(0); } }); + noneDB::migrateKey(' . var_export($this->testDbDir, true) . ', ' . var_export(self::SECRET, true) . ', ' . var_export(self::NEW, true) . '); + echo "not reached"; + '); + $this->assertSame('', $out); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW, ['abort' => true]); + $this->assertTrue($result['success'], (string)$result['error']); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n'), 'old key works again'); + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $this->code(function () { $this->with(self::NEW)->find('users', 0); })); + $after = $this->snapshot($this->testDbDir); + // The same files with the same content, except the state records (new epochs) and the store log dir + $strip = function (array $snap) { + return array_filter($snap, function ($k) { return substr($k, -4) !== 'LOCK' && strpos($k, '.nonedb-maintenance') !== 0; }, ARRAY_FILTER_USE_KEY); + }; + $this->assertSame($strip($before), $strip($after)); + } + + private function setRenameFn(?callable $fn): void + { + $property = new \noneDB\Tests\InternalValue('renameFn'); + $property->setValue(null, $fn); + } + + /** A rotation whose first renamed database fails verification (its CURRENT damaged right after the rename) */ + private function rotateWithAFailingVerification(): array + { + $damaged = null; + $this->setFaultHook(function ($point, $context) use (&$damaged) { + if ($point === 'rotate:renamed' && $damaged === null) { + $damaged = $context['db']; + file_put_contents($this->testDbDir . $this->locator($damaged, self::NEW) . '.ndb4/CURRENT', 'damaged'); + } + }); + try { + return [\noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW), $damaged]; + } finally { + $this->setFaultHook(null); + } + } + + /** @test */ + public function aRotationThatFailsVerificationIsUndone(): void + { + $this->seed(); + $store = $this->record('STORE'); + [$result, $damaged] = $this->rotateWithAFailingVerification(); + $this->assertNotNull($damaged); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $result['code'], (string)$result['error']); + $this->assertStringContainsString("'$damaged'", (string)$result['error']); + $this->assertSame($store, $this->record('STORE'), 'nothing committed'); + $this->assertSame($store, $this->record('STORE.2')); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json', 'undone and closed'); + foreach (['users', 'orders'] as $name) { + $this->assertDirectoryExists($this->testDbDir . $this->locator($name, self::SECRET) . '.ndb4', "$name moved back"); + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->locator($name, self::NEW) . '.ndb4', $name); + $this->assertNotContains('m', $this->lockStateOfDir($this->locator($name, self::SECRET))['flags'], "$name: m cleared"); + } + \noneDB::clearStaticCache(); + $db = $this->with(self::SECRET); + $other = $damaged === 'users' ? 'orders' : 'users'; + $this->assertSame($other === 'users' ? 4 : 2, $db->count($other), 'the other database works under the old key'); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () use ($db, $damaged) { $db->find($damaged, 0); }), 'the damage stays visible, never an empty database'); + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $this->code(function () { $this->with(self::NEW)->find('users', 0); })); + } + + /** @test */ + public function aFailedUndoKeepsTheStoreClosedUntilAnAbortFinishesIt(): void + { + $this->seed(); + $store = $this->record('STORE'); + $old = [$this->locator('users', self::SECRET), $this->locator('orders', self::SECRET)]; + $this->setRenameFn(function ($from, $to) use ($old) { + return in_array(basename($to, '.ndb4'), $old, true) ? false : rename($from, $to); // moving back fails + }); + try { + [$result, $damaged] = $this->rotateWithAFailingVerification(); + } finally { + $this->setRenameFn(null); + } + $this->assertFalse($result['success'], 'never reported as success or as "nothing changed"'); + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $result['code'], (string)$result['error']); + $this->assertFileExists($this->testDbDir . '.nonedb-maintenance/journal.json', 'the rotation stays open'); + $this->assertSame($store, $this->record('STORE')); + \noneDB::clearStaticCache(); + foreach ([self::SECRET, self::NEW] as $key) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $this->code(function () use ($key, $damaged) { $this->with($key)->find($damaged, 0); }), 'the store stays closed'); + \noneDB::clearStaticCache(); + } + $abort = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW, ['abort' => true]); + $this->assertTrue($abort['success'], (string)$abort['error']); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json'); + \noneDB::clearStaticCache(); + $other = $damaged === 'users' ? 'orders' : 'users'; + $this->assertSame($other === 'users' ? 4 : 2, $this->with(self::SECRET)->count($other)); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () use ($damaged) { $this->with(self::SECRET)->find($damaged, 0); })); + } + + /** @test */ + public function aBusyDatabaseStopsMaintenanceBeforeAnythingIsWritten(): void + { + $this->seed(); + $lock = $this->testDbDir . $this->locator('orders', self::SECRET) . '.ndb4/LOCK'; + $fp = fopen($lock, 'c+'); + $this->assertTrue(flock($fp, LOCK_SH)); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + flock($fp, LOCK_UN); + fclose($fp); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::MAINTENANCE_BUSY, $result['code']); + $this->assertSame($before, $this->snapshot($this->testDbDir), '§9.2 step 2: no journal, no epoch, no m'); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($result['success'], (string)$result['error']); + $this->assertRotated('after the lock was released'); + } + + /** @test */ + public function aRunningProcessNeverSeesAnEmptyDatabaseDuringARotation(): void + { + $this->seed(); + $go = $this->testDbDir . '../rot-go'; + $ready = $this->testDbDir . '../rot-ready'; + @unlink($go); + @unlink($ready); + $proc = proc_open([PHP_BINARY, $this->childScript(' + $db = new noneDB(["secretKey" => ' . var_export(self::SECRET, true) . ', "dbDir" => ' . var_export($this->testDbDir, true) . ', "shardSize" => 3, "lockTimeout" => 0.2]); + $seen = []; + $db->find("users", 0); // the store is open in this process + file_put_contents(' . var_export($ready, true) . ', "1"); + $deadline = microtime(true) + 15; + $after = 0; + while (microtime(true) < $deadline && $after < 30) { + $done = is_file(' . var_export($go, true) . '); + foreach (["users" => function () use ($db) { return count($db->find("users", 0)); }, + "missing" => function () use ($db) { return count($db->find("missing", 0)); }, + "write" => function () use ($db) { return $db->insert("users", ["n" => 99])["n"] ?? -1; }, + "create" => function () use ($db) { return $db->insert("newdb", ["z" => 1])["n"] ?? -1; }] as $op => $fn) { + try { $r = $fn(); $seen[] = [$op, "ok", $r, $done]; } catch (RuntimeException $e) { $seen[] = [$op, $e->getCode(), null, $done]; } + } + if ($done) { $after++; } + usleep(5000); + } + echo json_encode($seen); + ')], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $deadline = microtime(true) + 10; + while (!is_file($ready) && microtime(true) < $deadline) { + usleep(5000); + } + // The child keeps working on its locks while the rotation runs: retry while it is busy + $result = ['success' => false]; + for ($i = 0; $i < 200 && !$result['success']; $i++) { + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + if (!$result['success']) { + $this->assertSame(\noneDB::MAINTENANCE_BUSY, $result['code'], (string)$result['error']); + usleep(2000); + } + } + $this->assertTrue($result['success'], 'the rotation got its locks'); + file_put_contents($go, '1'); + $out = stream_get_contents($pipes[1]); + $err = stream_get_contents($pipes[2]); + proc_close($proc); + @unlink($go); + @unlink($ready); + $seen = json_decode($out, true); + $this->assertIsArray($seen, $out . $err); + $users = 4; + $statuses = array_map(function ($row) { return $row[3] ? 'after' : 'before'; }, array_filter($seen, function ($row) { return $row[1] === 'ok'; })); + $this->assertContains('before', $statuses, 'the child worked before the rotation'); + $this->assertNotEmpty(array_filter($seen, function ($row) { return $row[3] && $row[1] === \noneDB::KEY_NOT_REGISTERED; }), 'and was refused after it'); + foreach ($seen as [$op, $status, $value, $done]) { + if ($status === 'ok') { + $this->assertFalse($done, "$op succeeded after the rotation committed: the old key must be refused"); + if ($op === 'users') { + $this->assertGreaterThanOrEqual($users, $value, 'never an empty or shrunken database'); + } + if ($op === 'write' && $value === 1) { + $users++; + } + } else { + $this->assertContains($status, [\noneDB::MAINTENANCE_IN_PROGRESS, \noneDB::KEY_NOT_REGISTERED, \noneDB::LOCK_FAILED], "$op: only these errors"); + } + } + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->locator('users', self::SECRET) . '.ndb4', 'no old-key directory created afterwards'); + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->locator('missing', self::SECRET) . '.ndb4'); + \noneDB::clearStaticCache(); + $this->assertSame($users, count($this->with(self::NEW)->find('users', 0)), 'every acknowledged write is there'); + } + + /** @test */ + public function registerKeyAddsAKeyThatOnlyARegisteredKeyCanAdd(): void + { + $this->seed(); + $result = \noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($result['success'], (string)$result['error']); + $keys = [$this->keyCheck(self::SECRET), $this->keyCheck(self::NEW)]; + sort($keys); + $this->assertSame($keys, $this->record('STORE')['keys']); + $this->assertSame($this->record('STORE'), $this->record('STORE.2')); + \noneDB::clearStaticCache(); + $this->with(self::NEW)->insert('theirs', ['t' => 1]); + $this->assertSame([['t' => 1, 'key' => 0]], $this->with(self::NEW)->find('theirs', 0)); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n')); + $this->assertSame(['orders', 'users'], $this->with(self::SECRET)->getDBs(), 'every key sees only its databases'); + $this->assertSame(['theirs'], $this->with(self::NEW)->getDBs()); + $before = $this->snapshot($this->testDbDir); + $refused = \noneDB::registerKey($this->testDbDir, self::THIRD, 'fourth_secret_key_for_maintenance'); + $this->assertFalse($refused['success']); + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $refused['code'], 'a key that is not registered adds nothing'); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + + /** @test */ + public function everyStoreChangeKilledBetweenItsCopiesIsCompletedByARerun(): void + { + $ops = [ + 'register' => ['noneDB::registerKey(%s, %s, %s)', [self::SECRET, self::NEW]], + 'rotate' => ['noneDB::migrateKey(%s, %s, %s)', [self::SECRET, self::NEW]], + ]; + foreach ($ops as $name => [$call, $args]) { + foreach ([1, 2] as $copy) { + $context = "$name, killed before copy $copy"; + $this->cleanTestDirectory(); + $this->seed(); + $out = $this->runChild(' + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) use (&$n) { if ($p === "store:write" && ++$n === ' . $copy . ') { exit(0); } }); + ' . sprintf($call, var_export($this->testDbDir, true), var_export($args[0], true), var_export($args[1], true)) . '; + echo "not reached"; + '); + $this->assertSame('', $out, $context); + \noneDB::clearStaticCache(); + $this->assertFileExists($this->testDbDir . '.nonedb-maintenance/journal.json', "$context: COMMITTED journal open"); + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $this->code(function () { $this->with(self::SECRET)->find('users', 0); }), $context); + \noneDB::clearStaticCache(); + $rerun = $name === 'register' ? \noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW) + : \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($rerun['success'], "$context: " . $rerun['error']); + $this->assertSame($this->record('STORE'), $this->record('STORE.2'), "$context: identical copies"); + $this->assertContains($this->keyCheck(self::NEW), $this->record('STORE')['keys'], $context); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json', $context); + } + } + } + + /** @test */ + public function aRepairKilledAtEveryStepIsCompletedByARerun(): void + { + foreach ([['repair:committed', 1], ['store:write', 1], ['store:write', 2], ['maintenance:store-written', 1], ['maintenance:close', 1]] as [$point, $hit]) { + $context = "$point #$hit"; + $this->cleanTestDirectory(); + $this->seed(); + $manifest = \noneDB::registerKey($this->testDbDir, self::SECRET, self::THIRD)['storeManifest']; + file_put_contents($this->testDbDir . 'STORE', 'x'); + file_put_contents($this->testDbDir . 'STORE.2', 'y'); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) use (&$n) { if ($p === ' . var_export($point, true) . ' && ++$n === ' . $hit . ') { exit(0); } }); + noneDB::repairStore(' . var_export($this->testDbDir, true) . ', [' . var_export(self::SECRET, true) . '], ["manifest" => ' . var_export($manifest, true) . ']); + echo "not reached"; + '); + $this->assertSame('', $out, $context); + \noneDB::clearStaticCache(); + $this->assertFileExists($this->testDbDir . '.nonedb-maintenance/journal.json', "$context: committed, not closed"); + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $this->code(function () { $this->with(self::SECRET)->find('users', 0); }), $context); + \noneDB::clearStaticCache(); + // The re-run applies the committed journal; the copies on disk do not change its outcome + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], ['confirmManualRebuild' => true]); + $this->assertTrue($result['success'], "$context: " . $result['error']); + $this->assertSame('manifest', $result['mode'], "$context: the committed mode, not the re-run's options"); + $this->assertTrue($result['complete'], $context); + $this->assertSame($manifest['keys'], $this->record('STORE')['keys'], $context); + $this->assertSame($this->record('STORE'), $this->record('STORE.2'), $context); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json', $context); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n'), $context); + } + } + + /** The temp files of atomic replacements in the store directory and .nonedb-maintenance (STORAGE_FORMAT.md §2) */ + private function storeTemps(): array + { + return array_values(array_filter(array_keys($this->snapshot($this->testDbDir)), function ($path) { + return strpos($path, '.ndb4/') === false && preg_match('/\.tmp[0-9a-f]{8}$/', $path) === 1; + })); + } + + /** + * @test STORAGE_FORMAT.md §2: "A temp file left by a crash is never read. It is removed by the next exclusive + * session that owns the directory." A maintenance killed between writing a store file's temp and renaming it + * (the journal, STORE.2, STORE) leaves that temp; the re-run (or an abort) removes it + */ + public function aTempFileACrashedMaintenanceLeftIsRemovedByTheNextOne(): void + { + $cases = [ + 'register, journal' => ['noneDB::registerKey(%s, %s, %s)', 1, '.nonedb-maintenance/journal.json'], + 'register, STORE.2' => ['noneDB::registerKey(%s, %s, %s)', 2, 'STORE.2'], + 'register, STORE' => ['noneDB::registerKey(%s, %s, %s)', 3, 'STORE'], + 'rotate, its second journal state, then aborted' => ['noneDB::migrateKey(%s, %s, %s)', 2, '.nonedb-maintenance/journal.json'], + ]; + foreach ($cases as $context => [$call, $hit, $file]) { + $this->cleanTestDirectory(); + $this->seed(); + $this->assertSame([], $this->storeTemps(), "$context: none before"); + $out = $this->runChild(' + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) use (&$n) { if ($p === "store:renaming" && ++$n === ' . $hit . ') { exit(0); } }); + ' . sprintf($call, var_export($this->testDbDir, true), var_export(self::SECRET, true), var_export(self::NEW, true)) . '; + echo "not reached"; + '); + $this->assertSame('', $out, $context); + $temps = $this->storeTemps(); + $this->assertCount(1, $temps, "$context: the crash left the temp of the file it was writing"); + $this->assertMatchesRegularExpression('/^' . preg_quote($file, '/') . '\.tmp[0-9a-f]{8}$/', $temps[0], $context); + \noneDB::clearStaticCache(); + $rerun = strpos($context, 'aborted') !== false ? \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW, ['abort' => true]) + : \noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($rerun['success'], "$context: " . $rerun['error']); + $this->assertSame([], $this->storeTemps(), "$context: removed by the next maintenance"); + $this->assertFileDoesNotExist($this->testDbDir . '.nonedb-maintenance/journal.json', $context); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n'), $context); + } + } + + /** @test */ + public function aFailingSecondCopyIsNeverReportedAsSuccess(): void + { + $this->seed(); + $n = 0; + $this->setFaultHook(function ($p) use (&$n) { + if ($p === 'store:write' && ++$n === 2) { + throw new \RuntimeException('injected: disk full'); + } + }); + $result = \noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW); + $this->setFaultHook(null); + $this->assertFalse($result['success'], 'committed but not completed is not success'); + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $result['code']); + \noneDB::clearStaticCache(); + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $this->code(function () { $this->with(self::SECRET)->find('users', 0); }), 'the store stays closed'); + \noneDB::clearStaticCache(); + $this->assertTrue(\noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW)['success'], 'the rerun completes it'); + $this->assertSame($this->record('STORE'), $this->record('STORE.2')); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n')); + } + + /** @test */ + public function repairStoreFromAVerifiedManifestIsComplete(): void + { + foreach (['the result array' => 'array', 'a saved STORE copy' => 'file'] as $case => $form) { + $this->cleanTestDirectory(); + $this->seed(); + $registered = \noneDB::registerKey($this->testDbDir, self::SECRET, self::THIRD); // a key without databases + $this->assertSame($this->record('STORE'), $registered['storeManifest'], $case); + $manifest = $registered['storeManifest']; + if ($form === 'file') { + $manifest = $this->testDbDir . '../store-manifest.json'; + copy($this->testDbDir . 'STORE', $manifest); + } + file_put_contents($this->testDbDir . 'STORE', 'x'); + file_put_contents($this->testDbDir . 'STORE.2', 'y'); + \noneDB::clearStaticCache(); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->with(self::SECRET)->find('users', 0); }), $case); + \noneDB::clearStaticCache(); + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], ['manifest' => $manifest]); + if ($form === 'file') { + @unlink($manifest); + } + $this->assertTrue($result['success'], "$case: " . $result['error']); + $this->assertSame('manifest', $result['mode'], $case); + $this->assertTrue($result['complete'], $case); + $store = $this->record('STORE'); + $this->assertSame($registered['storeManifest']['keys'], $store['keys'], "$case: the key without databases is back"); + $this->assertSame($registered['storeManifest']['storeId'], $store['storeId'], "$case: the same store"); + $this->assertSame($registered['storeManifest']['seq'] + 1, $store['seq'], $case); + $this->assertSame($store, $this->record('STORE.2'), $case); + $this->assertSame($store, $result['storeManifest'], $case); + $this->assertStringContainsString('"store-restored-from-manifest"', file_get_contents($this->testDbDir . '.nonedb-maintenance/log.jsonl'), $case); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n'), $case); + $this->with(self::THIRD)->insert('t', ['x' => 1]); // still registered + $this->assertSame(1, $this->with(self::THIRD)->count('t'), $case); + } + } + + /** @test */ + public function repairStoreWithoutManifestNeedsAConfirmedManualRebuild(): void + { + $this->seed(); + \noneDB::registerKey($this->testDbDir, self::SECRET, self::THIRD); + file_put_contents($this->testDbDir . 'STORE', 'x'); + file_put_contents($this->testDbDir . 'STORE.2', 'y'); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $refused = \noneDB::repairStore($this->testDbDir, [self::SECRET]); + $this->assertFalse($refused['success']); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $refused['code']); + $this->assertSame($before, $this->snapshot($this->testDbDir), 'nothing rebuilt without confirmation'); + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], ['confirmManualRebuild' => true]); + $this->assertTrue($result['success'], (string)$result['error']); + $this->assertSame('manual', $result['mode']); + $this->assertFalse($result['complete'], 'never reported as a guaranteed restore'); + $this->assertSame([$this->keyCheck(self::SECRET)], $this->record('STORE')['keys'], 'the key without databases is lost, as stated'); + $this->assertStringContainsString('"manual-store-rebuild"', file_get_contents($this->testDbDir . '.nonedb-maintenance/log.jsonl')); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2, 3, 4], array_column($this->with(self::SECRET)->find('users', 0), 'n')); + } + + /** @test */ + public function repairStoreRefusesWhenADatabaseBelongsToNoGivenKey(): void + { + $this->seed(); + \noneDB::registerKey($this->testDbDir, self::SECRET, self::NEW); + \noneDB::clearStaticCache(); + $this->with(self::NEW)->insert('theirs', ['t' => 1]); + file_put_contents($this->testDbDir . 'STORE', 'x'); + file_put_contents($this->testDbDir . 'STORE.2', 'y'); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + foreach ([[], ['confirmManualRebuild' => true]] as $options) { + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], $options); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $result['code']); + $this->assertStringContainsString($this->locator('theirs', self::NEW) . '.ndb4', (string)$result['error'], 'the unattributed database is named'); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET, self::NEW], ['confirmManualRebuild' => true]); + $this->assertTrue($result['success'], (string)$result['error']); + $keys = [$this->keyCheck(self::SECRET), $this->keyCheck(self::NEW)]; + sort($keys); + $this->assertSame($keys, $this->record('STORE')['keys']); + } + + /** @test */ + public function repairStoreIsOnlyForTwoInvalidCopiesAndRejectsAForgedManifest(): void + { + $this->seed(); + $manifest = \noneDB::registerKey($this->testDbDir, self::SECRET, self::THIRD)['storeManifest']; + file_put_contents($this->testDbDir . 'STORE', 'x'); // STORE.2 still valid: repaired automatically + $before = $this->snapshot($this->testDbDir); + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], ['confirmManualRebuild' => true]); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $result['code']); + $this->assertSame($before, $this->snapshot($this->testDbDir), 'refused before anything is written'); + file_put_contents($this->testDbDir . 'STORE.2', 'y'); + $tampered = $manifest; + $tampered['keys'][] = $this->keyCheck('evil_key_that_was_never_registered'); + sort($tampered['keys']); + $reordered = $manifest; + unset($reordered['checksum']); + $reordered = ['checksum' => $manifest['checksum']] + $reordered; + $forgedFile = $this->testDbDir . '../forged.json'; + file_put_contents($forgedFile, '{"magic":"NONEDB4","kind":"store","storageFormat":4,"storeId":"x","seq":1,"keys":["' . $this->keyCheck('evil') . '"],"checksum":"' . str_repeat('0', 64) . '"}' . "\n"); + $before = $this->snapshot($this->testDbDir); + foreach (['a key added, old checksum' => $tampered, 'checksum not last' => $reordered, 'a file with a wrong checksum' => $forgedFile, + 'a missing file' => $this->testDbDir . '../nothing.json', 'not a record' => ['keys' => []]] as $case => $source) { + $result = \noneDB::repairStore($this->testDbDir, [self::SECRET], ['manifest' => $source]); + $this->assertFalse($result['success'], $case); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $result['code'], $case); + $this->assertSame($before, $this->snapshot($this->testDbDir), "$case: nothing written"); + } + @unlink($forgedFile); + $this->assertTrue(\noneDB::repairStore($this->testDbDir, [self::SECRET], ['manifest' => $manifest])['success'], 'the genuine manifest works'); + } +} diff --git a/tests/Crash/PowerLossSimulationTest.php b/tests/Crash/PowerLossSimulationTest.php new file mode 100644 index 0000000..bb25eb7 --- /dev/null +++ b/tests/Crash/PowerLossSimulationTest.php @@ -0,0 +1,210 @@ +db($config); + $db->createFieldIndex($name, 'c'); + $db->createSpatialIndex($name, 'loc'); + $db->insert($name, array_map(function ($n) { + return ['n' => $n, 'c' => 'c' . ($n % 3), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n / 2]]]; + }, range(0, 13))); + $db->delete($name, ['n' => 4]); + $db->update($name, [['n' => 5], ['set' => ['c' => 'upd']]]); + return $db; + } + + /** Expected state: n => c */ + private function expected(): array + { + $rows = []; + foreach (range(0, 13) as $n) { + if ($n !== 4) { + $rows[$n] = $n === 5 ? 'upd' : 'c' . ($n % 3); + } + } + return $rows; + } + + private function assertState(string $name, array $config, string $case): void + { + \noneDB::clearStaticCache(); + $db = $this->db($config); + $all = $db->find($name, 0); + $this->assertSame($this->expected(), array_column($all, 'c', 'n'), "$case: records"); + $this->assertSame(count($all), $db->count($name), "$case: count"); + foreach (['c0', 'c1', 'c2', 'upd'] as $c) { + $scan = array_values(array_filter($all, function ($r) use ($c) { return $r['c'] === $c; })); + $this->assertSame(array_column($scan, 'key'), array_column($db->find($name, ['c' => $c]), 'key'), "$case: field index $c"); + } + $box = array_column($db->withinBBox($name, 'loc', 0, 0, 6, 3), 'key'); + sort($box); + $expectedBox = []; + foreach ($all as $r) { + if ($r['loc']['coordinates'][0] <= 6 && $r['loc']['coordinates'][1] <= 3) { + $expectedBox[] = $r['key']; + } + } + sort($expectedBox); + $this->assertSame($expectedBox, $box, "$case: spatial index"); + $db->insert($name, ['n' => 100, 'c' => 'new', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + $new = $db->find($name, ['c' => 'new'])[0]['key']; + // build() assigned keys 0..13 (key 4 deleted): nothing was lost, so the next key is 14 + $this->assertSame(14, $new, "$case: no key reused, none skipped"); + $this->assertTrue($db->checkIntegrity($name)['ok'], "$case: consistent afterwards"); + } + + /** Files of a database whose path relative to its directory matches $pattern */ + private function filesOf(string $name, string $pattern): array + { + $home = $this->dbPath($name, 'home'); + return array_values(array_map(function ($f) use ($home) { return $home . $f; }, + array_filter($this->dbFiles($name), function ($f) use ($pattern) { + return preg_match($pattern, $f) === 1; + }))); + } + + /** @test */ + public function emptyOrGarbledDerivedFiles(): void + { + // Derived index files of the active generation (docs/STORAGE_FORMAT.md 5.1): segments/*.kix, + // indexes/*.fx, indexes/*.gfx, spatial/*.rx, spatial/*.grx; the index list indexes.json; the identity + // record FORMAT (holds the creation time; repaired from the generation's metadata, 8.3) + $cases = [ + 'empty key indexes' => '#^generations/[0-9]+/segments/[^/]+\.kix$#', + 'empty field indexes' => '#^generations/[0-9]+/indexes/[^/]+\.fx$#', + 'empty global field index' => '#^generations/[0-9]+/indexes/[^/]+\.gfx$#', + 'empty spatial indexes' => '#^generations/[0-9]+/spatial/[^/]+\.rx$#', + 'empty global spatial index' => '#^generations/[0-9]+/spatial/[^/]+\.grx$#', + 'empty index list' => '#^generations/[0-9]+/indexes\.json$#', + 'empty FORMAT' => '#^FORMAT$#', + ]; + foreach ([['shardingEnabled' => false], ['shardSize' => 5]] as $config) { + foreach ($cases as $case => $pattern) { + $sharded = isset($config['shardSize']); + if (!$sharded && strpos($case, 'global') !== false) { + continue; // global indexes exist only for sharded databases + } + $this->cleanTestDirectory(); + $name = 'pl'; + $this->build($name, $config); + $files = $this->filesOf($name, $pattern); + // one per segment (3 shards), or one per database + $this->assertCount($sharded && in_array($case, ['empty key indexes', 'empty field indexes', 'empty spatial indexes'], true) ? 3 : 1, + $files, "$case: the files this case empties exist"); + foreach ($files as $file) { + file_put_contents($file, ''); + } + $this->assertState($name, $config, $case . ' ' . json_encode($config)); + } + } + } + + /** @test */ + public function unreadableMetaKeepsTheRecordedShardSize(): void + { + // Shard 1 holds only tombstone 14 after compaction: the keys fit every shard size from 10 to 14, the + // configured 12 too; the size the database was created with decides + $db = $this->db(['shardSize' => 10]); + $db->insert('pz', array_map(function ($n) { return ['n' => $n]; }, range(0, 14))); + $db->query('pz')->where(['n' => ['$gte' => 10]])->delete(); + $db->compact('pz'); + file_put_contents($this->dbPath('pz', 'shards'), ''); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 12]); + $this->assertSame(range(0, 9), array_column($db->find('pz', 0), 'n')); + $meta = json_decode(file_get_contents($this->dbPath('pz', 'shards')), true); + $this->assertSame(10, $meta['shardSize']); + $db->insert('pz', ['n' => 'next']); + $this->assertSame(15, $db->find('pz', ['n' => 'next'])[0]['key']); + $this->assertTrue($db->checkIntegrity('pz')['ok']); + } + + /** @test */ + public function emptyOrGarbledMeta(): void + { + foreach (['', '{"kind":"shards","v":1,"shard', "\0\0\0\0"] as $content) { + $this->cleanTestDirectory(); + $this->build('plm', ['shardSize' => 5]); + file_put_contents($this->dbPath('plm', 'shards'), $content); + $this->assertState('plm', ['shardSize' => 5], 'meta ' . json_encode($content)); + } + } + + /** @test */ + public function zeroedLockRecord(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 5]] as $config) { + $this->cleanTestDirectory(); + $this->build('pll', $config); + \noneDB::clearStaticCache(); + file_put_contents($this->dbPath('pll', 'lock'), str_repeat("\0", 128)); + $this->assertState('pll', $config, 'zeroed lock ' . json_encode($config)); + } + } + + /** @test */ + public function dataFileExtendedWithZeroBytes(): void + { + // The file size reached the disk, the appended bytes didn't: a tail of NUL bytes + foreach ([['shardingEnabled' => false], ['shardSize' => 5]] as $config) { + $this->cleanTestDirectory(); + $this->build('plz', $config); + $data = $config === ['shardSize' => 5] ? $this->dbPath('plz', 'data', 2) : $this->dbPath('plz', 'data'); + clearstatcache(); + $offset = filesize($data); + file_put_contents($data, str_repeat("\0", 4096), FILE_APPEND); + $this->assertState('plz', $config, 'NUL tail ' . json_encode($config)); + // The insert of assertState() cut the NUL bytes: kept in recovery/q.{main|s2}.{offset}.4096.{sha256}.bin + $cuts = array_values(array_filter($this->db()->getRecoveryLog('plz'), function ($e) { return $e['action'] === 'cut-tail'; })); + $this->assertCount(1, $cuts, 'the cut tail was quarantined'); + $expected = 'q.' . ($config === ['shardSize' => 5] ? 's2' : 'main') . '.' . $offset . '.4096.' . hash('sha256', str_repeat("\0", 4096)) . '.bin'; + $this->assertSame($expected, $cuts[0]['quarantine']); + $this->assertSame(str_repeat("\0", 4096), file_get_contents($this->dbPath('plz', 'quarantine') . $expected)); + } + } + + /** @test */ + public function zeroBytesFollowedByACompleteLine(): void + { + // A NUL region in front of a line that did reach the disk (the line is recovered) + $this->build('pln', ['shardingEnabled' => false]); + $data = $this->dbPath('pln', 'data'); + file_put_contents($data, str_repeat("\0", 300) . '{"n":50,"c":"late","loc":{"type":"Point","coordinates":[1,1]},"key":14}' . "\n", FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(['late'], array_column($db->find('pln', ['key' => 14]), 'c')); + $this->assertSame('salvaged', $db->getRecoveryLog('pln')[0]['action']); + $db->insert('pln', ['n' => 51, 'c' => 'after']); + $this->assertSame(15, $db->find('pln', ['c' => 'after'])[0]['key']); + } + + /** @test */ + public function emptyBufferAndInterruptedFlush(): void + { + $db = $this->db(['bufferEnabled' => true]); + $db->insert('plb', ['n' => 1]); + $db->flush('plb'); + $buffer = $this->dbPath('plb', 'buffer'); + file_put_contents($buffer, ''); + file_put_contents($this->dbPath('plb', 'flushing', 1), ''); + \noneDB::clearStaticCache(); + $db = $this->db(['bufferEnabled' => true]); + $this->assertSame([1], array_column($db->find('plb', 0), 'n')); + $db->insert('plb', ['n' => 2]); + $db->flush('plb'); + $this->assertSame([1, 2], array_column($db->find('plb', 0), 'n')); + $this->assertSame([], glob($this->dbPath('plb', 'gen') . 'buffer/flushing.*')); + } +} diff --git a/tests/Crash/RollbackScopeTest.php b/tests/Crash/RollbackScopeTest.php new file mode 100644 index 0000000..7f7e6fe --- /dev/null +++ b/tests/Crash/RollbackScopeTest.php @@ -0,0 +1,762 @@ +dbPath($name, 'data'); + } + + /** + * Every file under the test directory (store and database directories), relative path => absolute path + * @return string[] + */ + private function allFiles(): array + { + clearstatcache(); + $files = []; + $root = rtrim($this->testDbDir, '/') . '/'; + $walk = function (string $dir) use (&$walk, &$files, $root): void { + foreach (@scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($dir . $entry)) { + $walk($dir . $entry . '/'); + } else { + $files[substr($dir . $entry, strlen($root))] = $dir . $entry; + } + } + }; + $walk($root); + ksort($files); + return $files; + } + + private function values(\noneDB $db, string $name, string $field = 'v'): array + { + return array_column($db->find($name, 0), $field); + } + + /** Same records after every index file is deleted and rebuilt from the data in a fresh process state */ + private function assertRebuildSame(string $name, array $expected, array $config = [], string $field = 'v'): void + { + \noneDB::clearStaticCache(); + foreach ($this->allFiles() as $file) { + if (substr($file, -4) === '.kix') { + unlink($file); // key-index checkpoints (STORAGE_FORMAT.md §6.4) + } + } + $this->assertSame($expected, $this->values($this->db($config), $name, $field), 'rebuilt from the data'); + } + + /** @test */ + public function nestedSameDbWriteJoinsOuterScope(): void + { + $db = $this->db(); + $db->insert('nest', ['v' => 'base']); + clearstatcache(); + $before = filesize($this->dataFile('nest')); + + $nested = false; + $this->setFaultHook(function ($point) use ($db, &$nested) { + if ($point === 'index:written' && !$nested) { + $nested = true; + $db->insert('nest', ['v' => 'inner']); // e.g. an error handler writing to the same database + throw new \RuntimeException('injected'); + } + }); + try { + $db->insert('nest', ['v' => 'outer']); + $this->fail('the injected exception must reach the caller'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + + clearstatcache(); + $this->assertSame($before, filesize($this->dataFile('nest')), 'outer and nested appends are cut back'); + $this->assertSame(['base'], $this->values($db, 'nest')); + $this->assertRebuildSame('nest', ['base']); + } + + /** @test */ + public function nestedSameDbWriteCommitsWithOuter(): void + { + $db = $this->db(); + $db->insert('nest', ['v' => 'base']); + $nested = false; + $this->setFaultHook(function ($point) use ($db, &$nested) { + if ($point === 'index:written' && !$nested) { + $nested = true; + $db->insert('nest', ['v' => 'inner']); + } + }); + $this->assertSame(1, $db->insert('nest', ['v' => 'outer'])['n']); + $this->setFaultHook(null); + + $this->assertSame(['base', 'outer', 'inner'], $this->values($db, 'nest')); + $this->assertRebuildSame('nest', ['base', 'outer', 'inner']); + } + + /** @test */ + public function flushBeforeOuterAppendsSurvivesOuterFailure(): void + { + $db = $this->db(['bufferEnabled' => true]); + $db->insert('buf', ['v' => 'stored', 'n' => 0]); + $db->flush('buf'); + foreach ([1, 2, 3] as $n) { + $db->insert('buf', ['v' => 'buffered', 'n' => $n]); + } + $this->assertSame(1, count($this->dataLines($this->dataFile('buf'))), 'three records wait in the buffer'); + + $appends = 0; + $data = $this->dataFile('buf'); + $this->setFaultHook(function ($point, $context) use (&$appends, $data) { + // first append to the data file: the flush at the start of update(); second: update() itself + if ($point === 'append:after' && $context['path'] === $data && ++$appends === 2) { + throw new \RuntimeException('injected'); + } + }); + try { + $db->update('buf', [['n' => 0], ['set' => ['v' => 'changed']]]); + $this->fail('the injected exception must reach the caller'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + + $this->assertSame(['stored', 'buffered', 'buffered', 'buffered'], $this->values($db, 'buf'), 'flushed records stay, the update is undone'); + $this->assertSame([], glob($this->dbPath('buf', 'gen') . 'buffer/flushing.*')); + $this->assertNotContains('b', $this->lockState('buf')['flags']); + $this->assertRebuildSame('buf', ['stored', 'buffered', 'buffered', 'buffered'], ['bufferEnabled' => true]); + } + + /** @test */ + public function flushRequestedAfterOuterAppendsIsDeferred(): void + { + // Failure after the flush was requested: the insert is undone, the flush never runs + $db = $this->db(['bufferEnabled' => true]); + $this->setInstanceProperty($db, 'bufferCountLimit', 3); + $db->insert('defer', ['n' => 1]); + $db->insert('defer', ['n' => 2]); + $this->setFaultHook(function ($point, $context) { + if ($point === 'job:deferred' && $context['job'] === 'flush') { + throw new \RuntimeException('injected'); + } + }); + try { + $db->insert('defer', ['n' => 3]); + $this->fail('the injected exception must reach the caller'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + $this->assertSame([], $this->dataLines($this->dataFile('defer')), 'nothing flushed'); + $this->assertCount(2, $this->dataLines($this->dbPath('defer', 'buffer')), 'third record cut from the buffer'); + $this->assertContains('b', $this->lockState('defer')['flags']); + $this->assertSame([1, 2], $this->values($db, 'defer', 'n'), 'the next operation flushes what is buffered'); + + // Without a failure the flush runs once the insert committed + $events = []; + $this->setFaultHook(function ($point, $context) use (&$events) { + if ($point === 'scope:commit' || strncmp($point, 'job:', 4) === 0) { + $events[] = $point . (isset($context['job']) ? ':' . $context['job'] : ''); + } + }); + $db->insert('defer', ['n' => 3]); + $db->insert('defer', ['n' => 4]); + $db->insert('defer', ['n' => 5]); + $this->setFaultHook(null); + $this->assertSame([1, 2, 3, 4, 5], array_column(array_map('json_decode', $this->dataLines($this->dataFile('defer')), array_fill(0, 5, true)), 'n')); + $this->assertSame(['job:deferred:flush', 'scope:commit', 'job:flush', 'scope:commit'], array_slice($events, -4), 'flush after the commit'); + } + + /** @test */ + public function theSwitchToShardsAndCompactionRunAfterCommit(): void + { + // The switch to shards: requested by the insert that reaches shardSize + $db = $this->db(['shardSize' => 5]); + $db->insert('mig', [['n' => 1], ['n' => 2], ['n' => 3], ['n' => 4]]); + $this->setFaultHook(function ($point, $context) { + if ($point === 'job:deferred' && $context['job'] === 'convertToSharded') { + throw new \RuntimeException('injected'); + } + }); + try { + $db->insert('mig', ['n' => 5]); + $this->fail('the injected exception must reach the caller'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + $this->assertFileDoesNotExist($this->dbPath('mig', 'shards')); + $this->assertSame([1, 2, 3, 4], $this->values($db, 'mig', 'n')); + + $events = []; + $this->setFaultHook(function ($point) use (&$events) { + $events[] = $point; + }); + $db->insert('mig', ['n' => 5]); + $this->setFaultHook(null); + $this->assertFileExists($this->dbPath('mig', 'shards')); + $this->assertLessThan(array_search('job:convertToSharded', $events, true), array_search('scope:commit', $events, true)); + $this->assertSame([1, 2, 3, 4, 5], $this->values($db, 'mig', 'n')); + + // Compaction: requested by an update that leaves too much garbage + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 0.1]); + $db->insert('cmp', array_map(function ($n) { return ['n' => $n, 'v' => 'a']; }, range(1, 10))); + $this->setFaultHook(function ($point, $context) { + if ($point === 'job:deferred' && $context['job'] === 'compact:main') { + throw new \RuntimeException('injected'); + } + }); + try { + $db->update('cmp', [['v' => 'a'], ['set' => ['v' => 'b']]]); + $this->fail('the injected exception must reach the caller'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + $this->assertCount(10, $this->dataLines($this->dataFile('cmp')), 'update undone, nothing compacted'); + $this->assertSame(array_fill(0, 10, 'a'), $this->values($db, 'cmp')); + + $events = []; + $this->setFaultHook(function ($point) use (&$events) { + $events[] = $point; + }); + $this->assertSame(10, $db->update('cmp', [['v' => 'a'], ['set' => ['v' => 'b']]])['n']); + $this->setFaultHook(null); + $this->assertCount(10, $this->dataLines($this->dataFile('cmp')), 'compacted after the commit'); + $this->assertLessThan(array_search('job:compact:main', $events, true), array_search('scope:commit', $events, true)); + $this->assertSame(array_fill(0, 10, 'b'), $this->values($db, 'cmp')); + } + + /** @test */ + public function deferredJobFailureKeepsCommittedResult(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 0.1]); + $db->insert('dj', array_map(function ($n) { return ['n' => $n, 'g' => $n <= 5 ? 'low' : 'high', 'v' => 'a']; }, range(1, 10))); + $this->setFaultHook(function ($point) { + if ($point === 'job:compact:main') { + throw new \RuntimeException('compaction broke'); + } + }); + $this->assertSame(5, $db->update('dj', [['g' => 'low'], ['set' => ['v' => 'b']]])['n'] ?? null, + 'the update reports its committed result'); + $this->setFaultHook(null); + $this->assertSame(['b', 'b', 'b', 'b', 'b', 'a', 'a', 'a', 'a', 'a'], $this->values($db, 'dj')); + $this->assertCount(15, $this->dataLines($this->dataFile('dj')), 'not compacted'); + $log = $db->getRecoveryLog('dj'); + $this->assertSame('deferred-failed', end($log)['action']); + $this->assertSame('compact:main', end($log)['job']); + $this->assertSame('compaction broke', end($log)['error']); + + // The next write that leaves garbage compacts + $db->update('dj', [['n' => 6], ['set' => ['v' => 'c']]]); + $this->assertCount(10, $this->dataLines($this->dataFile('dj'))); + $this->assertSame(['b', 'b', 'b', 'b', 'b', 'c', 'a', 'a', 'a', 'a'], $this->values($db, 'dj')); + } + + /** @test */ + public function independentWorkNeverRunsInDirtyScope(): void + { + $registry = new \noneDB\Tests\InternalValue('lockRegistry'); + $runs = 0; + $dirty = []; + $this->setFaultHook(function ($point, $context) use ($registry, &$runs, &$dirty) { + if ($point !== 'independent:run') { + return; + } + $runs++; + foreach ($registry->getValue() as $entry) { + if ($entry['dbname'] === $context['db'] && !empty($entry['scope']['marks'])) { + $dirty[] = $context['job']; + } + } + }); + $db = $this->db(['bufferEnabled' => true, 'shardSize' => 8, 'autoCompactThreshold' => 0.2]); + $this->setInstanceProperty($db, 'bufferCountLimit', 3); + $direct = $this->db(['shardSize' => 8, 'autoCompactThreshold' => 0.2]); + for ($i = 0; $i < 30; $i++) { + $db->insert('mix', ['n' => $i, 'v' => 'x']); + if ($i % 4 === 3) { + $direct->update('mix', [['n' => $i - 1], ['set' => ['v' => 'y']]]); + } + if ($i % 7 === 6) { + $direct->delete('mix', ['n' => $i - 3]); + } + } + $this->setFaultHook(null); + $this->assertGreaterThan(5, $runs); + $this->assertSame([], $dirty, 'flush/switch to shards/compaction never run on top of uncommitted appends'); + $expected = []; + for ($i = 0; $i < 30; $i++) { + if ($i % 7 !== 3 || $i + 3 >= 30) { + $expected[] = $i; + } + } + $this->assertSame($expected, array_column($db->find('mix', 0), 'n')); + } + + /** + * Run $nested inside a write of $name that appended a record and then fails (user code re-entering + * noneDB from a write, e.g. an error handler) + */ + private function failingWriteWith(\noneDB $db, string $name, callable $nested): \Throwable + { + $method = new \noneDB\Tests\InternalMethod('withLock'); + try { + $method->invoke($db, $name, true, function () use ($db, $name, $nested) { + $db->insert($name, ['v' => 'uncommitted']); + $nested(); + throw new \RuntimeException('outer failed'); + }); + } catch (\Throwable $e) { + return $e; + } + $this->fail('the outer write must fail'); + } + + private function fileSnapshot(): array + { + $files = []; + foreach ($this->allFiles() as $name => $file) { + if (basename($file) !== 'LOCK' && basename($file) !== 'STORE.LOCK') { + // shards.json keeps the keys a failed write reserved (never reused): its content may change + $files[$name] = basename($file) === 'shards.json' ? 'meta' : hash_file('sha256', $file); + } + } + return $files; + } + + /** @test */ + public function dataFileReplacementIsRefusedInsideAnUncommittedWrite(): void + { + // dropDB(), compact() and convertToSharded() replace or remove data files: a rollback can't undo that, so + // inside a write that appended and isn't committed they must refuse before touching any file + $cases = [ + // [operation, config, sharded database] + ['dropDB', ['shardingEnabled' => false], false], + ['dropDB', ['shardSize' => 5], true], + ['compact', ['shardingEnabled' => false], false], + ['compact', ['shardSize' => 5], true], + ['convertToSharded', ['shardSize' => 5, 'shardingEnabled' => false], false], + ['convertToSharded', ['shardSize' => 5], true], // already sharded: refused all the same, before looking + ]; + foreach ($cases as [$operation, $config, $sharded]) { + $this->cleanTestDirectory(); + $db = $this->db($config); + if ($sharded) { + // keys 0-4 (shard 0) deleted, the records below go to shard 1: a sharded database + $db->insert('dirty', array_fill(0, 5, ['v' => 'filler'])); + $db->delete('dirty', ['v' => 'filler']); + } + $db->insert('dirty', [['v' => 'original'], ['v' => 'second']]); + $db->delete('dirty', ['v' => 'second']); + $db->insert('dirty', ['v' => 'second']); // something for compaction to drop + $this->assertSame($sharded, $db->getShardInfo('dirty')['sharded'], 'set up'); + $before = $this->fileSnapshot(); + $label = $operation . ' ' . json_encode($config); + + $e = $this->failingWriteWith($db, 'dirty', function () use ($db, $operation) { + $db->$operation('dirty'); + }); + $this->assertInstanceOf(\LogicException::class, $e, $label . ': ' . get_class($e) . ' ' . $e->getMessage()); + $this->assertStringContainsString($operation . '()', $e->getMessage()); + $this->assertSame($before, $this->fileSnapshot(), $label . ': no file changed'); + $this->assertSame(['original', 'second'], $this->values($this->db($config), 'dirty'), $label); + + // Caught by the caller's code: the outer write commits, still nothing replaced + $method = new \noneDB\Tests\InternalMethod('withLock'); + $refused = $method->invoke($db, 'dirty', true, function () use ($db, $operation) { + $db->insert('dirty', ['v' => 'third']); + try { + $db->$operation('dirty'); + } catch (\LogicException $e) { + return true; + } + return false; + }); + $this->assertTrue($refused, $label); + $this->assertSame(['original', 'second', 'third'], $this->values($db, 'dirty'), $label); + + // Inside a write that has appended nothing yet it is refused too (workThatReplacesFilesIsRefusedBefore...) + $refused = $method->invoke($db, 'dirty', true, function () use ($db, $operation) { + try { + $db->$operation('dirty'); + } catch (\LogicException $e) { + return true; + } + return false; + }); + $this->assertTrue($refused, $label); + + // Outside a write it runs + $result = $db->$operation('dirty'); + $this->assertTrue($result['success'] ?? null, $label . ': ' . json_encode($result)); + if ($operation === 'convertToSharded') { + $this->assertSame($sharded ? 'already_sharded' : 'converted', $result['status'], $label); + } + $this->assertSame($operation === 'dropDB' ? [] : ['original', 'second', 'third'], $this->values($this->db($config), 'dirty'), $label); + $this->assertRebuildSame('dirty', $operation === 'dropDB' ? [] : ['original', 'second', 'third'], $config); + } + } + + /** @test */ + public function aFileWithUncommittedAppendsIsNeverReplaced(): void + { + // Internal paths too (not only the public methods): replacing or removing a file the scope + // appended to is refused, so the rollback can always cut it back + $db = $this->db(['shardingEnabled' => false]); + $db->insert('mark', [['v' => 'original'], ['v' => 'second']]); + $compact = new \noneDB\Tests\InternalMethod('compactJsonl'); + $before = $this->fileSnapshot(); + $e = $this->failingWriteWith($db, 'mark', function () use ($db, $compact) { + $compact->invoke($db, 'mark', null); + }); + $this->assertInstanceOf(\LogicException::class, $e, get_class($e) . ' ' . $e->getMessage()); + $this->assertNotSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode()); + $this->assertSame(['original', 'second'], $this->values($this->db(['shardingEnabled' => false]), 'mark')); + $this->assertSame(array_keys($before), array_keys($this->fileSnapshot())); + $this->assertRebuildSame('mark', ['original', 'second'], ['shardingEnabled' => false]); + + // Every file operation that replaces, renames or removes a file refuses one the scope appended to + $data = $this->dbPath('mark', 'data'); + $attempts = [ + 'removal' => function () use ($data) { return $this->fsUnlink($data); }, + 'rename' => function () use ($data) { return $this->fsRename($data, $data . '.moved'); }, + 'replacement by rename' => function () use ($data) { return $this->fsRename($data . '.other', $data); }, + 'replacement' => function () use ($data) { return $this->writeFileAtomic($data, "{}\n", 'replace'); }, + ]; + foreach ($attempts as $label => $attempt) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + file_put_contents($data . '.other', "{\"v\":\"other\",\"key\":0}\n"); + $before = $this->fileSnapshot(); + $e = $this->failingWriteWith($db, 'mark', function () use ($attempt, $db) { + $attempt->call(\noneDB\Tests\Support\Testing::engine($db)); + }); + $this->assertInstanceOf(\LogicException::class, $e, $label . ': ' . get_class($e) . ' ' . $e->getMessage()); + $this->assertStringContainsString('refused', $e->getMessage(), $label); + $this->assertSame($before, $this->fileSnapshot(), $label . ': the appends were cut back, nothing else changed'); + $this->assertFileDoesNotExist($data . '.moved', $label); + $this->assertSame(['original', 'second'], $this->values($this->db(['shardingEnabled' => false]), 'mark'), $label); + unlink($data . '.other'); + } + } + + /** @test the scope belongs to one database: a nested write to another database commits on its own */ + public function nestedWriteToAnotherDatabaseCommitsOnItsOwn(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('outer', ['v' => 'base']); + $db->insert('inner', ['v' => 'base']); + $e = $this->failingWriteWith($db, 'outer', function () use ($db) { + $db->insert('inner', ['v' => 'nested']); + }); + $this->assertSame('outer failed', $e->getMessage()); + $this->assertSame(['base'], $this->values($db, 'outer'), 'the failed write is undone'); + $this->assertSame(['base', 'nested'], $this->values($db, 'inner'), 'the other database committed its write'); + $this->assertRebuildSame('outer', ['base']); + $this->assertRebuildSame('inner', ['base', 'nested']); + } + + /** @test the refusal is per database: dropDB()/compact() of another database run inside a dirty write */ + public function replacingAnotherDatabaseInsideAnUncommittedWriteIsAllowed(): void + { + $config = ['shardingEnabled' => false, 'autoCompactThreshold' => 1000]; + $db = $this->db($config); + $db->insert('own', ['v' => 'base']); + $db->insert('other', [['v' => 'keep'], ['v' => 'gone']]); + $db->delete('other', ['v' => 'gone']); + $db->insert('third', ['v' => 'x']); + $results = []; + $e = $this->failingWriteWith($db, 'own', function () use ($db, &$results) { + $results[] = $db->compact('other'); + $results[] = $db->dropDB('third'); + }); + $this->assertSame('outer failed', $e->getMessage(), 'not refused'); + $this->assertTrue($results[0]['success']); + $this->assertSame(1, $results[0]['freedSlots']); + $this->assertTrue($results[1]['success']); + $this->assertSame(['base'], $this->values($db, 'own')); + $this->assertSame(['keep'], $this->values($db, 'other')); + $this->assertSame(['{"v":"keep","key":0}', '{"__nonedb_deleted":1}'], $this->dataLines($this->dataFile('other')), 'compacted'); + $this->assertSame([], $db->getDBs('third') ?: []); + } + + /** + * @test documented behaviour: "compaction, the switch to shards and a flush the call itself triggers run + * after the call committed. If that later work fails, the call still reports success; the failure goes to + * getRecoveryLog() and the work is retried later." + */ + public function failedDeferredFlushAndSwitchToShardsAreLoggedAndRetried(): void + { + // A flush the insert triggered (count limit reached) + $db = $this->db(['bufferEnabled' => true]); + $this->setInstanceProperty($db, 'bufferCountLimit', 3); + $db->insert('df', ['n' => 1]); + $db->insert('df', ['n' => 2]); + $this->setFaultHook(function ($point) { + if ($point === 'job:flush') { + throw new \RuntimeException('flush failed'); + } + }); + $this->assertSame(['n' => 1], $db->insert('df', ['n' => 3]), 'the insert committed'); + $this->setFaultHook(null); + $log = $db->getRecoveryLog('df'); + $this->assertSame(['deferred-failed', 'flush', 'flush failed'], [end($log)['action'], end($log)['job'], end($log)['error']]); + $this->assertSame(3, $db->getBufferInfo('df')['buffers']['main']['records'], 'still buffered'); + $this->assertSame([1, 2, 3], $this->values($db, 'df', 'n'), 'the next operation flushes'); + $this->assertSame(0, $db->getBufferInfo('df')['buffers']['main']['records']); + $this->assertCount(3, $this->dataLines($this->dataFile('df'))); + + // A switch to shards the insert triggered (shardSize reached) + $db = $this->db(['shardSize' => 5]); + $db->insert('dm', array_map(function ($n) { return ['n' => $n]; }, range(1, 4))); + $this->setFaultHook(function ($point) { + if ($point === 'job:convertToSharded') { + throw new \RuntimeException('switch failed'); + } + }); + $this->assertSame(['n' => 1], $db->insert('dm', ['n' => 5])); + $this->setFaultHook(null); + $log = $db->getRecoveryLog('dm'); + $this->assertSame(['deferred-failed', 'convertToSharded', 'switch failed'], [end($log)['action'], end($log)['job'], end($log)['error']]); + $this->assertFalse($db->getShardInfo('dm')['sharded']); + $this->assertSame(range(1, 5), $this->values($db, 'dm', 'n')); + $db->insert('dm', ['n' => 6]); // retried + $this->assertTrue($db->getShardInfo('dm')['sharded']); + $this->assertSame(range(1, 6), $this->values($db, 'dm', 'n')); + $this->assertSame(range(0, 5), $this->values($db, 'dm', 'key')); + } + + /** + * @test a file that was replaced or removed behind the scope's back can't be cut back to its mark: the outcome + * is unknown (OUTCOME_UNKNOWN, journal "J"), and the next access recovers + */ + public function rollbackOfAFileReplacedOrRemovedMeanwhileFails(): void + { + foreach (['replaced', 'removed'] as $what) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('gone', ['v' => 'base']); + $data = $this->dataFile('gone'); + $this->setFaultHook(function ($point, $context) use ($data, $what) { + if ($point === 'append:after' && $context['path'] === $data) { + if ($what === 'replaced') { + copy($data, $data . '.copy'); + rename($data . '.copy', $data); // same bytes, another file + } else { + unlink($data); + } + throw new \RuntimeException('injected'); + } + }); + try { + $db->insert('gone', ['v' => 'second']); + $this->fail($what . ': must throw'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode(), $what . ': ' . $e->getMessage()); + $this->assertSame('injected', $e->getPrevious()->getMessage(), $what); + } + $this->setFaultHook(null); + $this->assertContains('J', $this->lockState('gone')['flags'], $what); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + // replaced: the copy holds the complete line (roll forward); removed: the data is gone + $this->assertSame($what === 'replaced' ? ['base', 'second'] : [], $this->values($db, 'gone'), $what); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('gone')['flags']), $what . ': recovered'); + } + } + + /** @test */ + public function migrateKeyIsRefusedWhileThisProcessHoldsALock(): void + { + // It would wait forever for the lock this process holds (flock() of another handle) + $db = $this->db(['shardingEnabled' => false]); + $db->insert('mk', ['v' => 'a']); + $method = new \noneDB\Tests\InternalMethod('withLock'); + $before = $this->fileSnapshot(); + foreach ([false, true] as $exclusive) { + $result = $method->invoke($db, 'mk', $exclusive, function () { + return \noneDB::migrateKey($this->testDbDir, self::SECRET, 'another-secret-key-1234567890'); + }); + $this->assertSame(['code', 'databases', 'error', 'storeManifest', 'success'], $this->sortedKeys($result)); + $this->assertFalse($result['success']); + $this->assertSame([], $result['databases']); + $this->assertNull($result['storeManifest'], 'no new store record'); + $this->assertStringContainsString('lock', (string)$result['error']); + $this->assertSame($before, $this->fileSnapshot(), 'nothing renamed'); + } + $this->assertSame(['a'], $this->values($db, 'mk')); + // Outside every operation it runs + $this->assertTrue(\noneDB::migrateKey($this->testDbDir, self::SECRET, 'another-secret-key-1234567890')['success']); + $this->assertSame(['a'], $this->values(new \noneDB(['secretKey' => 'another-secret-key-1234567890', 'dbDir' => $this->testDbDir]), 'mk')); + } + + /** @test */ + public function multiShardInsertIsUndoneOnEveryShard(): void + { + // A-14: the append to shard 2 fails after shard 1's (and after the meta reserved both) + $db = $this->db(['shardSize' => 5]); + $db->insert('ms', array_map(function ($n) { return ['n' => $n]; }, range(0, 4))); + $this->assertFileExists($this->dbPath('ms', 'shards')); + $this->setFaultHook(function ($point, $context) { + if ($point === 'append:after' && substr($context['path'], -strlen('/segments/s2.jsonl')) === '/segments/s2.jsonl') { + throw new \RuntimeException('shard 2 failed'); + } + }); + try { + $db->insert('ms', array_map(function ($n) { return ['n' => $n]; }, range(5, 12))); + $this->fail('the append to shard 2 must fail'); + } catch (\RuntimeException $e) { + $this->assertSame('shard 2 failed', $e->getMessage()); + } + $this->setFaultHook(null); + clearstatcache(); + $this->assertSame(0, filesize($this->dbPath('ms', 'data', 1)), 'shard 1 cut back too'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $this->assertSame(range(0, 4), $this->values($db, 'ms', 'n'), 'no record of the failed insert on any shard'); + $this->assertSame(5, $db->count('ms'), 'meta counters follow the data'); + $db->insert('ms', ['n' => 99]); + $keys = array_column($db->find('ms', 0), 'key'); + $this->assertGreaterThan(12, end($keys), 'reserved keys are never reused'); + $this->assertRebuildSame('ms', [0, 1, 2, 3, 4, 99], ['shardSize' => 5], 'n'); + } + + /** @test */ + public function failedRollbackReportsUnknownOutcomeAndRecovers(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('rf', 'v'); + $db->insert('rf', ['v' => 'base']); + $this->setFaultHook(function ($point) { + if ($point === 'append:after') { + throw new \RuntimeException('injected'); + } + if ($point === 'rollback:truncate') { + throw new \RuntimeException('disk gone'); + } + }); + try { + $db->insert('rf', ['v' => 'second']); + $this->fail('must throw'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode()); + $this->assertNotNull($e->getPrevious()); + $this->assertSame('injected', $e->getPrevious()->getMessage()); + } + $this->setFaultHook(null); + $this->assertContains('J', $this->lockState('rf')['flags'], 'the next lock holder recovers everything'); + + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + // The line could not be cut: it is a complete record, so recovery keeps it (roll forward) + $this->assertSame(['base', 'second'], $this->values($db, 'rf')); + $this->assertSame(['second'], array_column($db->find('rf', ['v' => 'second']), 'v'), 'field index rebuilt'); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('rf')['flags'])); + } + + /** @test */ + public function theSwitchToShardsInterruptedAtEveryStepStaysConsistent(): void + { + // P0.4 (10): a process killed during the switch to shards (exit() skips finally blocks, like a fatal): + // while the new generation is built, and after each step of the generation switch (STORAGE_FORMAT.md + // §5.2: PREPARE, INTENT, PUBLISH, CLEANUP) + foreach (['switch:shard-written', 'generation:prepared', 'generation:intent', 'generation:published', 'generation:cleaned'] as $point) { + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 4]); + $db->createFieldIndex('mig', 'c'); + $db->insert('mig', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2)]; }, range(0, 2))); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + $db = ' . $this->childDb(['shardSize' => 4]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { + if ($p === ' . var_export($point, true) . ') { exit(0); } + }); + $db->insert("mig", [["n" => 3, "c" => "c1"], ["n" => 4, "c" => "c0"]]); // reaches shardSize: the switch to shards after the commit + echo "not reached"; + '); + $this->assertSame('', $out, $point); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + $this->assertSame(range(0, 4), array_column($db->find('mig', 0), 'n'), "$point: every record once"); + $this->assertSame([1, 3], array_column($db->find('mig', ['c' => 'c1']), 'n'), "$point: field index"); + $db->insert('mig', ['n' => 5, 'c' => 'c1']); + $this->assertSame(range(0, 5), array_column($db->find('mig', 0), 'n'), "$point: still writable"); + $this->assertSame([1, 3, 5], array_column($db->find('mig', ['c' => 'c1']), 'n'), $point); + $this->assertRebuildSame('mig', range(0, 5), ['shardSize' => 4], 'n'); + // The shards the interrupted switch wrote are its own: removed or reused, never kept as a + // second copy of the data under *.orphan.* + $this->assertSame([], preg_grep('/\.orphan\./', $this->dbFiles('mig')), "$point: no orphan copies"); + $this->assertArrayNotHasKey('orphanFiles', $db->getDBs('mig'), $point); + // Nothing of the interrupted switch is left: one generation, the one CURRENT names (§5.2) + $current = json_decode((string)file_get_contents($this->dbPath('mig', 'current')), true)['generation']; + $this->assertSame([(string)$current], array_values(array_diff(scandir($this->dbPath('mig', 'home') . 'generations'), ['.', '..'])), "$point: one generation left"); + $this->assertSame([], $this->lockState('mig')['flags'], "$point: no switch pending (§5.2 DONE)"); + } + } + + private function sortedKeys(array $result): array + { + $keys = array_keys($result); + sort($keys); + return $keys; + } + + /** + * @test The calls that replace or remove files are refused inside a write of the same database from its first step, + * not from its first append: a value's jsonSerialize() runs before it. A drop there left the write on a database + * that was gone, and every later call on FORMAT_DAMAGED. + */ + public function workThatReplacesFilesIsRefusedBeforeTheWriteHasAppended(): void + { + $calls = [ + 'dropDB' => function ($db, $name) { return $db->dropDB($name); }, + 'compact' => function ($db, $name) { return $db->compact($name); }, + 'convertToSharded' => function ($db, $name) { return $db->convertToSharded($name); }, + ]; + foreach ($calls as $label => $call) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ne', [['x' => 0]]); + $value = new class($db, $call) implements \JsonSerializable { + public $thrown = null; + private $calls = 0; + private $db; + private $call; + public function __construct($db, $call) { $this->db = $db; $this->call = $call; } + public function jsonSerialize(): mixed + { + // The first call checks the input, before the lock; the second encodes the line inside the write + if (++$this->calls === 2) { + try { + ($this->call)($this->db, 'ne'); + $this->thrown = false; + } catch (\Throwable $e) { + $this->thrown = $e; + } + } + return 1; + } + }; + $db->insert('ne', ['x' => $value]); + $this->assertInstanceOf(\LogicException::class, $value->thrown, $label); + $this->assertSame([['x' => 0, 'key' => 0], ['x' => 1, 'key' => 1]], $db->find('ne', []), $label); + $this->assertTrue($db->checkIntegrity('ne')['ok'], $label); + } + } +} diff --git a/tests/Crash/TornWriteRecoveryTest.php b/tests/Crash/TornWriteRecoveryTest.php new file mode 100644 index 0000000..ab0269d --- /dev/null +++ b/tests/Crash/TornWriteRecoveryTest.php @@ -0,0 +1,512 @@ +dbPath($name, 'data', $shard); + } + + private function values(\noneDB $db, string $name, string $field = 'v'): array + { + return array_column($db->find($name, 0), $field); + } + + private function freshDb(array $config = []): \noneDB + { + \noneDB::clearStaticCache(); + return $this->db($config); + } + + /** recovery/q.{segment}.{offset}.{length}.{sha256}.bin (STORAGE_FORMAT.md 7.2) */ + private function quarantineFiles(string $name): array + { + return glob($this->dbPath($name, 'quarantine') . 'q.*.bin') ?: []; + } + + /** Removes the key-index checkpoints of every database */ + private function deleteIndexes(): void + { + foreach (glob(rtrim($this->testDbDir, '/') . '/*.ndb4/generations/*/segments/*.kix') as $file) { + unlink($file); + } + } + + /** @test */ + public function completeUnindexedLineIsVisibleAndItsKeyNotReused(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('roll', ['v' => 'initial']); + file_put_contents($this->dataFile('roll'), "{\"v\":\"after-crash\",\"key\":1}\n", FILE_APPEND); + + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['initial', 'after-crash'], $this->values($db, 'roll')); + $db->insert('roll', ['v' => 'next']); + $this->assertSame([0, 1, 2], array_column($db->find('roll', 0), 'key'), 'key 1 is not handed out again'); + } + + /** @test */ + public function unindexedTombstoneDeletesImmediately(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('tomb', [['v' => 'a'], ['v' => 'b']]); + file_put_contents($this->dataFile('tomb'), "{\"__nonedb_deleted\":0}\n", FILE_APPEND); + $this->assertSame(['b'], $this->values($this->freshDb(['shardingEnabled' => false]), 'tomb')); + } + + /** @test */ + public function sameProcessWriterNoticesFileChangedBehindItsCache(): void + { + // Something that isn't noneDB appended a line: the generation didn't change, the cached index is + // stale. The writer notices (the size its append finds), writes nothing and drops the index; the + // next operation rebuilds it from the data (a torn tail alone is cut and the write goes on). + $db = $this->db(['shardingEnabled' => false]); + $db->insert('stale', ['v' => 'a']); + file_put_contents($this->dataFile('stale'), "{\"v\":\"b\",\"key\":1}\n", FILE_APPEND); + try { + $db->insert('stale', ['v' => 'c']); + $this->fail('the write must be refused'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('changed unexpectedly', $e->getMessage()); + } + $db->insert('stale', ['v' => 'c']); + $this->assertSame(['a', 'b', 'c'], $this->values($db, 'stale')); + $this->assertSame([0, 1, 2], array_column($db->find('stale', 0), 'key')); + $this->deleteIndexes(); + $this->assertSame(['a', 'b', 'c'], $this->values($this->freshDb(['shardingEnabled' => false]), 'stale')); + } + + /** @test */ + public function tornTailNeverSwallowsTheNextInsert(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('torn', ['v' => 'first']); + $fragment = '{"uncommitted":"fragment"'; + clearstatcache(); + $offset = filesize($this->dataFile('torn')); + file_put_contents($this->dataFile('torn'), $fragment, FILE_APPEND); + $db->insert('torn', ['v' => 'second']); + + $this->deleteIndexes(); + $this->assertSame(['first', 'second'], $this->values($this->freshDb(['shardingEnabled' => false]), 'torn')); + + // The cut bytes are kept, verified, and logged + $files = $this->quarantineFiles('torn'); + $this->assertCount(1, $files); + $this->assertSame($fragment, file_get_contents($files[0])); + // recovery/q.{segment}.{offset}.{length}.{sha256}.bin, segment main = the single layout's data file + $this->assertSame('q.main.' . $offset . '.' . strlen($fragment) . '.' . hash('sha256', $fragment) . '.bin', basename($files[0])); + $log = $this->db()->getRecoveryLog('torn'); + $this->assertSame('cut-tail', $log[0]['action']); + $this->assertTrue($log[0]['bytesVerified']); + $this->assertSame(basename($files[0]), $log[0]['quarantine']); + $this->assertSame(PHP_OS_FAMILY !== 'Windows' ? true : 'unknown', $log[0]['powerLossDurable']); + } + + /** @test */ + public function completeJsonWithoutNewlineIsNotCommitted(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('tornfull', ['v' => 'first']); + file_put_contents($this->dataFile('tornfull'), '{"v":"should-disappear","key":99}', FILE_APPEND); + // A reader in a fresh process ignores the tail and leaves the file alone + clearstatcache(); + $size = filesize($this->dataFile('tornfull')); + $this->assertSame(['first'], $this->values($this->freshDb(['shardingEnabled' => false]), 'tornfull')); + clearstatcache(); + $this->assertSame($size, filesize($this->dataFile('tornfull')), 'readers never cut'); + $this->assertSame([], $this->quarantineFiles('tornfull')); + + $db = $this->db(['shardingEnabled' => false]); + $db->insert('tornfull', ['v' => 'second']); + $this->deleteIndexes(); + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['first', 'second'], $this->values($db, 'tornfull')); + $this->assertSame([], $db->find('tornfull', ['key' => 99])); + } + + /** @test */ + public function tornTailOnAShardIsCutBeforeTheNextAppend(): void + { + $db = $this->db(['shardSize' => 5]); + $db->insert('tsh', array_map(function ($n) { return ['v' => $n]; }, range(0, 6))); + clearstatcache(); + $offset = filesize($this->dataFile('tsh', 1)); + file_put_contents($this->dataFile('tsh', 1), '{"v":"half', FILE_APPEND); + $db = $this->freshDb(['shardSize' => 5]); + $db->insert('tsh', ['v' => 7]); + $this->deleteIndexes(); + $this->assertSame(range(0, 7), $this->values($this->freshDb(['shardSize' => 5]), 'tsh')); + $this->assertSame(range(0, 7), array_column($this->freshDb(['shardSize' => 5])->find('tsh', 0), 'key')); + $files = $this->quarantineFiles('tsh'); + $this->assertCount(1, $files); + // the segment of shard N is sN + $this->assertSame('q.s1.' . $offset . '.10.' . hash('sha256', '{"v":"half') . '.bin', basename($files[0])); + $this->assertSame('{"v":"half', file_get_contents($files[0])); + } + + /** @test */ + public function gluedLineIsSalvaged(): void + { + $name = 'glued'; + $db = $this->db(['shardingEnabled' => false]); + $db->insert($name, ['v' => 'first']); + // A torn append followed by a committed one in the same line: the line does not decode as a whole, + // the salvage rule finds the record at its end (STORAGE_FORMAT.md 6.1); no checkpoint covers it + $data = $this->dataFile($name); + $fragment = '{"frag":"' . str_repeat('{', 300) . 'x'; + $record = '{"v":"second","n":{"key":5},"key":1}'; + file_put_contents($data, $fragment . $record . "\n", FILE_APPEND); + unlink($this->dbPath($name, 'kix')); + \noneDB::clearStaticCache(); + $this->db(['shardingEnabled' => false])->rebuildIndex($name); + + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['first', 'second'], $this->values($db, $name)); + $this->assertSame(['key' => 5], $db->find($name, ['key' => 1])[0]['n']); + $log = $db->getRecoveryLog($name); + $this->assertSame('salvaged', $log[0]['action']); + $this->assertSame(1, $log[0]['key']); + + // Compaction keeps the record and moves the whole damaged line to quarantine + $db->compact($name); + $this->assertSame(['first', 'second'], $this->values($this->freshDb(['shardingEnabled' => false]), $name)); + $files = $this->quarantineFiles($name); + $this->assertCount(1, $files); + $this->assertSame($fragment . $record . "\n", file_get_contents($files[0])); + $this->assertSame('drop-line', $this->db()->getRecoveryLog($name)[1]['action']); + } + + /** @test */ + public function damagedLineEndingInAKeyDoesNotMakeTheDatabaseUnreadable(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('bad', ['v' => 'ok']); + $data = $this->dataFile('bad'); + clearstatcache(); + $at = filesize($data); + $broken = '{"a":[1,2,"key":5}'; + file_put_contents($data, $broken . "\n", FILE_APPEND); + // A checkpoint that points at the damaged line (as if it had been written there); its crc "h" is the + // one of the old end, so it doesn't fit the data and is repaired + $ckpt = $this->dbPath('bad', 'kix'); + $file = $this->readKeyIndexFile($ckpt); + $index = $file['header']; + $entries = $file['entries']; + $entries[5] = [$at, strlen($broken)]; + $index['n'] = 6; + $index['s'] = $at + strlen($broken) + 1; + $this->writeKeyIndexFile($ckpt, $index, $entries); + + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['ok'], $this->values($db, 'bad'), 'forced repair classifies strictly, the read succeeds'); + $this->assertSame('skip-line', $db->getRecoveryLog('bad')[0]['action']); + $db->insert('bad', ['v' => 'next']); + // The repair keeps the key counter of the checkpoint it replaces (n = 6): the next key is 6 + $this->assertSame(6, $db->find('bad', ['v' => 'next'])[0]['key'], 'the index n is kept'); + } + + /** @test */ + public function checkpointBehindTheDataIsReplayedUnderTheJournal(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('leg', 'c'); + $db->insert('leg', ['c' => 'x', 'v' => 'a']); + clearstatcache(); + $covered = filesize($this->dataFile('leg')); + file_put_contents($this->dataFile('leg'), "{\"c\":\"x\",\"v\":\"b\",\"key\":1}\n", FILE_APPEND); + // The key checkpoint covers at most the data before the new line + $this->assertLessThanOrEqual($covered, $this->readKeyIndexFile($this->dbPath('leg', 'kix'))['header']['s']); + // Journal entry as a crashed writer leaves it + $state = $this->lockState('leg'); + $this->writeLockState('leg', ($state['gen'] + 1) . ':' . $state['epoch'] . ' j=main'); + + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['a', 'b'], $this->values($db, 'leg')); + $this->assertSame(['a', 'b'], array_column($db->find('leg', ['c' => 'x']), 'v'), 'field index follows'); + } + + /** @test */ + public function secondaryIndexesFollowACheckpointBehindTheData(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('sec', 'c'); + $db->createSpatialIndex('sec', 'loc'); + $db->insert('sec', ['c' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + // A writer died between its data append and the end of its session: the key checkpoint and the + // secondary indexes cover at most the data before the line + clearstatcache(); + $this->assertLessThanOrEqual(filesize($this->dataFile('sec')), $this->readKeyIndexFile($this->dbPath('sec', 'kix'))['header']['s']); + file_put_contents($this->dataFile('sec'), + '{"c":"x","loc":{"type":"Point","coordinates":[2,2]},"key":1}' . "\n", FILE_APPEND); + + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame([0, 1], array_column($db->find('sec', ['c' => 'x']), 'key')); + $got = array_column($db->withinBBox('sec', 'loc', 0, 0, 3, 3), 'key'); + sort($got); + $this->assertSame([0, 1], $got); + } + + /** @test */ + public function unencodableRecordLeavesNoJournal(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('enc', 'c'); + $db->insert('enc', ['c' => 'x']); + $gen = $this->lockState('enc')['gen']; + // docs/API.md: rejected with an error result (no exception), nothing written + $result = $db->insert('enc', ['c' => "\xB1\x31"]); + $this->assertSame(0, $result['n']); + $this->assertStringStartsWith('insert data cannot be encoded as JSON', $result['error']); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('enc')['flags'])); + $this->assertSame($gen, $this->lockState('enc')['gen'], 'no file written'); + $this->assertSame([['c' => 'x', 'key' => 0]], $db->find('enc', 0)); + } + + /** @test */ + public function bufferTornTailDoesNotLoseRecords(): void + { + $db = $this->db(['bufferEnabled' => true]); + $db->insert('btorn', ['v' => 'one']); + $buffer = $this->dbPath('btorn', 'buffer'); + clearstatcache(); + $offset = filesize($buffer); + file_put_contents($buffer, '{"v":"half', FILE_APPEND); + $db = $this->freshDb(['bufferEnabled' => true]); + $db->insert('btorn', ['v' => 'two']); + $db->flush('btorn'); + $this->assertSame(['one', 'two'], $this->values($db, 'btorn')); + $this->assertSame([0, 1], array_column($db->find('btorn', 0), 'key')); + $this->assertCount(1, $this->quarantineFiles('btorn')); + $this->assertSame('{"v":"half', file_get_contents($this->quarantineFiles('btorn')[0])); + // segment buffer = a buffer file + $this->assertSame('q.buffer.' . $offset . '.10.' . hash('sha256', '{"v":"half') . '.bin', basename($this->quarantineFiles('btorn')[0])); + $this->assertSame(['cut-tail'], array_column($db->getRecoveryLog('btorn'), 'action')); + } + + /** @test */ + public function flushedBufferWithDamagedLinesIsQuarantinedBeforeDeletion(): void + { + $db = $this->db(['bufferEnabled' => true]); + $db->insert('bdmg', ['v' => 'one']); + $buffer = $this->dbPath('bdmg', 'buffer'); + file_put_contents($buffer, "not json\n", FILE_APPEND); + $content = file_get_contents($buffer); + $db = $this->freshDb(['bufferEnabled' => true]); + $this->assertSame(['success' => true, 'flushed' => 1, 'error' => null], $db->flush('bdmg')); + $this->assertSame(['one'], $this->values($db, 'bdmg')); + $files = $this->quarantineFiles('bdmg'); + $this->assertCount(1, $files); + $this->assertSame($content, file_get_contents($files[0])); + $this->assertSame([], glob($this->dbPath('bdmg', 'gen') . 'buffer/*'), 'the flushed buffer is deleted once its bytes are kept'); + $this->assertSame(['drop-buffer'], array_column($db->getRecoveryLog('bdmg'), 'action')); + } + + /** @test */ + public function quarantineFailureRefusesTheWriteAndCutsNothing(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('qfail', ['v' => 'first']); + file_put_contents($this->dataFile('qfail'), '{"half', FILE_APPEND); + clearstatcache(); + $size = filesize($this->dataFile('qfail')); + $this->setFaultHook(function ($point) { + if ($point === 'quarantine:write') { + throw new \RuntimeException('disk full'); + } + }); + $db = $this->freshDb(['shardingEnabled' => false]); + try { + $db->insert('qfail', ['v' => 'second']); + $this->fail('the write must be refused'); + } catch (\RuntimeException $e) { + $this->assertSame('disk full', $e->getMessage()); + } + $this->setFaultHook(null); + clearstatcache(); + $this->assertSame($size, filesize($this->dataFile('qfail')), 'nothing cut, nothing appended'); + $this->assertSame(['first'], $this->values($db, 'qfail'), 'still readable'); + $db->insert('qfail', ['v' => 'second']); + $this->assertSame(['first', 'second'], $this->values($this->freshDb(['shardingEnabled' => false]), 'qfail')); + } + + /** @test */ + public function twoDistinctTornWritesAtSameOffsetAndLengthPreserveBothQuarantines(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('twice', ['v' => 'first']); + $data = $this->dataFile('twice'); + clearstatcache(); + $size = filesize($data); + file_put_contents($data, '{"aaaa', FILE_APPEND); + $this->freshDb(['shardingEnabled' => false])->delete('twice', ['v' => 'nothing']); // no append: no cut + // The next write cuts the tail, then fails: its own append is undone, the file is back at $size + $this->setFaultHook(function ($point) { + if ($point === 'append:after') { + throw new \RuntimeException('injected'); + } + }); + try { + $this->freshDb(['shardingEnabled' => false])->insert('twice', ['v' => 'x']); + $this->fail('must throw'); + } catch (\RuntimeException $e) { + $this->assertSame('injected', $e->getMessage()); + } + $this->setFaultHook(null); + clearstatcache(); + $this->assertSame($size, filesize($data)); + file_put_contents($data, '{"bbbb', FILE_APPEND); + $db = $this->freshDb(['shardingEnabled' => false]); + $db->insert('twice', ['v' => 'y']); + $this->assertSame(['first', 'y'], $this->values($db, 'twice')); + + $files = $this->quarantineFiles('twice'); + $this->assertCount(2, $files, 'same segment, offset and length, different content'); + $contents = array_map('file_get_contents', $files); + sort($contents); + $this->assertSame(['{"aaaa', '{"bbbb'], $contents); + $ids = array_column(array_filter($db->getRecoveryLog('twice'), function ($e) { + return $e['action'] === 'cut-tail'; + }), 'quarantine'); + sort($ids); + $names = array_map('basename', $files); + sort($names); + $this->assertSame($names, $ids); + } + + /** @test */ + public function repeatingARecoveryReusesItsQuarantineFile(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('again', ['v' => 'first']); + file_put_contents($this->dataFile('again'), '{"half', FILE_APPEND); + // The quarantine file is written, then the process fails before cutting + $this->setFaultHook(function ($point) { + static $published = false; + if ($point === 'append:after' && !$published) { + $published = true; + throw new \RuntimeException('crash after the cut'); + } + }); + $db = $this->freshDb(['shardingEnabled' => false]); + try { + $db->insert('again', ['v' => 'second']); + } catch (\RuntimeException $e) { + $this->assertSame('crash after the cut', $e->getMessage()); + } + $this->setFaultHook(null); + // Put the same torn tail back and recover again + file_put_contents($this->dataFile('again'), '{"half', FILE_APPEND); + $this->freshDb(['shardingEnabled' => false])->insert('again', ['v' => 'second']); + $this->assertCount(1, $this->quarantineFiles('again')); + $this->assertCount(1, $this->db()->getRecoveryLog('again'), 'the same event is logged once'); + $this->assertSame(['first', 'second'], $this->values($this->freshDb(['shardingEnabled' => false]), 'again')); + } + + /** @test */ + public function readerOfADamagedFileWritesNothing(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ro', [['v' => 'a'], ['v' => 'b']]); + // A damaged line after the checkpoint (replayed by readers) and an unterminated tail + file_put_contents($this->dataFile('ro'), "garbage line\n", FILE_APPEND); + file_put_contents($this->dataFile('ro'), '{"torn', FILE_APPEND); + \noneDB::clearStaticCache(); + $snapshot = function () { + clearstatcache(); + $files = []; + $root = rtrim($this->testDbDir, '/') . '/'; + $walk = function (string $dir) use (&$walk, &$files, $root) { + foreach (scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $f = $dir . $entry; + if (is_dir($f)) { + $walk($f . '/'); + continue; + } + // (Docker Desktop's file sharing renumbers inodes by itself: there the content stands in) + $files[substr($f, strlen($root))] = [filesize($f), filemtime($f), file_exists('/.dockerenv') ? sha1_file($f) : fileinode($f)]; + } + }; + $walk($root); + unset($files[substr($this->dbPath('ro', 'lock'), strlen($root))]); + return $files; + }; + $before = $snapshot(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(['a', 'b'], $this->values($db, 'ro')); + $this->assertSame(2, $db->count('ro')); + $this->assertSame($before, $snapshot(), 'no file created, changed or cut by reads'); + // The damaged line is a diagnostic until a write session of this process logs it + $log = $db->getRecoveryLog('ro'); + $this->assertSame(['skip-line'], array_column($log, 'action')); + $this->assertFalse($log[0]['persisted']); + $db->insert('ro', ['v' => 'c']); + $log = $db->getRecoveryLog('ro'); + $this->assertSame(['skip-line', 'cut-tail'], array_column($log, 'action')); + $this->assertArrayNotHasKey('persisted', $log[0]); + } + + /** @test */ + public function fatalInsideADeleteThenShutdownWriteStaysConsistent(): void + { + // A-13: exit() skips finally blocks but runs shutdown functions, like a fatal error + $db = $this->db(['shardingEnabled' => false]); + $db->insert('fatal', [['v' => 'keep'], ['v' => 'gone'], ['v' => 'keep2']]); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + $db = ' . $this->childDb(['shardingEnabled' => false]) . '; + $db->find("fatal", 0); // caches loaded, as in a long-running process + register_shutdown_function(function () use ($db) { + $r = $db->insert("fatal", ["v" => "from-shutdown"]); + echo "shutdown insert n=", $r["n"], "\n"; + }); + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { + static $done = false; + if ($point === "append:after" && !$done) { $done = true; exit(0); } // after the tombstone, before its index + }); + $db->delete("fatal", ["v" => "gone"]); + echo "not reached\n"; + '); + $this->assertSame('shutdown insert n=1', $out); + $db = $this->freshDb(['shardingEnabled' => false]); + $this->assertSame(['keep', 'keep2', 'from-shutdown'], $this->values($db, 'fatal')); + $this->deleteIndexes(); + $this->assertSame(['keep', 'keep2', 'from-shutdown'], $this->values($this->freshDb(['shardingEnabled' => false]), 'fatal')); + } + + /** + * @test §6.4 a header that announces more than its file holds is no checkpoint: it is not read, however large the + * number. An integer overflow gave a TypeError, a body of terabytes would have been asked of fread() + */ + public function aCheckpointThatAnnouncesAnImpossibleBodyIsNotRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('sz', array_map(function ($i) { return ['n' => $i]; }, range(0, 199))); + $path = $this->dbPath('sz', 'kix'); + $intact = (string)file_get_contents($path); + foreach ([PHP_INT_MAX, 1 << 62, 1 << 45, 1 << 40, 5000000000] as $slots) { + $header = $this->openHeaderLine($path, rtrim(substr($intact, 0, 511))); + $header['slots'] = $slots; + file_put_contents($path, str_pad($this->sealHeaderLine(json_encode($header)), 511) . "\n" . substr($intact, 512)); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . ';' + . ' echo count($db->find("sz", [])), " ", $db->find("sz", ["key" => 7])[0]["n"];'); + $this->assertSame('200 7', $out, "slots $slots"); + } + } +} diff --git a/tests/Regression/DifferentialFuzzTest.php b/tests/Differential/DifferentialFuzzTest.php similarity index 80% rename from tests/Regression/DifferentialFuzzTest.php rename to tests/Differential/DifferentialFuzzTest.php index 7ccf3e9..7c96efa 100644 --- a/tests/Regression/DifferentialFuzzTest.php +++ b/tests/Differential/DifferentialFuzzTest.php @@ -1,6 +1,8 @@ $rb; + if ($ra === 3) return (rtrim($a, " \t\n\r\v\f") + 0) <=> (rtrim($b, " \t\n\r\v\f") + 0); + if ($ra === 4) return strcmp($a, $b) <=> 0; + if ($ra === 5) return strcmp(json_encode($a), json_encode($b)) <=> 0; + return $a <=> $b; } private function modelQuery(array $q): array @@ -170,9 +191,10 @@ private function modelQuery(array $q): array foreach ($q['whereNotIn'] as [$f, $vals]) { if (array_key_exists($f, $r) && in_array($r[$f], $vals, true)) return false; } + // A field of whereNot() matches when all its conditions match; any matching field excludes foreach ($q['whereNot'] as $g) { foreach ($g as $f => $v) { - if (array_key_exists($f, $r) && $r[$f] === $v) return false; + if (self::groupMatch($r, [$f => $v])) return false; } } foreach ($q['between'] as [$f, $min, $max]) { @@ -190,11 +212,7 @@ private function modelQuery(array $q): array $decorated[] = [$i, $r]; } usort($decorated, function ($x, $y) use ($field, $order) { - $a = $x[1][$field] ?? null; - $b = $y[1][$field] ?? null; - $ra = self::sortRank($a); - $rb = self::sortRank($b); - $c = $ra !== $rb ? $ra <=> $rb : ($ra === 4 ? strcmp(json_encode($a), json_encode($b)) : $a <=> $b); + $c = self::sortCompare($x[1][$field] ?? null, $y[1][$field] ?? null); if ($order === 'desc') $c = -$c; return $c !== 0 ? $c : $x[0] <=> $y[0]; }); @@ -209,10 +227,13 @@ private function modelQuery(array $q): array private function randValue(string $field) { + if ($this->exotic && mt_rand(0, 3) === 0) { + return self::EXOTIC[mt_rand(0, count(self::EXOTIC) - 1)]; + } switch ($field) { case 'a': return [0, 1, 2, 3, 5, 8, '3', '8', 2.5, null, -0.0][mt_rand(0, 10)]; - case 'b': return ['x', 'y', 'z', 'Z', '', true, false, null][mt_rand(0, 7)]; - case 'c': return mt_rand(0, 1) ? mt_rand(-5, 5) / 2 : 'w' . mt_rand(0, 3); + case 'b': return ['x', 'y', 'z', 'Z', '', true, false, null, '10', '9', '1a', '9 '][mt_rand(0, 11)]; + case 'c': return mt_rand(0, 1) ? mt_rand(-5, 5) / 2 : ['w' . mt_rand(0, 3), '2', ' 7', '1e1', '1e1x'][mt_rand(0, 4)]; default: return [['t' => mt_rand(0, 1)], [mt_rand(0, 2)], []][mt_rand(0, 2)]; } } @@ -275,7 +296,7 @@ private function randQuery(): array for ($i = mt_rand(0, 2) === 0 ? 1 : 0; $i > 0; $i--) $q['or'][] = $this->randOperatorGroup(); if (mt_rand(0, 3) === 0) $q['whereIn'][] = ['b', ['x', 'y', false]]; if (mt_rand(0, 3) === 0) $q['whereNotIn'][] = ['a', [0, '3']]; - if (mt_rand(0, 3) === 0) $q['whereNot'][] = ['b' => 'z']; + if (mt_rand(0, 3) === 0) $q['whereNot'][] = [['b' => 'z'], ['a' => ['$gte' => 2, '$lt' => 5]], ['b' => 'z', 'c' => ['$gt' => 0]]][mt_rand(0, 2)]; if (mt_rand(0, 3) === 0) $q['between'][] = ['a', 1, '5']; if (mt_rand(0, 1)) $q['sort'] = [['a', 'b', 'c', 'd'][mt_rand(0, 3)], mt_rand(0, 1) ? 'asc' : 'desc']; if (mt_rand(0, 2) === 0) $q['limit'] = mt_rand(0, 5); @@ -378,6 +399,17 @@ private function fuzz(string $name, array $config, int $ops, int $seed, bool $al \noneDB::enableStaticCache(); \noneDB::clearStaticCache(); $this->assertSame(self::withKeys($this->modelSelect([])), $this->db($config)->find($name, 0), "seed=$seed final"); + + // Key indexes rebuilt from the data alone give the same (a checkpoint is only a shortcut): every + // database's segments/{seg}.kix (STORAGE_FORMAT.md §6.4) + foreach (glob(rtrim($this->testDbDir, '/') . '/*.ndb4/generations/*/segments/*.kix') ?: [] as $file) { + unlink($file); + } + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(self::withKeys($this->modelSelect([])), $db->find($name, 0), "seed=$seed without checkpoints"); + $db->insert($name, ['after' => true]); + $this->assertSame($this->nextKey, $db->find($name, ['after' => true])[0]['key'], "seed=$seed no key reused"); } /** @test */ @@ -387,7 +419,7 @@ public function nonSharded(): void } /** @test */ - public function shardedWithMigrationMidRun(): void + public function shardedWithTheSwitchMidRun(): void { $this->fuzz('fzs', ['shardSize' => 7], 600, 202); } @@ -413,11 +445,39 @@ public function twoInstancesWithSeparateCaches(): void $this->fuzz('fzts', ['shardSize' => 6], 500, 506, true, ['a']); } + /** + * @test CHANGES: stored floats don't depend on serialize_precision; docs/API.md: "Floats keep their + * type". The same model, with exotic values mixed in and serialize_precision = 5 during the whole run. + */ + public function exoticValuesWithALowSerializePrecision(): void + { + $precision = ini_get('serialize_precision'); + ini_set('serialize_precision', '5'); + $this->exotic = true; + try { + $this->fuzz('fze', [], 400, 606); + $this->fuzz('fzes', ['shardSize' => 5], 400, 607, false, ['a', 'c']); + } finally { + $this->exotic = false; + ini_set('serialize_precision', (string)$precision); + } + } + /** @test */ public function manySeeds(): void { - for ($seed = 1000; $seed < 1012; $seed++) { + // Extra seeds for longer runs: NONEDB_FUZZ_SEEDS=1,2,3 + $seeds = range(1000, 1011); + $extra = getenv('NONEDB_FUZZ_SEEDS'); + if (is_string($extra) && trim($extra) !== '') { + $seeds = array_map('intval', array_filter(array_map('trim', explode(',', $extra)), 'strlen')); + } + foreach ($seeds as $seed) { + // Some runs sharded; some with a cache limit of 1 byte (every cache entry evicted when unused) $config = $seed % 3 === 0 ? ['shardSize' => 5 + $seed % 4] : []; + if ($seed % 5 === 2) { + $config['cacheMemoryLimit'] = 1; + } $indexes = $seed % 2 === 0 ? ['a', 'b'] : []; $this->fuzz('fzm' . $seed, $config, 150, $seed, $seed % 4 === 1, $indexes); } diff --git a/tests/Differential/GeometryConsistencyTest.php b/tests/Differential/GeometryConsistencyTest.php new file mode 100644 index 0000000..5be4fc5 --- /dev/null +++ b/tests/Differential/GeometryConsistencyTest.php @@ -0,0 +1,146 @@ +db(['spatialIndexEnabled' => $indexed]); + if ($indexed) { + $db->createSpatialIndex($name, 'g'); + $db->insert($name, ['g' => $record]); + } + $results[] = count($db->withinPolygon($name, 'g', $query)) === 1; + } + $this->assertSame($results[0], $results[1], 'index == scan'); + return $results[0]; + } + + /** @test */ + public function polygonHolesAreRespected(): void + { + // B-3: the cases the plan verified by hand + $hole = [[[4, 4], [6, 4], [6, 6], [4, 6], [4, 4]]]; + $query = ['type' => 'Polygon', 'coordinates' => $this->square(0, 10, $hole)]; + + // A record around the query's hole is not within + $this->assertFalse($this->within(['type' => 'Polygon', 'coordinates' => $this->square(2, 8)], $query)); + // Same hole in the record, the query's hole ring starting elsewhere: within + $holeFrom66 = ['type' => 'Polygon', 'coordinates' => $this->square(0, 10, [[[6, 6], [4, 6], [4, 4], [6, 4], [6, 6]]])]; + $this->assertTrue($this->within(['type' => 'Polygon', 'coordinates' => $this->square(2, 8, $hole)], $holeFrom66)); + // A record hole that contains the query's hole: within + $this->assertTrue($this->within(['type' => 'Polygon', 'coordinates' => $this->square(1, 9, [[[3, 3], [7, 3], [7, 7], [3, 7], [3, 3]]])], $query)); + + // Diamond hole; the record's bottom edge runs through two hole vertices: not within + $diamond = ['type' => 'Polygon', 'coordinates' => $this->square(0, 10, [[[5, 3], [6, 5], [5, 7], [4, 5], [5, 3]]])]; + $this->assertFalse($this->within(['type' => 'Polygon', 'coordinates' => [[[3, 5], [9.5, 5], [9.5, 9], [3, 9], [3, 5]]]], $diamond)); + $this->assertFalse($this->within(['type' => 'LineString', 'coordinates' => [[3, 5], [9.5, 5]]], $diamond), 'a line through the hole'); + $this->assertTrue($this->within(['type' => 'LineString', 'coordinates' => [[1, 1], [9, 1]]], $diamond)); + + // Every hole vertex on the record's outer ring: the hole is inside the record, not within + $kite = ['type' => 'Polygon', 'coordinates' => $this->square(0, 10, [[[4, 2], [8, 4], [6, 8], [2, 6], [4, 2]]])]; + $this->assertFalse($this->within(['type' => 'Polygon', 'coordinates' => $this->square(2, 8)], $kite)); + } + + /** @test */ + public function polygonBoundaryIsClosedLikeTheBoundingBox(): void + { + // B-13: points on the east and north edges were outside, on the west and south edges inside + $rect = ['type' => 'Polygon', 'coordinates' => $this->square(0, 10)]; + foreach ([[10, 5], [5, 10], [10, 10], [0, 5], [5, 0], [0, 0], [3, 3]] as $point) { + $this->assertTrue($this->within(['type' => 'Point', 'coordinates' => $point], $rect), json_encode($point)); + } + $this->assertFalse($this->within(['type' => 'Point', 'coordinates' => [10.0001, 5]], $rect)); + $this->assertTrue($this->within(['type' => 'Polygon', 'coordinates' => $this->square(5, 10)], $rect), 'touching the east/north edges'); + + $db = $this->db(); + $db->createSpatialIndex('edges', 'g'); + foreach ([[10, 5], [5, 10], [0, 0], [10, 10], [11, 5], [4, 4]] as $point) { + $db->insert('edges', ['g' => ['type' => 'Point', 'coordinates' => $point]]); + } + $this->assertSame( + array_column($db->withinBBox('edges', 'g', 0, 0, 10, 10), 'key'), + array_column($db->withinPolygon('edges', 'g', $rect), 'key') + ); + } + + /** @test */ + public function multiPartRecordsMayUseDifferentQueryParts(): void + { + // B-14: each part of a multi-part record had to fit one and the same query part + $twoSquares = ['type' => 'MultiPolygon', 'coordinates' => [ + [[[0, 0], [5, 0], [5, 5], [0, 5], [0, 0]]], + [[[5, 0], [10, 0], [10, 5], [5, 5], [5, 0]]], + ]]; + $this->assertTrue($this->within(['type' => 'MultiPoint', 'coordinates' => [[2, 2], [7, 2]]], $twoSquares)); + $this->assertTrue($this->within(['type' => 'MultiLineString', 'coordinates' => [[[1, 1], [4, 1]], [[6, 1], [9, 1]]]], $twoSquares)); + // Documented limit: one part spanning two query parts is not within (the union isn't computed) + $this->assertFalse($this->within(['type' => 'LineString', 'coordinates' => [[2, 2], [7, 2]]], $twoSquares)); + } + + /** @test */ + public function nearestAfterAnotherSpatialFilterUsesItsOwnDistance(): void + { + // B-4: the intersection kept the _distance of the first filter, so nearest() sorted by it + $db = $this->db(); + $db->createSpatialIndex('nd', 'loc'); + $db->insert('nd', [['n' => 'P0', 'loc' => ['type' => 'Point', 'coordinates' => [29.00, 41.00]]], + ['n' => 'P1', 'loc' => ['type' => 'Point', 'coordinates' => [29.05, 41.00]]]]); + $rows = $db->query('nd')->withinDistance('loc', 29.0, 41.0, 10000)->nearest('loc', 29.05, 41.0, 1)->get(); + $this->assertSame(['P1'], array_column($rows, 'n')); + $this->assertSame(0.0, $rows[0]['_distance'], "nearest()'s distance, not withinDistance()'s 4.2 km"); + $both = $db->query('nd')->withinDistance('loc', 29.0, 41.0, 10000)->nearest('loc', 29.05, 41.0, 2)->get(); + $this->assertSame(['P1', 'P0'], array_column($both, 'n')); + $this->assertEqualsWithDelta(SpecSpatialTest::hav(29.05, 41.0, 29.0, 41.0), $both[1]['_distance'], 0.006); + } + + /** @test */ + public function rebuildSpatialIndexWhileDisabledKeepsTheIndex(): void + { + // B-15: it dropped the index first, then creating it failed with "disabled" + $db = $this->db(); + $db->createSpatialIndex('rbd', 'loc'); + $db->insert('rbd', ['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + $disabled = $this->db(['spatialIndexEnabled' => false]); + $result = $disabled->rebuildSpatialIndex('rbd', 'loc'); + $this->assertFalse($result['success']); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled'], $result); + $this->assertSame(['loc'], $db->getSpatialIndexes('rbd'), 'still listed'); + $this->assertCount(1, $db->withinBBox('rbd', 'loc', 0, 0, 2, 2)); + $this->assertFileExists($this->dbPath('rbd', 'rx', 'loc'), 'the index file is kept'); + } + + /** @test docs: "points on its boundary are inside, on every edge": diagonal edges too (exact coordinates) */ + public function diagonalEdgesAreClosedToo(): void + { + $triangle = ['type' => 'Polygon', 'coordinates' => [[[0, 0], [8, 0], [0, 8], [0, 0]]]]; + foreach ([[4, 4], [2, 6], [0.5, 7.5], [7.75, 0.25], [8, 0], [0, 8], [3, 0], [0, 3], [1, 1]] as $point) { + $this->assertTrue($this->within(['type' => 'Point', 'coordinates' => $point], $triangle), json_encode($point)); + } + foreach ([[4.0000001, 4], [4, 4.0000001], [8.0000001, 0], [-0.0000001, 3]] as $point) { + $this->assertFalse($this->within(['type' => 'Point', 'coordinates' => $point], $triangle), json_encode($point)); + } + $this->assertTrue($this->within(['type' => 'LineString', 'coordinates' => [[8, 0], [0, 8]]], $triangle), 'along the diagonal'); + $this->assertTrue($this->within(['type' => 'LineString', 'coordinates' => [[0, 0], [4, 4]]], $triangle), 'a chord to the diagonal'); + $this->assertFalse($this->within(['type' => 'LineString', 'coordinates' => [[0, 0], [5, 5]]], $triangle), 'through the diagonal'); + $this->assertTrue($this->within(['type' => 'Polygon', 'coordinates' => [[[0, 0], [8, 0], [0, 8], [0, 0]]]], $triangle), 'itself'); + } +} diff --git a/tests/Differential/PagedIndexDifferentialTest.php b/tests/Differential/PagedIndexDifferentialTest.php new file mode 100644 index 0000000..46474aa --- /dev/null +++ b/tests/Differential/PagedIndexDifferentialTest.php @@ -0,0 +1,178 @@ +> 1; + switch ($j % 8) { + case 0: return 'v' . $j; + case 1: return str_repeat(chr(97 + $j % 26), 1 + $j % 300); // up to 300 bytes + case 2: return $j; + case 3: return (string)$j; // "7" next to 7 + case 4: return $j + 0.5; + case 5: return $j % 4 === 1; + case 6: return null; + default: return -$j; + } + } + + /** The model grouped by value: serialize(value) => keys holding a value identical (===) to it, ascending */ + private static function modelGroups(array $model): array + { + $groups = []; + foreach ($model as $key => $v) { + $groups[serialize($v)][] = $key; + } + return $groups; + } + + /** Every value in the model and a few that are not there */ + private static function probes(array $model): array + { + $probes = []; + foreach ($model as $v) { + $probes[serialize($v)] = $v; + } + foreach (['absent', '', 999999, -999999, 1.25, 'v-1', str_repeat('z', 301)] as $v) { + $probes[serialize($v)] = $v; + } + return array_values($probes); + } + + /** + * Every value through find() and through the index itself: the candidates of every segment (point + * lookups, or the loaded index) must be exactly the model's keys. find() checks its candidates against + * the records, so only the candidates show an index that holds too many keys. + */ + private function assertLookups(array $config, array $model, string $phase): void + { + // Twice: with room in the cache (an index used by a second operation is loaded whole), and near the + // cache limit (pages only, every lookup through them) + $this->assertLookupsWith($config + ['cacheMemoryLimit' => 0], $model, "$phase (cache with room)"); // process-wide: set each time + \noneDB::clearStaticCache(); + $this->assertLookupsWith($config + ['cacheMemoryLimit' => memory_get_usage() + 8 * 1048576], $model, "$phase (pages only)"); + } + + private function assertLookupsWith(array $config, array $model, string $phase): void + { + \noneDB::clearStaticCache(); + $db = $this->db($config); + $probes = self::probes($model); + $groups = self::modelGroups($model); + $segments = Testing::call($db, 'listSegments', 'pd'); + foreach ([1, 2] as $round) { + foreach ($probes as $value) { + $label = "$phase, round $round: " . var_export($value, true); + $expected = $groups[serialize($value)] ?? []; + $this->assertSame($expected, array_column($db->find('pd', ['v' => $value]), 'key'), $label); + $candidates = Testing::call($db, 'withLock', 'pd', false, function () use ($db, $segments, $value) { + $keys = []; + foreach ($segments as $segment) { + $found = Testing::call($db, 'fieldIndexCandidates', 'pd', $segment, ['v' => $value]); + $this->assertIsArray($found, 'the index answers'); + $keys = array_merge($keys, $found); + } + sort($keys); + return $keys; + }, 'read'); + $this->assertSame($expected, $candidates, "$label (index candidates)"); + } + } + } + + /** + * Writes after the index, the model kept in step: values moved (into and out of "shared"), keys + * deleted, new keys. $pad: bytes added to each line (a long tail passes 64 KiB). $next: the key the + * next insert gets (keys are never used again, a deleted highest key included) + */ + private function write(array $config, array &$model, int &$next, int $seed, int $moves, int $pad): void + { + \noneDB::clearStaticCache(); + $db = $this->db($config); + mt_srand($seed); + $extra = $pad > 0 ? ['pad' => str_repeat('p', $pad)] : []; + for ($n = 0; $n < $moves; $n++) { + $key = mt_rand(0, $next - 1); + $value = $n % 3 === 0 ? 'shared' : self::value(mt_rand(0, 5000) | 1); + $db->update('pd', [['key' => $key], ['set' => ['v' => $value] + $extra]]); + if (array_key_exists($key, $model)) { + $model[$key] = $value; + } + } + for ($n = 0; $n < intdiv($moves, 3); $n++) { + $key = mt_rand(0, $next - 1); + $db->delete('pd', ['key' => $key]); + unset($model[$key]); + } + $new = []; + for ($i = $next; $i < $next + intdiv($moves, 2); $i++) { + $new[] = ['v' => self::value($i * 7)] + $extra; + $model[$i] = self::value($i * 7); + } + $this->assertSame(count($new), $db->insert('pd', $new)['n']); + $next += count($new); + ksort($model); + } + + /** @test */ + public function everyValueEqualsTheModelThroughWritesACheckpointAndATinyCache(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 1000]] as $config) { + $this->cleanTestDirectory(); + $label = isset($config['shardSize']) ? 'sharded' : 'single'; + $db = $this->db($config); + $model = []; + $rows = []; + for ($i = 0; $i < 2700; $i++) { + $model[$i] = self::value($i); + $rows[] = ['v' => $model[$i]]; + } + $db->insert('pd', $rows); + $next = 2700; + $db->createFieldIndex('pd', 'v'); + $fx = $this->dbPath('pd', 'fx', 'v', isset($config['shardSize']) ? 0 : null); + if (!isset($config['shardSize'])) { + [$header, $map] = $this->readFieldIndexFile($fx); + $this->assertGreaterThan(1, $header['pg']['pages'], 'precondition: several pages'); + $this->assertGreaterThan(4096, 4 * count($map['sshared']), 'precondition: an entry larger than a page'); + } + $this->assertLookups($config, $model, "$label: after createFieldIndex"); + + // A short tail (well under 64 KiB): answered by point lookups, the pages and the newer lines + $covered = $this->readFieldIndexFile($fx)[0]['s']; + $this->write($config, $model, $next, 7, 120, 0); + $this->assertLookups($config, $model, "$label: a short tail"); + $this->assertSame($covered, $this->readFieldIndexFile($fx)[0]['s'], 'precondition: no checkpoint for a short tail'); + + // A long tail: the first reader loads the index and writes a checkpoint + $this->write($config, $model, $next, 8, 300, 200); + $this->assertLookups($config, $model, "$label: a long tail"); + if (!isset($config['shardSize'])) { + $this->assertGreaterThan($covered, $this->readFieldIndexFile($fx)[0]['s'], 'precondition: the reader wrote a checkpoint'); + } + $this->assertLookups($config, $model, "$label: after the checkpoint"); + + // A short tail after the new file + $this->write($config, $model, $next, 9, 120, 0); + $this->assertLookups($config, $model, "$label: a short tail after the checkpoint"); + $this->assertLookupsWith($config + ['cacheMemoryLimit' => 1], $model, "$label: a tiny cache"); + } + } +} diff --git a/tests/Differential/QueryConsistencyTest.php b/tests/Differential/QueryConsistencyTest.php new file mode 100644 index 0000000..a974ae5 --- /dev/null +++ b/tests/Differential/QueryConsistencyTest.php @@ -0,0 +1,221 @@ +db(); + $db->insert('fk', [ + ['id' => 1, 'v' => 'int 1'], ['id' => 1.5, 'v' => 'float 1.5'], ['id' => 1.2, 'v' => 'float 1.2'], + ['id' => true, 'v' => 'true'], ['id' => '007', 'v' => 'string 007'], ['id' => 'Ada', 'v' => 'Ada'], + ['id' => 100000000000000000, 'v' => 'int 1e17'], ['id' => '9223372036854775808', 'v' => 'big string'], + ['id' => -0.0, 'v' => 'minus zero'], + ]); + $db->insert('lk', [ + ['ref' => 1], ['ref' => '1'], ['ref' => 1.0], ['ref' => 1.5], ['ref' => 1.2], ['ref' => true], ['ref' => '1.5'], + ['ref' => 7], ['ref' => '007'], ['ref' => 'Ada'], ['ref' => 1e17], ['ref' => '9223372036854775808'], ['ref' => 0], + ['ref' => null], ['ref' => [1]], + ]); + $joined = array_map(function ($r) { return $r['f']['v'] ?? null; }, $db->query('lk')->join('fk', 'ref', 'id', 'f')->get()); + $this->assertSame([ + 'int 1', 'int 1', 'int 1', // 1, "1" and 1.0 are the same key + 'float 1.5', 'float 1.2', // not truncated to 1 + 'true', // true only with true + null, // "1.5" is not the float 1.5 + null, 'string 007', // "007" is not 7 + 'Ada', 'int 1e17', 'big string', 'minus zero', + null, null, // null and arrays never join + ], $joined); + $this->assertSame($joined, array_map(function ($r) { return $r['f']['v'] ?? null; }, + $db->query('lk')->join('fk', 'ref', 'id', 'f')->limit(100)->get()), 'same through every path'); + } + + /** @test */ + public function noneDbMinAndMaxFollowTheSortOrder(): void + { + // C-4: loose < and > made the result depend on insertion order and PHP version + $db = $this->db(); + foreach ([[5, true], [true, 5], [5, 'abc'], ['abc', 5], [50, '10'], ['10', 50], [null, 3, 'x', [1]]] as $i => $values) { + $name = 'mm' . $i; + $db->insert($name, array_map(function ($v) { return ['v' => $v]; }, $values)); + $this->assertSame($db->query($name)->min('v'), $db->min($name, 'v'), json_encode($values)); + $this->assertSame($db->query($name)->max('v'), $db->max($name, 'v'), json_encode($values)); + $sorted = array_column($db->sort($db->find($name, 0), 'v'), 'v'); + $present = array_values(array_filter($sorted, function ($v) { return $v !== null; })); + $this->assertSame($present[0], $db->min($name, 'v')); + $this->assertSame(end($present), $db->max($name, 'v')); + } + $this->assertSame('10', $db->max('mm4', 'v'), 'numeric strings sort after numbers'); + } + + /** @test */ + public function likeAndSearchFoldUnicodeCase(): void + { + // C-5/C-11: PCRE /iu missed İ↔i and SS↔ß, mb_strtolower missed ſ/s, ς/σ, SS/ß + $db = $this->db(); + $db->insert('uc', [['n' => 'İzmir'], ['n' => 'STRASSE'], ['n' => 'ſtar'], ['n' => 'ὈΔΥΣΣΕΎΣ'], ['n' => 'Ğüneş'], ['n' => 'plain']]); + $cases = [['izmir', 'İzmir'], ['IZMIR', 'İzmir'], ['straße', 'STRASSE'], ['star', 'ſtar'], ['ὀδυσσεύς', 'ὈΔΥΣΣΕΎΣ'], ['ğÜNEŞ', 'Ğüneş']]; + foreach ($cases as [$pattern, $expected]) { + $this->assertSame([$expected], array_column($db->like('uc', 'n', $pattern), 'n'), "like $pattern"); + $this->assertSame([$expected], array_column($db->query('uc')->like('n', $pattern)->get(), 'n'), "builder like $pattern"); + $this->assertSame([$expected], array_column($db->query('uc')->where(['n' => ['$like' => $pattern]])->get(), 'n'), "\$like $pattern"); + $this->assertSame([$expected], array_column($db->query('uc')->search($pattern)->get(), 'n'), "search $pattern"); + } + $this->assertSame(['İzmir'], array_column($db->like('uc', 'n', '^izm'), 'n'), 'anchors keep their meaning'); + $this->assertSame([], $db->like('uc', 'n', 'Izmirx')); + $this->assertSame(['plain'], array_column($db->query('uc')->notLike('n', 'x')->where(['n' => ['$like' => 'pl']])->get(), 'n')); + $this->assertSame([], $db->query('uc')->notLike('n', 'A')->where(['n' => ['$like' => 'pl']])->get()); + } + + /** + * @test + * @group no-mbstring + */ + public function likeWithoutMbstringStillMatchesCaseInsensitively(): void + { + // C-5: without mbstring like() keeps the PCRE /iu path (better than stripos for Ğ/ğ) + $hook = new \noneDB\Tests\InternalValue('mbstringFn'); + $hook->setValue(null, function () { return false; }); + try { + $db = $this->db(); + $db->insert('nomb', [['n' => 'Ğüneş'], ['n' => 'ABC']]); + $this->assertSame(['Ğüneş'], array_column($db->like('nomb', 'n', 'ğüneş'), 'n')); + $this->assertSame(['Ğüneş'], array_column($db->query('nomb')->like('n', 'ĞÜNEŞ')->get(), 'n')); + $this->assertSame(['ABC'], array_column($db->query('nomb')->search('abc')->get(), 'n')); + // QUERY.md: without mbstring like() misses ß/ss and İ/i, search() folds only ASCII letters + $db->insert('nomb', [['n' => 'STRASSE'], ['n' => 'İzmir']]); + $this->assertSame([], $db->like('nomb', 'n', 'straße')); + $this->assertSame([], $db->like('nomb', 'n', 'izmir')); + $this->assertSame([], $db->query('nomb')->search('ğüneş')->get()); + $this->assertSame(['STRASSE'], array_column($db->query('nomb')->search('strasse')->get(), 'n')); + } finally { + $hook->setValue(null, null); + } + } + + /** @test */ + public function builderKeyFilterIsTheSameInEveryClause(): void + { + // C-6: only a single plain where() went through find(); elsewhere "1" didn't match key 1 + $db = $this->db(); + $db->insert('kf', [['n' => 'a'], ['n' => 'b'], ['n' => 'c']]); + $this->assertSame([1], array_column($db->query('kf')->where(['key' => '1'])->get(), 'key')); + $this->assertSame([1], array_column($db->query('kf')->where(['key' => '1'])->where(['n' => 'b'])->get(), 'key')); + $this->assertSame([0, 2], array_column($db->query('kf')->where(['n' => 'a'])->orWhere(['key' => ['2']])->get(), 'key')); + $this->assertSame([0, 1], array_column($db->query('kf')->where(['key' => [0, '1']])->where(['n' => ['$ne' => 'z']])->get(), 'key')); + $this->assertSame([0, 2], array_column($db->query('kf')->whereNot(['key' => '1'])->get(), 'key')); + $this->assertSame([], $db->query('kf')->where(['key' => "1\n"])->get(), 'C-12: "1\n" is not key 1'); + $this->assertSame([], $db->find('kf', ['key' => "1\n"]), 'C-12'); + } + + /** @test */ + public function smallInconsistencies(): void + { + // C-8 + $db = $this->db(); + $db->insert('sm', [['p' => 2, 'n' => 'a', 'flag' => true], ['p' => 3, 'n' => 'b', 'flag' => false], ['p' => 4, 'n' => 'a1']]); + + // count() with filters it can't use: 0 (TypeError before) + $this->assertSame(0, $db->count('sm', 'x')); + $this->assertSame(0, $db->count('sm', 1.5)); + $this->assertSame(0, $db->count('sm', false)); + $this->assertSame(3, $db->count('sm', null)); + // noneDB::like() ignores booleans (true matched "1") and refuses non-string patterns + $this->assertSame([], $db->like('sm', 'flag', '1')); + $this->assertFalse($db->like('sm', 'n', ['a'])); + // limit(0) means nothing + $this->assertNull($db->query('sm')->limit(0)->first()); + $this->assertFalse($db->query('sm')->limit(0)->exists()); + // $regex: an escaped backslash before "~" stays a backslash + $db->insert('rx', [['s' => 'a\\~b'], ['s' => 'a~b']]); + $this->assertSame(['a\\~b'], array_column($db->query('rx')->where(['s' => ['$regex' => 'a\\\\~b']])->get(), 's')); + $this->assertSame(['a~b'], array_column($db->query('rx')->where(['s' => ['$regex' => '^a~b$']])->get(), 's')); + // Aggregates read the fields, not the projection + $this->assertSame(9.0, $db->query('sm')->select(['n'])->sum('p')); + $this->assertSame(3.0, $db->query('sm')->select(['n'])->avg('p')); + $this->assertSame(2, $db->query('sm')->select(['n'])->min('p')); + $this->assertSame(4, $db->query('sm')->except(['p'])->max('p')); + $this->assertSame([2, 3, 4], $db->query('sm')->select(['n'])->distinct('p')); + } + + /** @test */ + public function searchDoesNotLookIntoTheAddedDistance(): void + { + // C-8: without a field list search() also searched _distance added by a spatial filter + $db = $this->db(); + $db->insert('sd', [['name' => 'one', 'loc' => ['type' => 'Point', 'coordinates' => [0, 0]]], + ['name' => '1234', 'loc' => ['type' => 'Point', 'coordinates' => [10, 10]]]]); + $rows = $db->query('sd')->withinDistance('loc', 0.0, 0.0, 100000)->withDistance('loc', 0.0, 0.0)->search('0')->get(); + $this->assertSame([], $rows, '"0" is only in the computed _distance (0.0)'); + } + + /** @test */ + public function groupKeysAreExactValues(): void + { + // C-9: json_encode() keys depend on serialize_precision; -0.0 and 0.0 were two groups + $before = ini_get('serialize_precision'); + try { + $db = $this->db(); + $db->insert('gk', [['v' => 28.980345], ['v' => 28.98], ['v' => -0.0], ['v' => 0.0], ['v' => 0], ['v' => '0']]); + ini_set('serialize_precision', '5'); + $groups = $db->query('gk')->groupBy('v')->get(); + $this->assertSame([28.980345, 28.98, -0.0, 0, '0'], array_column($groups, '_group')); + $this->assertSame([1, 1, 2, 1, 1], array_column($groups, '_count')); + } finally { + ini_set('serialize_precision', (string)$before); + } + } + + /** @test */ + public function groupByWithSelectProjectsTheItems(): void + { + // C-13: the projection was applied to the group rows: every group became [] + $db = $this->db(); + $db->insert('gs', [['c' => 'x', 'name' => 'a', 'p' => 1], ['c' => 'y', 'name' => 'b', 'p' => 2], ['c' => 'x', 'name' => 'c', 'p' => 3]]); + $groups = $db->query('gs')->groupBy('c')->select(['name'])->get(); + $this->assertSame(['x', 'y'], array_column($groups, '_group')); + $this->assertSame([2, 1], array_column($groups, '_count')); + $this->assertSame([['name' => 'a', 'key' => 0], ['name' => 'c', 'key' => 2]], $groups[0]['_items']); + } + + /** @test */ + public function aggregateMethodsTreatFiltersLikeFind(): void + { + // T2-5: one rule for the filter parameter of every method that has one + $db = $this->db(); + $db->insert('ag', [['c' => 'x', 'p' => 1], ['c' => 'y', 'p' => 2], ['c' => 'x', 'p' => 3]]); + foreach ([0, null, [], ['c' => 'x']] as $filter) { + $records = $db->find('ag', $filter); + $values = array_column($records, 'p'); + $this->assertSame((float)array_sum($values), (float)$db->sum('ag', 'p', $filter), json_encode($filter)); + $this->assertSame(min($values), $db->min('ag', 'p', $filter), json_encode($filter)); + $this->assertSame(max($values), $db->max('ag', 'p', $filter), json_encode($filter)); + $this->assertSame($records[0], $db->first('ag', $filter), json_encode($filter)); + $this->assertSame(end($records), $db->last('ag', $filter), json_encode($filter)); + $this->assertTrue($db->exists('ag', $filter === 0 || $filter === null ? [] : $filter), json_encode($filter)); + } + foreach (['x', 1.5, false] as $invalid) { + $this->assertFalse($db->find('ag', $invalid)); + $this->assertSame(0, $db->count('ag', $invalid)); + foreach (['sum' => $db->sum('ag', 'p', $invalid), 'avg' => $db->avg('ag', 'p', $invalid)] as $method => $zero) { + $this->assertTrue($zero === 0 || $zero === 0.0, "$method with an invalid filter: the number 0, got " . var_export($zero, true)); + } + $this->assertFalse($db->exists('ag', $invalid)); + $this->assertFalse($db->between('ag', 'p', 0, 5, $invalid)); + $this->assertNull($db->min('ag', 'p', $invalid)); + $this->assertNull($db->max('ag', 'p', $invalid)); + $this->assertNull($db->first('ag', $invalid)); + $this->assertNull($db->last('ag', $invalid)); + } + } +} diff --git a/tests/Docs/doctest.php b/tests/Docs/doctest.php new file mode 100644 index 0000000..fc9e7c7 --- /dev/null +++ b/tests/Docs/doctest.php @@ -0,0 +1,107 @@ + is not +// run (a signature, a fragment, an example of a file). A block fails on an exception, an error, a warning, a +// notice or a deprecation. +// usage: php tests/Docs/doctest.php [--library=noneDB.php] [document.md ...] (default: the documents below) +error_reporting(E_ALL); +$root = dirname(__DIR__, 2); +$options = getopt('', ['library:'], $rest); +$library = realpath($options['library'] ?? $root . '/noneDB.php'); +$documents = array_slice($argv, $rest); +if (empty($documents)) { + $documents = array_merge([$root . '/README.md'], glob($root . '/docs/*.md')); +} +if ($library === false) { + fwrite(STDERR, "no library\n"); + exit(2); +} + +/** The runnable blocks of a document: [line, code] */ +function blocks(string $markdown): array +{ + $blocks = []; + $lines = explode("\n", $markdown); + $skip = false; + for ($i = 0, $n = count($lines); $i < $n; $i++) { + $line = rtrim($lines[$i]); + if ($line === '') { + $skip = true; + continue; + } + if (preg_match('/^(\s*)```(\w*)/', $line, $m) !== 1) { + if (trim($line) !== '') { + $skip = false; + } + continue; + } + $start = $i + 1; + $code = []; + for ($i++; $i < $n && preg_match('/^\s*```\s*$/', $lines[$i]) !== 1; $i++) { + $code[] = substr($lines[$i], strlen($m[1])); + } + if ($m[2] === 'php' && !$skip) { + $blocks[] = [$start, implode("\n", $code)]; + } + $skip = false; + } + return $blocks; +} + +function removeTree(string $path): void +{ + if (is_dir($path) && !is_link($path)) { + foreach (array_diff(scandir($path), ['.', '..']) as $entry) { + removeTree("$path/$entry"); + } + rmdir($path); + } else { + unlink($path); + } +} + +$failed = 0; +$ran = 0; +foreach ($documents as $document) { + $name = substr(realpath($document), strlen($root) + 1); + $blocks = blocks((string)file_get_contents($document)); + if (empty($blocks)) { + continue; + } + $dir = sys_get_temp_dir() . '/nonedb_doctest_' . bin2hex(random_bytes(6)); + mkdir($dir, 0700); + copy($library, "$dir/noneDB.php"); + file_put_contents("$dir/.nonedb", json_encode(['secretKey' => bin2hex(random_bytes(16)), 'dbDir' => './db/'])); + // Warnings, notices and deprecations fail the block; what the library suppresses with @ does not + $script = " [$line, $code]) { + $code = preg_replace('/^\s*<\?php\s*/', '', $code); + // The documents include the library as an application does: here it is loaded once + $code = preg_replace('/^\s*(include|require)(_once)?\s*\(?\s*([\'"])noneDB\.php\3\s*\)?\s*;/m', '', $code); + $script .= "fwrite(STDERR, \"\\n@block $line\\n\");\n$code\n"; + } + $script .= "fwrite(STDERR, \"\\n@done\\n\");\n"; + file_put_contents("$dir/doc.php", $script); + // NONEDB_SECRET: the documents read keys from the environment, as applications should + $env = getenv() + ['NONEDB_SECRET' => bin2hex(random_bytes(16))]; + $process = proc_open([PHP_BINARY, '-d', 'display_errors=stderr', "$dir/doc.php"], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $dir, $env); + stream_get_contents($pipes[1]); + $errors = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($process); + $ran += count($blocks); + if ($status !== 0 || strpos($errors, "\n@done\n") === false) { + $failed++; + $at = preg_match_all('/@block (\d+)/', $errors, $m) ? end($m[1]) : '?'; + $detail = trim(substr($errors, strrpos($errors, "@block $at") + strlen("@block $at"))); + echo "FAIL $name:$at\n" . preg_replace('/^/m', ' ', $detail) . "\n"; + } else { + echo "ok $name (" . count($blocks) . " blocks)\n"; + } + removeTree($dir); +} +echo "$ran blocks, " . ($failed === 0 ? 'all documents passed' : "$failed documents failed") . "\n"; +exit($failed === 0 ? 0 : 1); diff --git a/tests/Feature/AggregationTest.php b/tests/Feature/AggregationTest.php deleted file mode 100644 index 96fbe95..0000000 --- a/tests/Feature/AggregationTest.php +++ /dev/null @@ -1,326 +0,0 @@ - 'Apple', 'price' => 10, 'quantity' => 100, 'category' => 'fruit'], - ['product' => 'Banana', 'price' => 5, 'quantity' => 150, 'category' => 'fruit'], - ['product' => 'Carrot', 'price' => 3, 'quantity' => 200, 'category' => 'vegetable'], - ['product' => 'Orange', 'price' => 8, 'quantity' => 80, 'category' => 'fruit'], - ['product' => 'Potato', 'price' => 2, 'quantity' => 300, 'category' => 'vegetable'], - ]; - - $this->noneDB->insert($this->testDbName, $data); - } - - // ========================================== - // SUM TESTS - // ========================================== - - /** - * @test - */ - public function sumCalculatesTotal(): void - { - $result = $this->noneDB->sum($this->testDbName, 'price'); - - $this->assertEquals(28, $result); // 10+5+3+8+2 - } - - /** - * @test - */ - public function sumWithFilter(): void - { - $result = $this->noneDB->sum($this->testDbName, 'price', ['category' => 'fruit']); - - $this->assertEquals(23, $result); // 10+5+8 - } - - /** - * @test - */ - public function sumReturnsZeroForEmptyDB(): void - { - $this->noneDB->createDB('emptydb'); - $result = $this->noneDB->sum('emptydb', 'price'); - - $this->assertEquals(0, $result); - } - - /** - * @test - */ - public function sumReturnsZeroForNonExistentField(): void - { - $result = $this->noneDB->sum($this->testDbName, 'nonexistent'); - - $this->assertEquals(0, $result); - } - - /** - * @test - */ - public function sumIgnoresNonNumericValues(): void - { - $this->noneDB->insert($this->testDbName, [ - ['product' => 'Test', 'price' => 'free', 'category' => 'other'], - ]); - - $result = $this->noneDB->sum($this->testDbName, 'price'); - - $this->assertEquals(28, $result); // Ignores 'free' - } - - /** - * @test - */ - public function sumWorksWithFloats(): void - { - $this->noneDB->insert($this->testDbName, [ - ['product' => 'Test', 'price' => 1.5, 'category' => 'other'], - ]); - - $result = $this->noneDB->sum($this->testDbName, 'price'); - - $this->assertEquals(29.5, $result); - } - - // ========================================== - // AVG TESTS - // ========================================== - - /** - * @test - */ - public function avgCalculatesAverage(): void - { - $result = $this->noneDB->avg($this->testDbName, 'price'); - - $this->assertEquals(5.6, $result); // 28/5 - } - - /** - * @test - */ - public function avgWithFilter(): void - { - $result = $this->noneDB->avg($this->testDbName, 'price', ['category' => 'vegetable']); - - $this->assertEquals(2.5, $result); // (3+2)/2 - } - - /** - * @test - */ - public function avgReturnsZeroForEmptyDB(): void - { - $this->noneDB->createDB('emptydb'); - $result = $this->noneDB->avg('emptydb', 'price'); - - $this->assertEquals(0, $result); - } - - /** - * @test - */ - public function avgIgnoresNonNumericValues(): void - { - $this->noneDB->insert($this->testDbName, [ - ['product' => 'Test', 'price' => 'free', 'category' => 'other'], - ]); - - $result = $this->noneDB->avg($this->testDbName, 'price'); - - // Should still be 28/5 = 5.6, ignoring 'free' - $this->assertEquals(5.6, $result); - } - - // ========================================== - // MIN TESTS - // ========================================== - - /** - * @test - */ - public function minFindsMinimum(): void - { - $result = $this->noneDB->min($this->testDbName, 'price'); - - $this->assertEquals(2, $result); - } - - /** - * @test - */ - public function minWithFilter(): void - { - $result = $this->noneDB->min($this->testDbName, 'price', ['category' => 'fruit']); - - $this->assertEquals(5, $result); - } - - /** - * @test - */ - public function minReturnsNullForEmptyDB(): void - { - $this->noneDB->createDB('emptydb'); - $result = $this->noneDB->min('emptydb', 'price'); - - $this->assertNull($result); - } - - /** - * @test - */ - public function minWorksWithStrings(): void - { - $result = $this->noneDB->min($this->testDbName, 'product'); - - $this->assertEquals('Apple', $result); - } - - /** - * @test - */ - public function minReturnsNullForNonExistentField(): void - { - $result = $this->noneDB->min($this->testDbName, 'nonexistent'); - - $this->assertNull($result); - } - - // ========================================== - // MAX TESTS - // ========================================== - - /** - * @test - */ - public function maxFindsMaximum(): void - { - $result = $this->noneDB->max($this->testDbName, 'price'); - - $this->assertEquals(10, $result); - } - - /** - * @test - */ - public function maxWithFilter(): void - { - $result = $this->noneDB->max($this->testDbName, 'price', ['category' => 'vegetable']); - - $this->assertEquals(3, $result); - } - - /** - * @test - */ - public function maxReturnsNullForEmptyDB(): void - { - $this->noneDB->createDB('emptydb'); - $result = $this->noneDB->max('emptydb', 'price'); - - $this->assertNull($result); - } - - /** - * @test - */ - public function maxWorksWithStrings(): void - { - $result = $this->noneDB->max($this->testDbName, 'product'); - - $this->assertEquals('Potato', $result); - } - - /** - * @test - */ - public function maxReturnsNullForNonExistentField(): void - { - $result = $this->noneDB->max($this->testDbName, 'nonexistent'); - - $this->assertNull($result); - } - - // ========================================== - // COUNT TESTS - // ========================================== - - /** - * @test - */ - public function countReturnsTotal(): void - { - $result = $this->noneDB->count($this->testDbName); - - $this->assertEquals(5, $result); - } - - /** - * @test - */ - public function countWithFilter(): void - { - $result = $this->noneDB->count($this->testDbName, ['category' => 'fruit']); - - $this->assertEquals(3, $result); - } - - /** - * @test - */ - public function countReturnsZeroForEmptyDB(): void - { - $this->noneDB->createDB('emptydb'); - $result = $this->noneDB->count('emptydb'); - - $this->assertEquals(0, $result); - } - - /** - * @test - */ - public function countReturnsZeroForNoMatch(): void - { - $result = $this->noneDB->count($this->testDbName, ['category' => 'nonexistent']); - - $this->assertEquals(0, $result); - } - - /** - * @test - */ - public function countAfterDelete(): void - { - $this->noneDB->delete($this->testDbName, ['category' => 'vegetable']); - $result = $this->noneDB->count($this->testDbName); - - $this->assertEquals(3, $result); - } - - /** - * @test - */ - public function countWithMultipleFilters(): void - { - $result = $this->noneDB->count($this->testDbName, ['category' => 'fruit', 'price' => 10]); - - $this->assertEquals(1, $result); - } -} diff --git a/tests/Feature/ChainingTest.php b/tests/Feature/ChainingTest.php deleted file mode 100644 index 3e8ea0c..0000000 --- a/tests/Feature/ChainingTest.php +++ /dev/null @@ -1,558 +0,0 @@ -noneDB->insert($this->testDbName, [ - ['name' => 'Alice', 'age' => 25, 'score' => 85, 'city' => 'Istanbul', 'email' => 'alice@gmail.com', 'active' => true], - ['name' => 'Bob', 'age' => 30, 'score' => 90, 'city' => 'Ankara', 'email' => 'bob@yahoo.com', 'active' => true], - ['name' => 'Charlie', 'age' => 35, 'score' => 75, 'city' => 'Istanbul', 'email' => 'charlie@gmail.com', 'active' => false], - ['name' => 'David', 'age' => 28, 'score' => 95, 'city' => 'Izmir', 'email' => 'david@hotmail.com', 'active' => true], - ['name' => 'Eve', 'age' => 22, 'score' => 80, 'city' => 'Istanbul', 'email' => 'eve@gmail.com', 'active' => true], - ]); - } - - // ========================================== - // QUERY BUILDER CREATION - // ========================================== - - /** - * @test - */ - public function queryReturnsQueryBuilder(): void - { - $query = $this->noneDB->query($this->testDbName); - - $this->assertInstanceOf(\noneDBQuery::class, $query); - } - - // ========================================== - // CHAINABLE METHODS - // ========================================== - - /** - * @test - */ - public function whereFiltersResults(): void - { - $results = $this->noneDB->query($this->testDbName) - ->where(['city' => 'Istanbul']) - ->get(); - - $this->assertCount(3, $results); - foreach ($results as $record) { - $this->assertEquals('Istanbul', $record['city']); - } - } - - /** - * @test - */ - public function whereWithMultipleFilters(): void - { - $results = $this->noneDB->query($this->testDbName) - ->where(['city' => 'Istanbul', 'active' => true]) - ->get(); - - $this->assertCount(2, $results); - } - - /** - * @test - */ - public function likeFiltersWithContains(): void - { - $results = $this->noneDB->query($this->testDbName) - ->like('email', 'gmail') - ->get(); - - $this->assertCount(3, $results); - } - - /** - * @test - */ - public function likeFiltersWithStartsWith(): void - { - $results = $this->noneDB->query($this->testDbName) - ->like('name', '^A') - ->get(); - - $this->assertCount(1, $results); - $this->assertEquals('Alice', $results[0]['name']); - } - - /** - * @test - */ - public function likeFiltersWithEndsWith(): void - { - $results = $this->noneDB->query($this->testDbName) - ->like('email', 'gmail.com$') - ->get(); - - $this->assertCount(3, $results); - } - - /** - * @test - */ - public function betweenFiltersRange(): void - { - $results = $this->noneDB->query($this->testDbName) - ->between('age', 25, 30) - ->get(); - - $this->assertCount(3, $results); - foreach ($results as $record) { - $this->assertGreaterThanOrEqual(25, $record['age']); - $this->assertLessThanOrEqual(30, $record['age']); - } - } - - /** - * @test - */ - public function sortAscending(): void - { - $results = $this->noneDB->query($this->testDbName) - ->sort('age', 'asc') - ->get(); - - $this->assertEquals(22, $results[0]['age']); - $this->assertEquals(35, $results[4]['age']); - } - - /** - * @test - */ - public function sortDescending(): void - { - $results = $this->noneDB->query($this->testDbName) - ->sort('score', 'desc') - ->get(); - - $this->assertEquals(95, $results[0]['score']); - $this->assertEquals(75, $results[4]['score']); - } - - /** - * @test - */ - public function limitResults(): void - { - $results = $this->noneDB->query($this->testDbName) - ->limit(3) - ->get(); - - $this->assertCount(3, $results); - } - - /** - * @test - */ - public function offsetResults(): void - { - $results = $this->noneDB->query($this->testDbName) - ->offset(2) - ->get(); - - $this->assertCount(3, $results); - } - - /** - * @test - */ - public function limitWithOffset(): void - { - $results = $this->noneDB->query($this->testDbName) - ->offset(1) - ->limit(2) - ->get(); - - $this->assertCount(2, $results); - } - - /** - * @test - */ - public function multipleChaining(): void - { - $results = $this->noneDB->query($this->testDbName) - ->where(['active' => true]) - ->between('age', 20, 30) - ->sort('score', 'desc') - ->limit(2) - ->get(); - - $this->assertCount(2, $results); - $this->assertEquals('David', $results[0]['name']); // highest score - } - - // ========================================== - // TERMINAL METHODS - // ========================================== - - /** - * @test - */ - public function getReturnsArray(): void - { - $results = $this->noneDB->query($this->testDbName)->get(); - - $this->assertIsArray($results); - $this->assertCount(5, $results); - } - - /** - * @test - */ - public function firstReturnsFirstRecord(): void - { - $result = $this->noneDB->query($this->testDbName)->first(); - - $this->assertIsArray($result); - $this->assertEquals('Alice', $result['name']); - } - - /** - * @test - */ - public function firstWithFilterReturnsFirstMatch(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['city' => 'Istanbul']) - ->first(); - - $this->assertEquals('Alice', $result['name']); - } - - /** - * @test - */ - public function firstReturnsNullForNoMatch(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['city' => 'NonExistent']) - ->first(); - - $this->assertNull($result); - } - - /** - * @test - */ - public function lastReturnsLastRecord(): void - { - $result = $this->noneDB->query($this->testDbName)->last(); - - $this->assertIsArray($result); - $this->assertEquals('Eve', $result['name']); - } - - /** - * @test - */ - public function lastWithSortReturnsCorrectRecord(): void - { - $result = $this->noneDB->query($this->testDbName) - ->sort('age', 'asc') - ->last(); - - $this->assertEquals('Charlie', $result['name']); // oldest - } - - /** - * @test - */ - public function countReturnsInteger(): void - { - $count = $this->noneDB->query($this->testDbName)->count(); - - $this->assertIsInt($count); - $this->assertEquals(5, $count); - } - - /** - * @test - */ - public function countWithFilter(): void - { - $count = $this->noneDB->query($this->testDbName) - ->where(['active' => true]) - ->count(); - - $this->assertEquals(4, $count); - } - - /** - * @test - */ - public function existsReturnsTrueWhenFound(): void - { - $exists = $this->noneDB->query($this->testDbName) - ->where(['name' => 'Alice']) - ->exists(); - - $this->assertTrue($exists); - } - - /** - * @test - */ - public function existsReturnsFalseWhenNotFound(): void - { - $exists = $this->noneDB->query($this->testDbName) - ->where(['name' => 'NonExistent']) - ->exists(); - - $this->assertFalse($exists); - } - - // ========================================== - // AGGREGATION METHODS - // ========================================== - - /** - * @test - */ - public function sumCalculatesTotal(): void - { - $sum = $this->noneDB->query($this->testDbName)->sum('score'); - - $this->assertEquals(425, $sum); // 85+90+75+95+80 - } - - /** - * @test - */ - public function sumWithFilter(): void - { - $sum = $this->noneDB->query($this->testDbName) - ->where(['city' => 'Istanbul']) - ->sum('score'); - - $this->assertEquals(240, $sum); // 85+75+80 - } - - /** - * @test - */ - public function avgCalculatesAverage(): void - { - $avg = $this->noneDB->query($this->testDbName)->avg('score'); - - $this->assertEquals(85, $avg); // 425/5 - } - - /** - * @test - */ - public function avgWithFilter(): void - { - $avg = $this->noneDB->query($this->testDbName) - ->where(['active' => true]) - ->avg('age'); - - $this->assertEquals(26.25, $avg); // (25+30+28+22)/4 - } - - /** - * @test - */ - public function minReturnsMinimum(): void - { - $min = $this->noneDB->query($this->testDbName)->min('age'); - - $this->assertEquals(22, $min); - } - - /** - * @test - */ - public function maxReturnsMaximum(): void - { - $max = $this->noneDB->query($this->testDbName)->max('score'); - - $this->assertEquals(95, $max); - } - - /** - * @test - */ - public function distinctReturnsUniqueValues(): void - { - $cities = $this->noneDB->query($this->testDbName)->distinct('city'); - - $this->assertCount(3, $cities); - $this->assertContains('Istanbul', $cities); - $this->assertContains('Ankara', $cities); - $this->assertContains('Izmir', $cities); - } - - /** - * @test - */ - public function distinctWithFilter(): void - { - $cities = $this->noneDB->query($this->testDbName) - ->where(['active' => true]) - ->distinct('city'); - - $this->assertCount(3, $cities); - } - - // ========================================== - // WRITE METHODS - // ========================================== - - /** - * @test - */ - public function updateWithChaining(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['city' => 'Istanbul']) - ->update(['verified' => true]); - - $this->assertEquals(3, $result['n']); - - // Verify update - $istanbul = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); - foreach ($istanbul as $record) { - $this->assertTrue($record['verified']); - } - } - - /** - * @test - */ - public function updateWithMultipleFilters(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['active' => true]) - ->between('age', 25, 30) - ->update(['tier' => 'gold']); - - $this->assertEquals(3, $result['n']); - } - - /** - * @test - */ - public function updateReturnsZeroForNoMatch(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['city' => 'NonExistent']) - ->update(['verified' => true]); - - $this->assertEquals(0, $result['n']); - } - - /** - * @test - */ - public function deleteWithChaining(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['active' => false]) - ->delete(); - - $this->assertEquals(1, $result['n']); - $this->assertEquals(4, $this->noneDB->count($this->testDbName)); - } - - /** - * @test - */ - public function deleteWithLikeFilter(): void - { - $result = $this->noneDB->query($this->testDbName) - ->like('email', 'yahoo') - ->delete(); - - $this->assertEquals(1, $result['n']); - $this->assertEquals(4, $this->noneDB->count($this->testDbName)); - } - - /** - * @test - */ - public function deleteReturnsZeroForNoMatch(): void - { - $result = $this->noneDB->query($this->testDbName) - ->where(['city' => 'NonExistent']) - ->delete(); - - $this->assertEquals(0, $result['n']); - $this->assertEquals(5, $this->noneDB->count($this->testDbName)); - } - - // ========================================== - // EDGE CASES - // ========================================== - - /** - * @test - */ - public function emptyDatabaseReturnsEmptyArray(): void - { - $this->noneDB->createDB('emptydb'); - $results = $this->noneDB->query('emptydb')->get(); - - $this->assertIsArray($results); - $this->assertEmpty($results); - } - - /** - * @test - */ - public function chainMethodsReturnSelf(): void - { - $query = $this->noneDB->query($this->testDbName); - - $this->assertSame($query, $query->where(['active' => true])); - $this->assertSame($query, $query->like('name', 'A')); - $this->assertSame($query, $query->between('age', 20, 40)); - $this->assertSame($query, $query->sort('name')); - $this->assertSame($query, $query->limit(10)); - $this->assertSame($query, $query->offset(5)); - } - - /** - * @test - */ - public function paginationExample(): void - { - $pageSize = 2; - - // Page 1 - $page1 = $this->noneDB->query($this->testDbName) - ->sort('name') - ->limit($pageSize) - ->offset(0) - ->get(); - - // Page 2 - $page2 = $this->noneDB->query($this->testDbName) - ->sort('name') - ->limit($pageSize) - ->offset(2) - ->get(); - - $this->assertCount(2, $page1); - $this->assertCount(2, $page2); - $this->assertNotEquals($page1[0]['name'], $page2[0]['name']); - } -} diff --git a/tests/Feature/CheckDBTest.php b/tests/Feature/CheckDBTest.php deleted file mode 100644 index 6e22cd2..0000000 --- a/tests/Feature/CheckDBTest.php +++ /dev/null @@ -1,192 +0,0 @@ -noneDB->checkDB(null); - - $this->assertFalse($result); - } - - /** - * @test - */ - public function checkDBReturnsFalseForEmptyString(): void - { - $result = $this->noneDB->checkDB(''); - - $this->assertFalse($result); - } - - /** - * @test - */ - public function checkDBReturnsFalseForOnlySpecialChars(): void - { - // v3.2.0: invalid names are rejected instead of being silently stripped - $this->expectException(\InvalidArgumentException::class); - $this->noneDB->checkDB('@#$%^&*()'); - } - - /** - * @test - */ - public function checkDBReturnsTrueForExistingDB(): void - { - $this->noneDB->createDB('existing'); - - $result = $this->noneDB->checkDB('existing'); - - $this->assertTrue($result); - } - - /** - * @test - */ - public function checkDBAutoCreatesWhenEnabled(): void - { - // autoCreateDB is true by default - $result = $this->noneDB->checkDB('autotest'); - - $this->assertTrue($result); - $this->assertDatabaseExists('autotest'); - } - - /** - * @test - */ - public function checkDBReturnsFalseWhenAutoCreateDisabled(): void - { - $this->setPrivateProperty('autoCreateDB', false); - - $result = $this->noneDB->checkDB('nonexistent'); - - $this->assertFalse($result); - $this->assertDatabaseNotExists('nonexistent'); - } - - /** - * @test - */ - public function checkDBSanitizesDBName(): void - { - // v3.2.0: names with characters outside [A-Za-z0-9' -] are rejected, nothing is created - try { - $this->noneDB->checkDB('test', $result[0]['content']); + $this->assertSame([$xssData + ['key' => 0]], $result); } /** @@ -68,6 +68,7 @@ public function xssAttemptInData(): void */ public function pathTraversalAttemptInDBName(): void { + $before = scandir($this->testDbDir); try { $this->noneDB->createDB('../../../etc/passwd'); $this->fail('Expected InvalidArgumentException'); @@ -77,7 +78,8 @@ public function pathTraversalAttemptInDBName(): void // Nothing created outside (or inside) the db directory $this->assertFileDoesNotExist('/etc/passwd.json'); - $this->assertSame([], glob($this->testDbDir . '*passwd*')); + clearstatcache(); + $this->assertSame($before, scandir($this->testDbDir)); } /** @@ -98,7 +100,7 @@ public function nullByteInjectionAttempt(): void */ public function unicodeInDatabaseName(): void { - // v3.2.0: non-ASCII names are rejected instead of being stripped (stripping let names collide) + // Non-ASCII names are rejected, never stripped (stripping would let names collide) try { $this->noneDB->createDB('veritabanı_测试_データ'); $this->fail('Expected InvalidArgumentException'); @@ -106,7 +108,7 @@ public function unicodeInDatabaseName(): void $this->assertStringContainsString('invalid database name', $e->getMessage()); } - // 'çalışan' used to be stripped to 'alan' and write into that database; now it is rejected + // Stripped, 'çalışan' would become 'alan' and write into that database: it is rejected $this->noneDB->insert('alan', ['tip' => 'alan']); try { $this->noneDB->insert('çalışan', ['tip' => 'çalışan']); @@ -136,9 +138,7 @@ public function unicodeInData(): void $this->noneDB->insert($dbName, $unicodeData); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals('Türkçe karakterler: ğüşıöç', $result[0]['turkish']); - $this->assertEquals('中文字符', $result[0]['chinese']); - $this->assertEquals('😀🎉🚀💻', $result[0]['emoji']); + $this->assertSame([$unicodeData + ['key' => 0]], $result, 'every string byte for byte'); } /** @@ -159,8 +159,8 @@ public function specialCharactersInFieldNames(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals('value1', $result[0]['field-with-dash']); - $this->assertEquals('value4', $result[0]['field with space']); + $this->assertSame([$data + ['key' => 0]], $result); + $this->assertSame([0], array_column($this->noneDB->find($dbName, ['field.with.dot' => 'value2']), 'key'), 'a dot is part of the name, not a path'); } // ========================================== @@ -179,6 +179,9 @@ public function reasonablyLongDatabaseNameWorks(): void $result = $this->noneDB->createDB($longName); $this->assertTrue($result); + $this->assertFalse($this->noneDB->createDB($longName), 'docs/API.md: false when it already exists'); + $this->assertSame(['n' => 1], $this->noneDB->insert($longName, ['v' => 1])); + $this->assertSame([['v' => 1, 'key' => 0]], $this->noneDB->find($longName, 0)); } /** @@ -194,7 +197,7 @@ public function veryLongFieldName(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals('value', $result[0][$longFieldName]); + $this->assertSame([[$longFieldName => 'value', 'key' => 0]], $result); } /** @@ -210,7 +213,7 @@ public function veryLongStringValue(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals(1000000, strlen($result[0]['content'])); + $this->assertSame($longValue, $result[0]['content']); } /** @@ -229,8 +232,7 @@ public function deeplyNestedData(): void $this->noneDB->insert($dbName, $nested); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals(1, $result[0]['level']); - $this->assertEquals(2, $result[0]['child']['level']); + $this->assertSame([$nested + ['key' => 0]], $result, 'all ten levels'); } /** @@ -238,7 +240,7 @@ public function deeplyNestedData(): void */ public function emptyStringDatabaseName(): void { - // v3.2.0: an empty name is invalid; checkDB() reports false, other operations throw + // An empty name is invalid; checkDB() reports false, other operations throw $this->assertFalse($this->noneDB->checkDB('')); $this->expectException(\InvalidArgumentException::class); $this->noneDB->createDB(''); @@ -254,7 +256,7 @@ public function numericDatabaseName(): void $result = $this->noneDB->find('12345', 0); - $this->assertCount(1, $result); + $this->assertSame([['data' => 'test', 'key' => 0]], $result); } // ========================================== @@ -278,8 +280,9 @@ public function booleanValues(): void $trueResult = $this->noneDB->find($dbName, ['flag' => true]); $falseResult = $this->noneDB->find($dbName, ['flag' => false]); - $this->assertCount(1, $trueResult); - $this->assertCount(1, $falseResult); + $this->assertSame([['flag' => true, 'key' => 0]], $trueResult); + $this->assertSame([['flag' => false, 'key' => 1]], $falseResult); + $this->assertSame([], $this->noneDB->find($dbName, ['flag' => 1]), 'identical (===) comparison'); } /** @@ -302,9 +305,9 @@ public function numericStringVsInteger(): void // String search $strResult = $this->noneDB->find($dbName, ['value' => '123']); - // Results depend on type comparison behavior - $this->assertIsArray($intResult); - $this->assertIsArray($strResult); + // docs/API.md: filters compare with === ("25" does not match 25) + $this->assertSame([['value' => 123, 'key' => 0]], $intResult); + $this->assertSame([['value' => '123', 'key' => 1]], $strResult); } /** @@ -323,8 +326,8 @@ public function floatValues(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals(19.99, $result[0]['price']); - $this->assertEquals(3.14159, $result[0]['quantity']); + $this->assertSame([$data + ['key' => 0]], $result, 'floats are stored exactly'); + $this->assertSame([0], array_column($this->noneDB->find($dbName, ['negative' => -123.456]), 'key')); } /** @@ -343,7 +346,7 @@ public function nullValues(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertNull($result[0]['description']); + $this->assertSame([$data + ['key' => 0]], $result, 'null fields are kept'); } /** @@ -361,8 +364,7 @@ public function emptyArrayValue(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertIsArray($result[0]['items']); - $this->assertCount(0, $result[0]['items']); + $this->assertSame([['items' => [], 'tags' => [], 'key' => 0]], $result); } /** @@ -379,12 +381,8 @@ public function mixedArrayValue(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertNotEmpty($result); - $this->assertArrayHasKey('mixed', $result[0]); - $this->assertEquals(1, $result[0]['mixed'][0]); - $this->assertEquals('two', $result[0]['mixed'][1]); - $this->assertTrue($result[0]['mixed'][3]); - $this->assertNull($result[0]['mixed'][4]); + // Every element with its type: 3.0 stays a float (docs/API.md "Floats keep their type") + $this->assertSame([['mixed' => [1, 'two', 3.0, true, null, ['nested' => 'value']], 'key' => 0]], $result); } // ========================================== @@ -409,8 +407,7 @@ public function jsonSpecialCharacters(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertEquals('He said "Hello"', $result[0]['quotes']); - $this->assertEquals("line1\nline2", $result[0]['newline']); + $this->assertSame([$data + ['key' => 0]], $result); } /** @@ -427,7 +424,7 @@ public function jsonControlCharacters(): void $this->noneDB->insert($dbName, $data); $result = $this->noneDB->find($dbName, 0); - $this->assertIsString($result[0]['content']); + $this->assertSame("text\r\nwith\tcontrol\x00chars", $result[0]['content'], 'byte for byte, NUL included'); } // ========================================== @@ -448,9 +445,11 @@ public function rapidCreateDeleteCycle(): void $result = $this->noneDB->find($dbName, 0); - // All records deleted (but nulled in place) - $nonNullCount = count(array_filter($result, fn($r) => $r !== null)); - $this->assertEquals(0, $nonNullCount); + // All records deleted + $this->assertSame([], $result); + // Keys 0..9 were used: the next record gets 10 (keys are never reused) + $this->noneDB->insert($dbName, ['iteration' => 'after']); + $this->assertSame([['iteration' => 'after', 'key' => 10]], $this->noneDB->find($dbName, 0)); } /** @@ -472,7 +471,8 @@ public function findAfterMultipleDeletes(): void // Find remaining $remaining = $this->noneDB->find($dbName, ['type' => 'test']); - $this->assertCount(3, $remaining); + $this->assertSame([0, 2, 4], array_column($remaining, 'index')); + $this->assertSame([0, 2, 4], array_column($remaining, 'key')); } /** @@ -497,7 +497,10 @@ public function updateAfterPartialDelete(): void ['set' => ['status' => 'inactive']] ]); - $this->assertEquals(2, $result['n']); + $this->assertSame(['n' => 2], $result); + $this->assertSame([[1, 'inactive', 0], [3, 'inactive', 2]], array_map(function ($r) { + return [$r['id'], $r['status'], $r['key']]; + }, $this->noneDB->find($dbName, 0))); } // ========================================== @@ -506,7 +509,7 @@ public function updateAfterPartialDelete(): void /** * @test - * v3.0: JSONL format gracefully handles corrupted data (returns empty array) + * Corrupted data is handled gracefully (returns empty array) */ public function operationsOnCorruptedDataReturnsEmptyArray(): void { @@ -520,11 +523,20 @@ public function operationsOnCorruptedDataReturnsEmptyArray(): void file_put_contents($filePath, '{invalid json}', LOCK_EX); clearstatcache(true, $filePath); - // JSONL gracefully returns empty array on corrupted data + // The bytes have no line end: an unterminated tail, never a record (docs/STORAGE_FORMAT.md §6.1) $findResult = $this->noneDB->find($dbName, 0); - $this->assertIsArray($findResult); - $this->assertEmpty($findResult); + $this->assertSame([], $findResult); + clearstatcache(); + $this->assertSame('{invalid json}', file_get_contents($filePath), 'readers never cut'); + + // The next write keeps the bytes in a quarantine file recovery/q.{seg}.{off}.{len}.{sha256}.bin (§7.2), + // cuts them and starts the data at offset 0 + $this->assertSame(['n' => 1], $this->noneDB->insert($dbName, ['v' => 'after'])); + $this->assertSame([['v' => 'after', 'key' => 0]], $this->noneDB->find($dbName, 0)); + $quarantine = glob($this->dbPath($dbName, 'quarantine') . 'q.*'); + $this->assertSame([$this->dbPath($dbName, 'quarantine') . 'q.main.0.14.' . hash('sha256', '{invalid json}') . '.bin'], $quarantine); + $this->assertSame('{invalid json}', file_get_contents($quarantine[0])); } /** @@ -539,31 +551,31 @@ public function operationsOnEmptyFile(): void $filePath = $this->getDbFilePath($dbName); file_put_contents($filePath, '', LOCK_EX); - // Operations should handle gracefully $findResult = $this->noneDB->find($dbName, 0); - $this->assertIsArray($findResult); + $this->assertSame([], $findResult); + $this->assertSame(['n' => 1], $this->noneDB->insert($dbName, ['v' => 1])); + $this->assertSame([['v' => 1, 'key' => 0]], $this->noneDB->find($dbName, 0)); } /** * @test - * v3.0: JSONL format uses index file, non-JSONL data is treated as empty + * Bytes that are not a complete JSONL line are no records: the database reads as empty */ public function operationsOnMissingDataKeyReturnsEmptyArray(): void { $dbName = 'missingdatakeytest'; - // Create file with valid JSON but missing 'data' key (non-JSONL format) + // Valid JSON without a line end and without a key $this->noneDB->createDB($dbName); $filePath = $this->getDbFilePath($dbName); file_put_contents($filePath, '{"items": []}', LOCK_EX); clearstatcache(true, $filePath); - // JSONL uses index file for records, returns empty for non-JSONL data + // No complete line: nothing is a record (docs/STORAGE_FORMAT.md §6.1) $findResult = $this->noneDB->find($dbName, 0); - $this->assertIsArray($findResult); - $this->assertEmpty($findResult); + $this->assertSame([], $findResult); } // ========================================== @@ -589,10 +601,10 @@ public function keyFieldInNestedData(): void $result = $this->noneDB->insert($dbName, $data); // Should succeed - 'key' is only reserved at top level - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); $found = $this->noneDB->find($dbName, 0); - $this->assertEquals('api_key_12345', $found[0]['metadata']['key']); + $this->assertSame([$data + ['key' => 0]], $found); } /** @@ -610,8 +622,7 @@ public function findByKeyZero(): void // Find by key 0 (first record) $result = $this->noneDB->find($dbName, ['key' => 0]); - $this->assertCount(1, $result); - $this->assertEquals('first', $result[0]['name']); + $this->assertSame([['name' => 'first', 'key' => 0]], $result); } /** @@ -629,10 +640,11 @@ public function deleteByKeyZero(): void // Delete by key 0 $result = $this->noneDB->delete($dbName, ['key' => [0]]); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); - // First record should be null + // The first record is gone, the second keeps key 1 $remaining = $this->noneDB->find($dbName, ['name' => 'first']); - $this->assertCount(0, $remaining); + $this->assertSame([], $remaining); + $this->assertSame([['name' => 'second', 'key' => 1]], $this->noneDB->find($dbName, 0)); } } diff --git a/tests/Integration/FieldIndexCheckpointTest.php b/tests/Integration/FieldIndexCheckpointTest.php new file mode 100644 index 0000000..53dffa0 --- /dev/null +++ b/tests/Integration/FieldIndexCheckpointTest.php @@ -0,0 +1,307 @@ +dbPath($name, 'fx', $field, $shard); + } + + private function header(string $file): array + { + $fp = fopen($file, 'rb'); + $line = fgets($fp); + fclose($fp); + return json_decode($line, true); + } + + /** @test */ + public function bucketsAreExactValues(): void + { + $values = [1, 1.0, '1', true, -0.0, 0, '0', 0.0, null, false, '', 'a', 2.5, '2.5']; + foreach ([['shardingEnabled' => false], ['shardSize' => 4]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->createFieldIndex('ty', 'v'); + foreach ($values as $v) { + $db->insert('ty', ['v' => $v]); + } + $plain = $this->db($config + ['fieldIndexEnabled' => false]); + foreach ($values as $v) { + $expected = array_column($plain->find('ty', ['v' => $v]), 'key'); + $this->assertSame($expected, array_column($db->find('ty', ['v' => $v]), 'key'), json_encode($v)); + $this->assertSame($expected, $db->getKeysFromFieldIndex('ty', 'v', $v), 'strict: ' . json_encode($v)); + } + $this->assertSame([0], $db->getKeysFromFieldIndex('ty', 'v', 1), '1 is not "1", 1.0 or true'); + $this->assertSame([4, 7], $db->getKeysFromFieldIndex('ty', 'v', -0.0), '-0.0 === 0.0'); + $created = $this->db($config)->rebuildFieldIndex('ty', 'v'); + $this->assertSame(14, $created['indexed'], 'every record holds an indexable value'); + $this->assertSame([4, 7], $this->db($config)->getKeysFromFieldIndex('ty', 'v', 0.0), 'rebuilt: -0.0 and 0.0 are one value'); + } + } + + /** @test */ + public function creatingAnIndexAgainRebuildsIt(): void + { + // Documented: a second createFieldIndex() rebuilds the index, no duplicate entry + foreach ([['shardingEnabled' => false], ['shardSize' => 4]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('again', [['c' => 'a'], ['c' => 'b'], ['c' => 'a']]); + $this->assertTrue($db->createFieldIndex('again', 'c')['success']); + $db->insert('again', [['c' => 'a'], ['c' => 'c'], ['c' => 'a']]); + $result = $db->createFieldIndex('again', 'c'); + $this->assertTrue($result['success'], json_encode($result)); + $this->assertSame([6, null], [$result['indexed'], $result['error']]); + $this->assertSame(isset($config['shardSize']) ? 2 : null, $result['shards'] ?? null, '6 records in shards of 4'); + $this->assertSame(['c'], $db->getFieldIndexes('again')['fields']); + $this->assertSame([0, 2, 3, 5], $this->db($config)->getKeysFromFieldIndex('again', 'c', 'a')); + } + } + + /** @test */ + public function floatBucketsDontDependOnIniSettings(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('fl', 'p'); + $db->insert('fl', [['p' => 0.1], ['p' => 0.30000000000000004], ['p' => 28.980345]]); + $before = [ini_get('precision'), ini_get('serialize_precision')]; + try { + foreach ([['17', '-1'], ['5', '5'], ['14', '10']] as [$precision, $serialize]) { + ini_set('precision', $precision); + ini_set('serialize_precision', $serialize); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0], $db->getKeysFromFieldIndex('fl', 'p', 0.1)); + $this->assertSame([1], $db->getKeysFromFieldIndex('fl', 'p', 0.30000000000000004)); + $this->assertSame([2], array_column($db->find('fl', ['p' => 28.980345]), 'key')); + $db->createFieldIndex('fl2', 'p'); + $db->insert('fl2', ['p' => 0.30000000000000004]); + $this->assertSame([], $db->getKeysFromFieldIndex('fl2', 'p', 0.3)); + $db->dropDB('fl2'); + } + } finally { + ini_set('precision', $before[0]); + ini_set('serialize_precision', $before[1]); + } + } + + /** @test */ + public function singleWritesLeaveTheIndexFileAlone(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('sw', 'c'); + $db->insert('sw', array_map(function ($n) { return ['c' => 'c' . ($n % 5), 'n' => $n]; }, range(0, 99))); + $file = $this->fx('sw', 'c'); + clearstatcache(); + $bytes = file_get_contents($file); + for ($i = 0; $i < 10; $i++) { + $db->insert('sw', ['c' => 'new' . $i]); + $db->update('sw', [['n' => $i], ['set' => ['c' => 'moved']]]); + $db->delete('sw', ['n' => 50 + $i]); + } + clearstatcache(); + $this->assertSame($bytes, file_get_contents($file), 'writes replay later, they never rewrite the index'); + $plain = $this->db(['shardingEnabled' => false, 'fieldIndexEnabled' => false]); + foreach (['c0', 'c3', 'moved', 'new4', 'nothing'] as $c) { + $expected = array_column($plain->find('sw', ['c' => $c]), 'key'); + $this->assertSame($expected, array_column($db->find('sw', ['c' => $c]), 'key'), "warm $c"); + \noneDB::clearStaticCache(); + $this->assertSame($expected, array_column($this->db(['shardingEnabled' => false])->find('sw', ['c' => $c]), 'key'), "cold $c"); + } + } + + /** @test */ + public function aLongTailIsCheckpointed(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('lt', 'c'); + $db->insert('lt', ['c' => 'first']); + $before = $this->header($this->fx('lt', 'c'))['s']; + // The documented rule (this writer has the index loaded): at the end of a write session, a new + // checkpoint once at least 64 KiB of data is new (the index file is far smaller than 256 KiB) + $data = $this->dbPath('lt', 'data'); + $covered = $before; + $checkpoints = 0; + for ($i = 0; $i < 30; $i++) { + $db->insert('lt', array_map(function ($n) use ($i) { return ['c' => 'c' . ($n % 7), 'pad' => str_repeat('x', 100), 'i' => $i]; }, range(0, 19))); + clearstatcache(); + if (filesize($data) - $covered >= 65536) { + $covered = filesize($data); + $checkpoints++; + } + } + clearstatcache(); + $after = $this->header($this->fx('lt', 'c')); + $this->assertSame(1, $checkpoints, 'precondition: about 79 KiB written'); + $this->assertGreaterThan($before, $after['s'], 'checkpointed after about 64 KiB of data'); + $this->assertSame($covered, $after['s'], 'at the end of the first session that reached 64 KiB'); + $this->assertSame(1, $after['v']); // STORAGE_FORMAT.md §6.5.1 + $this->assertSame('fx', $after['kind']); + $this->assertCount(90, $db->find('lt', ['c' => 'c3'])); + } + + /** @test */ + public function aWriterThatNeverReadsLeavesTheCheckpointsToItsReaders(): void + { + // A process that only writes never loads field or spatial indexes (no memory, no index I/O); the + // first reader that replays the long tail writes the new checkpoints, global entries included + foreach ([['shardingEnabled' => false], ['shardSize' => 2000]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $sharded = isset($config['shardSize']); + // Sharded: shard 0 full, shard 1 holds one record and gets the writer's lines + $db->insert('wr', array_fill(0, $sharded ? 2001 : 1, ['c' => 'first', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]])); + $db->createFieldIndex('wr', 'c'); + $db->createSpatialIndex('wr', 'loc'); + $shard = $sharded ? 1 : null; + $fx = $this->fx('wr', 'c', $shard); + $rx = $this->dbPath('wr', 'rx', 'loc', $shard); + $files = function () use ($fx, $rx) { clearstatcache(); return [file_get_contents($fx), file_get_contents($rx)]; }; + \noneDB::clearStaticCache(); + $writer = $this->db($config); + $before = $files(); + for ($i = 0; $i < 30; $i++) { + $writer->insert('wr', array_map(function ($n) use ($i) { + return ['c' => 'c' . ($n % 7), 'pad' => str_repeat('x', 100), 'loc' => ['type' => 'Point', 'coordinates' => [$n % 10, $i % 10]]]; + }, range(0, 19))); + } + $this->assertSame($before, $files(), 'the writer wrote no index file'); + $loaded = array_filter(array_keys($this->getStatic('staticFieldIndexCache') + $this->getStatic('staticSpatialIndexCache')), + function ($key) { return preg_match('~(^|[./])(fx|rx)($|\.)~', basename($key)) === 1; }); // a segment's .fx / .rx + $this->assertSame([], array_values($loaded), 'and loaded none'); + $this->assertTrue($writer->checkIntegrity('wr')['ok']); + $this->assertSame($before, $files(), 'checkIntegrity() writes nothing'); + + \noneDB::clearStaticCache(); + $reader = $this->db($config); + $this->assertCount(90, $reader->find('wr', ['c' => 'c3'])); + $this->assertCount(60, $reader->withinBBox('wr', 'loc', 2.5, -1, 3.5, 10)); + clearstatcache(); + $data = $this->dbPath('wr', 'data', $shard); + $this->assertSame(filesize($data), $this->header($fx)['s'], 'the reader checkpointed the field index'); + $this->assertSame(filesize($data), $this->header($rx)['s'], 'and the R-tree'); + if ($shard !== null) { + $global = $this->header($this->dbPath('wr', 'gfx', 'c')); + $this->assertSame(filesize($data), $global['sh']['1']['s'], 'and its global entry'); + } + \noneDB::clearStaticCache(); + $this->assertCount(90, $this->db($config + ['fieldIndexEnabled' => false])->find('wr', ['c' => 'c3'])); + } + } + + private function getStatic(string $name): array + { + $property = new \noneDB\Tests\InternalValue($name); + return $property->getValue(); + } + + /** @test */ + public function anOldOrForeignIndexFileIsReplayedOrRebuilt(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('old', 'c'); + $db->createFieldIndex('old', 'd'); + $db->insert('old', array_map(function ($n) { return ['c' => 'c' . ($n % 3), 'd' => $n % 2]; }, range(0, 9))); + $stale = file_get_contents($this->fx('old', 'c')); + $db->insert('old', array_map(function ($n) { return ['c' => 'c' . ($n % 3), 'd' => $n % 2, 'pad' => str_repeat('y', 300)]; }, range(0, 299))); + $db->update('old', [['c' => 'c1'], ['set' => ['c' => 'c4']]]); + $plain = $this->db(['shardingEnabled' => false, 'fieldIndexEnabled' => false]); + $expect = function ($field, $value) use ($plain) { return array_column($plain->find('old', [$field => $value]), 'key'); }; + + // An older checkpoint of the same data file: its tail is replayed + file_put_contents($this->fx('old', 'c'), $stale); + \noneDB::clearStaticCache(); + $this->assertSame($expect('c', 'c4'), array_column($this->db(['shardingEnabled' => false])->find('old', ['c' => 'c4']), 'key')); + // The file of another field at this field's place: not used + copy($this->fx('old', 'd'), $this->fx('old', 'c')); + \noneDB::clearStaticCache(); + $this->assertSame($expect('c', 'c0'), array_column($this->db(['shardingEnabled' => false])->find('old', ['c' => 'c0']), 'key')); + // A header covering more than the data, or another epoch: rebuilt + $file = $this->fx('old', 'c'); + $header = $this->header($file); + $body = substr(file_get_contents($file), strpos(file_get_contents($file), "\n") + 1); + foreach ([['s' => $header['s'] + 1000000], ['e' => 'abcdef01']] as $change) { + file_put_contents($file, json_encode(array_merge($header, $change)) . "\n" . $body); + \noneDB::clearStaticCache(); + $this->assertSame($expect('c', 'c2'), array_column($this->db(['shardingEnabled' => false])->find('old', ['c' => 'c2']), 'key'), json_encode($change)); + } + // A new epoch of the key index (rebuildIndex): every field index is rebuilt for it + $this->db(['shardingEnabled' => false])->rebuildIndex('old'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame($expect('d', 1), array_column($db->find('old', ['d' => 1]), 'key')); + $this->assertTrue($db->checkIntegrity('old')['ok']); + } + + /** @test */ + public function caseVariantsAndLongNamesGetTheirOwnFiles(): void + { + // "City" and "city" have different index files, and a long field name is fine (file names stay + // within 255 bytes) + $db = $this->db(['shardSize' => 5]); + $long = str_repeat('ü', 110); + foreach (['city', 'City', $long] as $field) { + $this->assertTrue($db->createFieldIndex('cv', $field)['success'], $field); + } + $db->insert('cv', array_map(function ($n) use ($long) { return ['city' => 'a' . ($n % 2), 'City' => 'b' . ($n % 3), $long => $n % 4]; }, range(0, 19))); + $plain = $this->db(['shardSize' => 5, 'fieldIndexEnabled' => false]); + foreach ([['city', 'a1'], ['City', 'b2'], [$long, 3]] as [$field, $value]) { + $this->assertSame(array_column($plain->find('cv', [$field => $value]), 'key'), array_column($db->find('cv', [$field => $value]), 'key')); + } + foreach ($this->dbFiles('cv') as $file) { + $this->assertLessThanOrEqual(255, strlen(basename($file)), $file); + } + $this->assertNotSame($this->fx('cv', 'city', 0), $this->fx('cv', 'City', 0)); + $this->assertFileExists($this->fx('cv', 'City', 0)); + // Index file names hold no database name (STORAGE_FORMAT.md §5.1): a long database name is fine too + $longDb = str_repeat('n', 220); + $this->assertTrue($db->createFieldIndex($longDb, 'v')['success'], 'a long database name'); + $db->insert($longDb, [['v' => 1], ['v' => 2]]); + $this->assertSame([1], array_column($db->find($longDb, ['v' => 2]), 'key')); + $this->assertSame([1], $db->getKeysFromFieldIndex($longDb, 'v', 2)); + $this->assertFileExists($this->fx($longDb, 'v')); + foreach ($this->dbFiles($longDb) as $file) { + $this->assertLessThanOrEqual(255, strlen(basename($file)), $file); + } + } + + /** @test */ + public function rangeCandidatesEqualAScan(): void + { + mt_srand(31); + $pool = [0, 1, -1, 2.5, -2.5, 9007199254740993, 9007199254740992.0, 1e300, -0.0, '5', ' 5', '5 ', '1e3', '.5', '+5', '007', + '0x1A', 'abc', 'Abc', '', 'z', 'ü', null, true, false, [1], '9', '10', '1a']; + foreach ([['shardingEnabled' => false], ['shardSize' => 7]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->createFieldIndex('rg', 'v'); + $rows = []; + for ($i = 0; $i < 60; $i++) { + $rows[] = mt_rand(0, 9) === 0 ? ['w' => 1] : ['v' => $pool[mt_rand(0, count($pool) - 1)]]; + } + $db->insert('rg', $rows); + $plain = $this->db($config + ['fieldIndexEnabled' => false]); + foreach ($pool as $lo) { + foreach ([$pool[mt_rand(0, count($pool) - 1)], $pool[mt_rand(0, count($pool) - 1)], NAN, INF, -INF] as $hi) { + $label = json_encode([$lo, is_float($hi) && !is_finite($hi) ? (is_nan($hi) ? 'NAN' : ($hi > 0 ? 'INF' : '-INF')) : $hi]); + $expected = array_column($plain->between('rg', 'v', $lo, $hi), 'key'); + $this->assertSame($expected, array_column($db->between('rg', 'v', $lo, $hi), 'key'), $label); + $this->assertSame($expected, array_column($db->query('rg')->between('v', $lo, $hi)->get(), 'key'), "query builder: $label"); + if (is_int($lo) || is_float($lo) || is_string($lo)) { + // A comparable bound: the field index is the range index, no segment is scanned + $this->assertSame(0, $db->query('rg')->between('v', $lo, $hi)->explain()['segments']['scanned'], $label); + } + } + } + } + } +} diff --git a/tests/Integration/FieldIndexFormatTest.php b/tests/Integration/FieldIndexFormatTest.php new file mode 100644 index 0000000..f91794f --- /dev/null +++ b/tests/Integration/FieldIndexFormatTest.php @@ -0,0 +1,1613 @@ + key | [keys] of a field over data files (last line wins, tombstones delete) */ + private function modelMap(array $dataFiles, string $field): array + { + $records = []; + foreach ($dataFiles as $file) { + foreach (array_filter(explode("\n", (string)file_get_contents($file)), 'strlen') as $line) { + $record = json_decode($line, true); + if (isset($record['__nonedb_deleted'])) { + unset($records[$record['__nonedb_deleted']]); + } elseif (isset($record['key'])) { + $records[$record['key']] = $record; + } + } + } + ksort($records); + $map = []; + foreach ($records as $key => $record) { + if (array_key_exists($field, $record) && ($id = self::typedId($record[$field])) !== null) { + $map[$id][] = $key; + } + } + ksort($map, SORT_STRING); + return array_map(function ($keys) { return count($keys) === 1 ? $keys[0] : $keys; }, $map); + } + + private function crcBefore(string $file, int $s): string + { + $start = max(0, $s - 256); + return hash('crc32b', substr((string)file_get_contents($file), $start, $s - $start)); + } + + private function stats(): array + { + return Testing::get('indexStats'); + } + + /** @test §6.5.1: a segment's field index is a paged file of the values of its data */ + public function aSegmentIndexIsAPagedFileOfItsData(): void + { + $db = $this->db(['shardingEnabled' => false]); + $rows = []; + for ($i = 0; $i < 3000; $i++) { + $rows[] = ['g' => $i % 7 === 0 ? $i % 50 : ($i % 11 === 0 ? null : 'g' . ($i % 100)), 'pad' => 'x']; + } + $db->insert('fi', $rows); + $db->createFieldIndex('fi', 'g'); + $data = $this->dbPath('fi', 'data'); + [$header, $map] = $this->readFieldIndexFile($this->dbPath('fi', 'fx', 'g')); + $kix = $this->readKeyIndexFile($this->dbPath('fi', 'kix'))['header']; + $this->assertSame(['fx', 1, 'g'], [$header['kind'], $header['v'], $header['field']]); + $this->assertSame([$kix['s'], $kix['e']], [$header['s'], $header['e']], 'covers the bytes of the key index, its epoch'); + $this->assertSame($this->crcBefore($data, $header['s']), $header['h']); + $model = $this->modelMap([$data], 'g'); + $this->assertSame($model, $map); + $this->assertSame(count($model), $header['c']); + $this->assertSame(3000, $header['n'], 'every record has the field (null included)'); + $this->assertSame(2999, $header['mk']); + $this->assertGreaterThanOrEqual(1, $header['pg']['levels'], 'more than one data page: a directory'); + $this->assertGreaterThan(1, $header['pg']['pages']); + } + + /** + * @test §6.5.1 "Directory pages": bucket ids of more than 2 KiB fit one per page, yet a directory page + * holds at least two entries, so every level is smaller than the one below and the file is written + * (a segment's index; the global index of the sharded layout hashes them into its filters). Its own process: + * a builder that doesn't shrink runs out of memory. + * @runInSeparateProcess + */ + public function aDirectoryOfLongBucketIdsStillShrinks(): void + { + foreach (['one segment' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 100]] as $layout => $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $rows = []; + for ($i = 0; $i < 400; $i++) { + $rows[] = ['c' => 'c' . ($i % 40) . str_repeat('-', 2500)]; + } + $db->insert('fd', $rows); + $db->createFieldIndex('fd', 'c'); + [$header, $map] = $this->readFieldIndexFile($this->dbPath('fd', 'fx', 'c', $layout === 'sharded' ? 0 : null)); + $this->assertCount(40, $map, $layout); + $this->assertGreaterThanOrEqual(2, $header['pg']['levels'], "$layout: directory levels over 40 one-entry data pages"); + if ($layout === 'sharded') { + // Every shard holds the 40 values: 2048 bits (the smallest power of two of at least 1280) + [$global, $filters] = $this->readGlobalFieldFile($this->dbPath('fd', 'gfx', 'c')); + $this->assertSame(2048, $global['bits']); + $filter = $this->shardFilter(array_map(function ($v) { return 'c' . $v . str_repeat('-', 2500); }, range(0, 39)), 2048); + $this->assertSame(array_fill(0, 4, $filter), $filters); + } + foreach ([0, 17, 39] as $v) { + $this->assertSame(range($v, 399, 40), array_column($db->find('fd', ['c' => 'c' . $v . str_repeat('-', 2500)]), 'key'), "$layout: c$v"); + } + } + } + + /** + * @test Reading a paged file whole, and dropping a database, leave no garbage cycles: nothing waits for + * PHP's cycle collector (a recursive closure over the file's bytes kept them allocated until it ran) + */ + public function wholeReadsAndDropsLeaveNoGarbageCycles(): void + { + $db = $this->db(['shardSize' => 500, 'cacheMemoryLimit' => 0]); + $rows = []; + for ($i = 0; $i < 3000; $i++) { + $rows[] = ['u' => 'u' . $i, 'g' => 'g' . ($i % 40)]; + } + $db->insert('fg', $rows); + $db->createFieldIndex('fg', 'u'); + $db->createFieldIndex('fg', 'g'); + $db->insert('fd', [['a' => 1]]); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 500, 'cacheMemoryLimit' => 0]); + $db->find('fg', ['key' => 0]); // the process's first operation + gc_collect_cycles(); + $this->assertSame([2500], array_column($db->find('fg', ['u' => 'u2500']), 'key')); // .fx loaded whole + $this->assertCount(75, $db->find('fg', ['g' => 'g7'])); + $this->assertSame(0, gc_collect_cycles(), 'whole reads of paged files'); + $this->assertTrue($db->dropDB('fd')['success']); + $this->assertSame(0, gc_collect_cycles(), 'a drop'); + } + + /** @test §6.5.1: a large index gets more directory levels; a lookup still reads one page per level */ + public function aLargeIndexHasMoreLevels(): void + { + $db = $this->db(['shardingEnabled' => false]); + // Values of 200 bytes: entries of 211 bytes, 19 per page; 1000 values = 53 data pages, 3 directory + // pages, a root + $rows = []; + for ($i = 0; $i < 1000; $i++) { + $rows[] = ['u' => str_pad((string)$i, 200, '-', STR_PAD_LEFT)]; + } + $db->insert('fl', $rows); + $db->createFieldIndex('fl', 'u'); + [$header, $map] = $this->readFieldIndexFile($this->dbPath('fl', 'fx', 'u')); + $this->assertGreaterThanOrEqual(2, $header['pg']['levels'], 'directory pages under a root'); + $this->assertSame($this->modelMap([$this->dbPath('fl', 'data')], 'u'), $map); + + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stats(); + $value = str_pad('123', 200, '-', STR_PAD_LEFT); + $this->assertSame([123], array_column($db->find('fl', ['u' => $value]), 'key')); + $after = $this->stats(); + $this->assertSame(1, $after['pointLookups'] - $before['pointLookups'], 'answered by a point lookup'); + $this->assertSame($header['pg']['levels'] + 1, $after['pagesRead'] - $before['pagesRead'], 'one page per level'); + $this->assertNull(Testing::getOf($db, 'fieldIndexCache')[$this->dbPath('fl', 'fx', 'u')] ?? null, 'the whole index was not loaded'); + } + + /** + * @test §6.5.1 lookup step 4: lines written after the index are applied to a point lookup (an index file + * above 64 KiB: a smaller one is loaded whole at its first use) + */ + public function pointLookupsFollowTheNewerLines(): void + { + $db = $this->db(['shardingEnabled' => false]); + for ($start = 0; $start < 14000; $start += 2000) { // in batches: the suite's memory + $db->insert('fn', array_map(function ($i) { return ['c' => 'c' . ($i % 10)]; }, range($start, $start + 1999))); + } + $db->createFieldIndex('fn', 'c'); + clearstatcache(); + $this->assertGreaterThan(65536, filesize($this->dbPath('fn', 'fx', 'c')), 'precondition: read by pages'); + $covered = $this->readFieldIndexFile($this->dbPath('fn', 'fx', 'c'))[0]['s']; + // Written after the index: moved away from c3, deleted from c3, a new c3, moved into c3 + $db->update('fn', [['key' => 3], ['set' => ['c' => 'c4']]]); + $db->delete('fn', ['key' => 13]); + $db->insert('fn', ['c' => 'c3']); + $db->update('fn', [['key' => 4], ['set' => ['c' => 'c3']]]); + $this->assertSame($covered, $this->readFieldIndexFile($this->dbPath('fn', 'fx', 'c'))[0]['s'], 'writers never rewrite the index'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stats(); + $expected = $this->modelMap([$this->dbPath('fn', 'data')], 'c')['sc3']; + $this->assertSame($expected, array_column($db->find('fn', ['c' => 'c3']), 'key')); + $this->assertNotContains(3, $expected); + $this->assertNotContains(13, $expected); + $this->assertContains(4, $expected); + $this->assertContains(14000, $expected); + $this->assertSame(1, $this->stats()['pointLookups'] - $before['pointLookups']); + } + + /** + * @test §6.5.1 "Whole read": an .fx file of at most 64 KiB is loaded whole at its first use in an operation + * other than the process's first, a larger one is read by pages (point lookups); a process without room + * under its cache limit keeps the pages + */ + public function aSmallIndexIsLoadedWholeAtItsFirstUse(): void + { + // 30 values of 10 keys each; a value of 3000 bytes puts the file above 64 KiB + foreach (['small' => 20, 'large' => 3000] as $size => $pad) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]); + $rows = []; + for ($i = 0; $i < 300; $i++) { + $rows[] = ['c' => 'c' . ($i % 30) . str_repeat('-', $pad)]; + } + $db->insert('fs', $rows); + $db->createFieldIndex('fs', 'c'); + $fx = $this->dbPath('fs', 'fx', 'c'); + clearstatcache(); + $small = filesize($fx) <= 65536; + $this->assertSame($size === 'small', $small, "precondition: $size file"); + $value = 'c7' . str_repeat('-', $pad); + foreach ([true, false] as $room) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => $room ? 0 : memory_get_usage() + 8 * 1048576]); + $label = "$size file, " . ($room ? 'with room' : 'no room'); + $before = $this->stats(); + $this->assertSame(range(7, 299, 30), array_column($db->find('fs', ['c' => $value]), 'key'), $label); + $whole = $small && $room; + $this->assertSame($whole, is_array(Testing::getOf($db, 'fieldIndexCache')[$fx] ?? null), "$label: loaded whole"); + $this->assertSame($whole ? 0 : 1, $this->stats()['pointLookups'] - $before['pointLookups'], "$label: point lookups"); + } + } + // A process's first operation (a single cold one) reads the pages; its next use loads the file. 1000 + // entries (a file below 64 KiB): the rent-or-buy rule of larger files would wait for 50 operations + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]); + $db->insert('fc', array_map(function ($i) { return ['c' => 'v' . $i]; }, range(0, 999))); + $db->createFieldIndex('fc', 'c'); + $fx = $this->dbPath('fc', 'fx', 'c'); + clearstatcache(); + $this->assertLessThanOrEqual(65536, filesize($fx), 'precondition: a small file'); + $this->assertSame(1000, $this->readFieldIndexFile($fx)[0]['c']); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]) . '; + $fx = ' . var_export($fx, true) . '; + foreach ([7, 8] as $v) { + $before = \noneDB\Tests\Support\Testing::get("indexStats")["pointLookups"]; + $keys = array_column($db->find("fc", ["c" => "v" . $v]), "key"); + echo json_encode([$keys === [$v], \noneDB\Tests\Support\Testing::get("indexStats")["pointLookups"] - $before, + is_array(\noneDB\Tests\Support\Testing::getOf($db, "fieldIndexCache")[$fx] ?? null)]), "\n"; + }'); + $this->assertSame("[true,1,false]\n[true,0,true]", $out, 'first operation: a point lookup; second: loaded whole'); + } + + /** + * @test §6.5.2: the global index of a sharded database holds a filter of each shard's values, bit-sliced into + * rows of one byte per shard; its size follows the largest shard + */ + public function aShardedDatabaseHasAGlobalIndexOfShardFilters(): void + { + $db = $this->db(['shardSize' => 500]); + $rows = []; + for ($i = 0; $i < 1500; $i++) { + $rows[] = ['g' => $i < 500 ? 'only0' : ($i % 3 === 0 ? 'every' : 'v' . ($i % 40))]; + } + $db->insert('fg', $rows); + $db->createFieldIndex('fg', 'g'); + [$header, $filters] = $this->readGlobalFieldFile($this->dbPath('fg', 'gfx', 'g')); + $this->assertSame(['gfx', 1, 'g'], [$header['kind'], $header['v'], $header['field']]); + $this->assertSame([0, 1, 2], array_keys($header['sh'])); + $ids = []; + foreach ([0, 1, 2] as $shard) { + $kix = $this->readKeyIndexFile($this->dbPath('fg', 'kix', $shard))['header']; + $entry = $header['sh'][$shard]; + $this->assertSame([$kix['s'], $kix['e'], $this->crcBefore($this->dbPath('fg', 'data', $shard), $kix['s'])], [$entry['s'], $entry['e'], $entry['h']], "shard $shard"); + // Each shard's own file is local to it + $ids[$shard] = array_keys($this->modelMap([$this->dbPath('fg', 'data', $shard)], 'g')); + [, $local] = $this->readFieldIndexFile($this->dbPath('fg', 'fx', 'g', $shard)); + $this->assertSame($this->modelMap([$this->dbPath('fg', 'data', $shard)], 'g'), $local, "shard $shard"); + } + // The largest shard holds 41 values: 2048 bits (the smallest power of two of at least 1312); 3 shards: 8 wide + $this->assertSame([1, 41, 41], array_map('count', $ids)); + $this->assertSame([2048, 8], [$header['bits'], $header['width']]); + $this->assertSame(array_map(function ($shardIds) { return $this->shardFilterOfIds($shardIds, 2048); }, $ids), $filters); + + // A lookup skips the shards whose filter rules the value out + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 500]); + $this->assertSame(500, count($db->find('fg', ['g' => 'only0']))); + $expected = array_map(function ($filter) { return $this->filterMayHold($filter, 'only0') ? 'index' : 'skip'; }, $filters); + $this->assertSame(['index', 'skip', 'skip'], $expected, 'precondition: no other filter answers "may" for it'); + $this->assertSame($expected, array_column($db->query('fg')->where(['g' => 'only0'])->explain()['plan'], 'access')); + } + + /** @test §6.5: a page whose crc doesn't match, or a file without pages, is never trusted */ + public function aDamagedFileOrOneWithoutPagesIsNeverTrusted(): void + { + $cases = [ + 'page crc' => function (string $fx) { + $raw = file_get_contents($fx); + $at = strpos($raw, "\n") + 20; + $raw[$at] = chr(ord($raw[$at]) ^ 0x10); + file_put_contents($fx, $raw); + }, + 'cut body' => function (string $fx) { file_put_contents($fx, substr(file_get_contents($fx), 0, -100)); }, + 'a body of JSON, no pages' => function (string $fx) { + [$header, $map] = $this->readFieldIndexFile($fx); + unset($header['pg']); + file_put_contents($fx, $this->sealHeaderLine(json_encode($header)) . "\n" . json_encode($map) . "\n"); + }, + ]; + foreach ($cases as $case => $damage) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $rows = []; + for ($i = 0; $i < 2000; $i++) { + $rows[] = ['c' => 'c' . ($i % 20)]; + } + $db->insert('fd', $rows); + $db->createFieldIndex('fd', 'c'); + $model = $this->modelMap([$this->dbPath('fd', 'data')], 'c'); + \noneDB::clearStaticCache(); + $damage($this->dbPath('fd', 'fx', 'c')); + $db = $this->db(['shardingEnabled' => false]); + foreach (['c0', 'c7', 'c19'] as $value) { + $this->assertSame($model['s' . $value], array_column($db->find('fd', ['c' => $value]), 'key'), "$case: $value"); + } + [$header, $map] = $this->readFieldIndexFile($this->dbPath('fd', 'fx', 'c')); // rebuilt as a valid file + $this->assertSame($model, $map, $case); + $this->assertTrue($db->checkIntegrity('fd')['ok'], $case); + } + } + + /** + * @test §6.5.2 "Damage": a row whose crc doesn't match makes the global index invalid, even when its header + * is valid. The damage below clears one of value-0's probe bits in shard 0's filter: trusted, the row would + * rule shard 0 out and value-0's keys there would be lost. Lookups stay exact and the file is written again. + */ + public function aDamagedRowOfTheGlobalIndexIsRebuilt(): void + { + $config = ['shardSize' => 500]; + $db = $this->db($config); + $rows = []; + for ($i = 0; $i < 1500; $i++) { + $rows[] = ['g' => 'value-' . ($i % 1000)]; + } + $db->insert('gd', $rows); + $db->createFieldIndex('gd', 'g'); + $gfx = $this->dbPath('gd', 'gfx', 'g'); + [$header, $model] = $this->readGlobalFieldFile($gfx); + [$at, $bit] = $this->globalBitAt($gfx, $header, $this->filterProbes($this->bucketIdOf('value-0'), $header['bits'])[0], 0); + $raw = (string)file_get_contents($gfx); + $this->assertNotSame(0, ord($raw[$at]) & $bit, 'precondition: shard 0 holds value-0'); + $raw[$at] = chr(ord($raw[$at]) & ~$bit); + file_put_contents($gfx, $raw); + + \noneDB::clearStaticCache(); + $db = $this->db($config); + $scan = $this->db($config + ['fieldIndexEnabled' => false]); + foreach (['value-0', 'value-499', 'value-500', 'value-998', 'value-999', 'none'] as $value) { + $this->assertSame(array_column($scan->find('gd', ['g' => $value]), 'key'), array_column($db->find('gd', ['g' => $value]), 'key'), $value); + } + $this->assertSame([0, 1000], array_column($db->find('gd', ['g' => 'value-0']), 'key')); + [, $filters] = $this->readGlobalFieldFile($gfx); // a valid file again + $this->assertSame($model, $filters); + } + + /** + * @test §6.5.2 "Update": a shard's filter changed in a session is written when the session ends; a lookup in + * the same session before that uses the changed filter, not the file's. Here shard 2 got a new value behind + * the index (a writer without indexes) and lost its field index file: a read in the session builds it again + * from the data, so the session's filter of shard 2 holds the new value and its entry covers the new line. + */ + public function aLookupInTheSessionSeesAFilterTheSessionChanged(): void + { + $config = ['shardSize' => 10]; + $db = $this->db($config); + $db->insert('sf', array_map(function ($n) { return ['g' => 'g' . intdiv($n, 10)]; }, range(0, 39))); + $db->createFieldIndex('sf', 'g'); + $this->assertSame(1, $this->db($config + ['fieldIndexEnabled' => false])->update('sf', [['key' => 25], ['set' => ['g' => 'new']]])['n']); + unlink($this->dbPath('sf', 'fx', 'g', 2)); + [, $filters] = $this->readGlobalFieldFile($this->dbPath('sf', 'gfx', 'g')); + $this->assertFalse($this->filterMayHold($filters[2], 'new'), 'precondition: the file\'s filter of shard 2 doesn\'t hold it'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $found = []; + (function () use (&$found) { + $this->withLock('sf', true, function () use (&$found) { + $found['g2'] = array_column($this->findUnlocked('sf', ['g' => 'g2']), 'key'); // builds shard 2's index + $found['new'] = array_column($this->findUnlocked('sf', ['g' => 'new']), 'key'); + }); + })->call(Testing::engine($db)); + $this->assertSame(['g2' => [20, 21, 22, 23, 24, 26, 27, 28, 29], 'new' => [25]], $found); + [, $filters] = $this->readGlobalFieldFile($this->dbPath('sf', 'gfx', 'g')); + $this->assertSame($this->shardFilter(['g2', 'new'], 64), $filters[2], 'written when the session ended'); + } + + /** + * @test §6.5.2 "Damage" and "Update": an update that copies the rows of a damaged file (a row whose crc doesn't + * match) makes the file again from the shards instead, and leaves no temporary file behind. The damage is in a + * row only the copy reads. + */ + public function anUpdateThatMeetsADamagedRowMakesTheFileAgain(): void + { + $config = ['shardSize' => 100, 'autoCompactThreshold' => 1000]; // no compaction: it would replace shard 1's data + $db = $this->db($config); + $db->insert('ud', array_map(function ($n) { return ['g' => 'g' . ($n % 7)]; }, range(0, 299))); + $db->createFieldIndex('ud', 'g'); + $gfx = $this->dbPath('ud', 'gfx', 'g'); + [$header] = $this->readGlobalFieldFile($gfx); + $this->assertSame(256, $header['bits'], 'precondition: seven values per shard'); + [$at, $bit] = $this->globalBitAt($gfx, $header, 5, 1); // shard 1's bit of row 5 + $raw = (string)file_get_contents($gfx); + $raw[$at] = chr(ord($raw[$at]) ^ $bit); + file_put_contents($gfx, $raw); + // This process uses shard 1's field index (read directly, no global lookup), then writes more than 64 KiB + // of new lines into shard 1: the session checkpoints the index, and its filter goes into the global file, + // whose other rows are copied + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(102, 199, 7), $db->getKeysFromFieldIndex('ud', 'g', 'g4', 1)); + $this->assertSame(100, $db->update('ud', [['key' => range(100, 199)], ['set' => ['pad' => str_repeat('x', 1000)]]])['n']); + [$after, $filters] = $this->readGlobalFieldFile($gfx); + $this->assertNotSame($header['w'], $after['w'], 'written again'); + $this->assertSame([0, 1, 2], array_keys($after['sh'])); + $this->assertSame(array_fill(0, 3, $this->shardFilter(['g0', 'g1', 'g2', 'g3', 'g4', 'g5', 'g6'], 256)), $filters, 'made from the shards'); + $this->assertSame([], glob(dirname($gfx) . '/*.tmp*'), 'no temporary file left'); + $this->assertSame(range(4, 299, 7), array_column($this->db($config)->find('ud', ['g' => 'g4']), 'key')); + } + + /** + * @test §6.5.2 "Update": the filters of the shards a session changed replaced by copying the file's pages: their + * columns hold their new filters and every other bit of every row is the old file's, in rows of 1, 2 and 4 bytes. A bit that only a copy + * keeps (another shard's, set by hand, its page crc made again) shows the file was copied, not made again. + */ + public function anUpdateReplacesTheShardsColumnAndCopiesEveryOtherBit(): void + { + foreach (['1-byte rows' => [3, 8], '2-byte rows' => [12, 16], '4-byte rows' => [20, 32]] as $label => [$shards, $width]) { + $this->cleanTestDirectory(); + $config = ['shardSize' => 100, 'autoCompactThreshold' => 1000]; // no compaction: it would replace shard 1's data + $db = $this->db($config); + $db->insert('uc', array_map(function ($n) { return ['g' => 'g' . ($n % 7) . '-' . intdiv($n, 100)]; }, range(0, 100 * $shards - 1))); + $db->createFieldIndex('uc', 'g'); + $gfx = $this->dbPath('uc', 'gfx', 'g'); + [$header, $before] = $this->readGlobalFieldFile($gfx); + $this->assertSame([256, $width, range(0, $shards - 1)], [$header['bits'], $header['width'], array_keys($header['sh'])], "precondition, $label"); + // The last shard's bit of a row its filter leaves clear, set by hand: the page stays valid + $other = $shards - 1; + for ($row = 0; (ord($before[$other][intdiv($row, 8)]) >> ($row % 8)) & 1; $row++); + [$at, $bit] = $this->globalBitAt($gfx, $header, $row, $other); + $raw = (string)file_get_contents($gfx); + $raw[$at] = chr(ord($raw[$at]) | $bit); + $rb = intdiv($width, 8); + $per = $this->globalPageRows(256, $width); + $page = strpos($raw, "\n") + 1 + intdiv($row, $per) * ($per * $rb + 4); + $raw = substr_replace($raw, pack('V', (int)hexdec(hash('crc32b', substr($raw, $page, $per * $rb)))), $page + $per * $rb, 4); + file_put_contents($gfx, $raw); + $expected = $before; + $expected[$other][intdiv($row, 8)] = chr(ord($expected[$other][intdiv($row, 8)]) | (1 << ($row % 8))); + $this->assertSame($expected, $this->readGlobalFieldFile($gfx)[1], "precondition, $label: a valid file"); + // This process uses the field indexes of shards 1 and 2, then writes more than 64 KiB of new lines into + // each: the session checkpoints both, and their filters go into the global file, whose rows are copied + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(104, 199, 7), $db->getKeysFromFieldIndex('uc', 'g', 'g6-1', 1), $label); + $this->assertSame(range(206, 299, 7), $db->getKeysFromFieldIndex('uc', 'g', 'g3-2', 2), $label); + $this->assertSame(200, $db->update('uc', [['key' => range(100, 299)], ['set' => ['g' => 'new', 'pad' => str_repeat('x', 1000)]]])['n']); + [$after, $filters] = $this->readGlobalFieldFile($gfx); + $this->assertNotSame($header['w'], $after['w'], "$label: written again"); + $expected[1] = $expected[2] = $this->shardFilter(['new'], 256); + $this->assertSame($expected, $filters, $label); + $this->assertSame(range(100, 299), array_column($this->db($config)->find('uc', ['g' => 'new']), 'key'), $label); + } + } + + /** + * @test STORAGE_FORMAT.md §6.5.1, §6.5.2 "Update": a checkpoint is the index of the data at `s`, whatever led + * to it. When the lines since the file moved no key to another value, the new file is the old body under a new + * header (no body is built); when they made or emptied no bucket, the shard's filter is the one the global + * index holds (no filter is built, its bits stay). Each file equals the test's own model of the data. + */ + public function aCheckpointBuildsOnlyWhatTheDataChanged(): void + { + foreach (['single layout' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 500]] as $label => $config) { + $this->cleanTestDirectory(); + $config += ['autoCompactThreshold' => 1000]; // no compaction: it would make every index again + $sharded = isset($config['shardSize']); + $db = $this->db($config); + $db->insert('cb', array_map(function ($n) { return ['g' => 'g' . ($n % 7), 'pad' => '']; }, range(0, 999))); + $db->createFieldIndex('cb', 'g'); + $shard = $sharded ? 0 : null; + $fx = $this->dbPath('cb', 'fx', 'g', $shard); + $data = $this->dbPath('cb', 'data', $shard); + $gfx = $sharded ? $this->dbPath('cb', 'gfx', 'g') : null; + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(3, 999, 7), $db->getKeysFromFieldIndex('cb', 'g', 'g3'), "$label: this process uses the index"); + $update = function (array $keys, array $set) use ($db) { return $db->update('cb', [['key' => $keys], ['set' => $set]])['n']; }; + $every = function (int $from, int $to, int $value) { return array_values(array_filter(range($from, $to), function ($k) use ($value) { return $k % 7 === $value; })); }; + $last = $sharded ? 499 : 999; // the segment's keys: shard 0, or the whole database + // Each phase writes more than 64 KiB into the segment in one session: [write, records written, bodies built, filters built] + $phases = [ + // every line keeps its key's value: the same buckets, the same keys + 'another field' => [function () use ($update) { return $update(range(0, 99), ['pad' => str_repeat('x', 1000)]); }, 100, 0, 0], + // keys of g1 to g2, g1 keeps keys: the same buckets, other keys + 'keys moved' => [function () use ($update, $every) { return $update($every(100, 299, 1), ['g' => 'g2', 'pad' => str_repeat('y', 3000)]); }, 28, 1, 0], + // a value no record had: a new bucket + 'a new value' => [function () use ($update) { return $update(range(400, 419), ['g' => 'new', 'pad' => str_repeat('z', 4000)]); }, 20, 1, 1], + // keys leave their buckets, every bucket keeps others: no key added + 'a field removed' => [function () use ($db) { return $db->query('cb')->where(['key' => range(0, 69)])->removeFields(['g'])['n']; }, 70, 1, 0], + // the same buckets as the file the last phase wrote + 'another field again' => [function () use ($update) { return $update(range(300, 399), ['pad' => str_repeat('w', 1000)]); }, 100, 0, 0], + // another valid file put in its place (with a value no record has): its body is not the process's + 'a file replaced behind it' => [function () use ($update, $fx) { + [$header, $map] = $this->readFieldIndexFile($fx); + $this->writeFieldIndexFile($fx, $header, $map + ['snone' => 1]); + return $update(range(0, 99), ['pad' => str_repeat('u', 1000)]); + }, 100, 1, 0], + // the last keys of g5 to g6: one bucket less, none new + 'a value emptied' => [function () use ($update, $every, $last) { return $update($every(70, $last, 5), ['g' => 'g6', 'pad' => str_repeat('v', 1200)]); }, + count($every(70, $last, 5)), 1, 1], + ]; + foreach ($phases as $phase => [$write, $written, $bodies, $filters]) { + clearstatcache(); + [$before, $raw] = [$this->readFieldIndexFile($fx)[0], (string)file_get_contents($fx)]; + $globalBefore = $sharded ? $this->readGlobalFieldFile($gfx) : null; + $stats = Testing::get('indexStats'); + $this->assertSame($written, $write(), "$label, $phase"); + clearstatcache(); + $after = $this->readFieldIndexFile($fx)[0]; + $this->assertSame(filesize($data), $after['s'], "$label, $phase: checkpointed at the end of the session"); + $this->assertNotSame($before['w'], $after['w'], "$label, $phase: a new file"); + [, $map] = $this->readFieldIndexFile($fx); + $this->assertSame($this->modelMap([$data], 'g'), $map, "$label, $phase: the index of the data"); + $built = Testing::get('indexStats'); + $this->assertSame($bodies, $built['bodiesBuilt'] - $stats['bodiesBuilt'], "$label, $phase: bodies built"); + if ($bodies === 0) { + $body = function (string $file) { return substr($file, strpos($file, "\n") + 1); }; + $this->assertSame($body($raw), $body((string)file_get_contents($fx)), "$label, $phase: the old body"); + $this->assertSame([$before['pg'], $before['c'], $before['n']], [$after['pg'], $after['c'], $after['n']], "$label, $phase"); + } + if ($sharded) { + $this->assertSame($filters, $built['filtersBuilt'] - $stats['filtersBuilt'], "$label, $phase: filters built"); + [$header, $columns] = $this->readGlobalFieldFile($gfx); + $this->assertSame(['s' => $after['s'], 'e' => $after['e'], 'h' => $after['h']], $header['sh'][0], "$label, $phase: the shard's new entry"); + $this->assertSame($globalBefore[0]['sh'][1], $header['sh'][1], "$label, $phase: the other shard's entry"); + $this->assertSame([$this->shardFilterOfIds(array_keys($map), $header['bits']), $globalBefore[1][1]], [$columns[0], $columns[1]], "$label, $phase: the filters"); + } + } + $this->assertArrayNotHasKey('sg5', $map, "$label: g5 is gone from the shard"); + $plain = $this->db($config + ['fieldIndexEnabled' => false]); + foreach (['g1', 'g2', 'new', 'g5', 'g6'] as $value) { + $this->assertSame(array_column($plain->find('cb', ['g' => $value]), 'key'), array_column($db->find('cb', ['g' => $value]), 'key'), "$label: $value"); + } + } + } + + /** + * @test §6.5.2 "Lookup": a reader in another process replaces the global index (a checkpoint under the shared + * lock, no new generation) while this process holds the old header. The new file has eight times the rows: + * probes worked out with the old size would read rows of the new file that belong to other bytes of the + * filters, and a shard could be ruled out wrongly. The lookup reads the header again and follows the new one. + */ + public function rowsOfAReplacedFileAreNeverReadWithTheOldHeader(): void + { + $config = ['shardSize' => 1000]; + $db = $this->db($config); + $rows = []; + for ($i = 0; $i < 4000; $i++) { + $rows[] = ['u' => sprintf('s%d-%d', intdiv($i, 1000), $i % 10)]; // shards 0-3, ten values each + } + $db->insert('rp', $rows); + $line = function ($prefix, $i) { return ['u' => sprintf('%s%04d', $prefix, $i), 'pad' => str_repeat('x', 120)]; }; + $db->insert('rp', array_map(function ($i) use ($line) { return $line('b', $i); }, range(0, 7))); // shard 4 + $db->createFieldIndex('rp', 'u'); + $gfx = $this->dbPath('rp', 'gfx', 'u'); + [$old] = $this->readGlobalFieldFile($gfx); + $this->assertSame(512, $old['bits'], 'precondition: ten values in the largest shard'); + // A writer that never reads (no index loaded) appends more than 64 KiB to shard 4 + \noneDB::clearStaticCache(); + $this->db($config)->insert('rp', array_map(function ($i) use ($line) { return $line('a', $i); }, range(0, 499))); + $this->assertSame($old['w'], $this->readGlobalFieldFile($gfx)[0]['w'], 'precondition: the writer left the global file'); + + // This process reads the global header. Its cache limit leaves no room for the exact map or the whole body: + // its lookups read rows from the file + \noneDB::clearStaticCache(); + $a = $this->db($config + ['cacheMemoryLimit' => memory_get_usage() + 1048576]); + $this->assertSame(range(5, 995, 10), array_column($a->find('rp', ['u' => 's0-5']), 'key')); + $state = $this->lockState('rp'); + + // Another process reads shard 4: its long tail makes it write the shard's checkpoint and the global file + $out = $this->runChild('$db = ' . $this->childDb($config) . '; echo json_encode(array_column($db->find("rp", ["u" => "a0003"]), "key"));'); + $this->assertSame('[4011]', $out); + [$new, $filters] = $this->readGlobalFieldFile($gfx); + $this->assertNotSame($old['w'], $new['w'], 'precondition: the global file was replaced'); + $this->assertSame(16384, $new['bits'], 'precondition: 508 values in shard 4, more than 512 bits allow: made again with 16384'); + $this->assertTrue($this->filterMayHold($filters[4], 'a0003'), 'precondition: with the new values'); + $this->assertSame($state, $this->lockState('rp'), 'precondition: no new generation or epoch'); + + $scan = $this->db($config + ['fieldIndexEnabled' => false, 'cacheMemoryLimit' => memory_get_usage() + 1048576]); + foreach (['s3-5', 's2-0', 's0-5', 'a0100', 'a0499', 'b0005', 'none'] as $value) { + $this->assertSame(array_column($scan->find('rp', ['u' => $value]), 'key'), array_column($a->find('rp', ['u' => $value]), 'key'), $value); + } + $this->assertArrayNotHasKey($gfx . '#hot', array_filter(Testing::get('staticFieldIndexCache')), 'precondition: no exact map in this process'); + // The new file is valid: followed, not taken for damage (which would make it again) + $this->assertSame($new['w'], $this->readGlobalFieldFile($gfx)[0]['w'], 'the file the other reader wrote is used as it is'); + } + + /** + * @test §6.5.2 "Lookup": from the global index's second operation, a process with room for it answers from an + * exact map of the shards' values (built from their field indexes) and one without room from the filters; both + * answer as a scan, also for values only newer lines hold and for a shard another process compacted + */ + public function warmLookupsUseAnExactMapWhenMemoryAllowsAndTheFiltersElse(): void + { + $config = ['shardSize' => 100]; + $db = $this->db($config); + $db->insert('hm', array_map(function ($n) { return ['u' => 'u' . $n, 'g' => 'g' . ($n % 9)]; }, range(0, 799))); + $db->createFieldIndex('hm', 'u'); + $gfx = $this->dbPath('hm', 'gfx', 'u'); + $values = ['u0', 'u99', 'u100', 'u450', 'u799', 'none', 'u-1']; + foreach (['an exact map' => 0, 'the filters' => memory_get_usage() + 1048576] as $label => $limit) { + \noneDB::clearStaticCache(); + $a = $this->db($config + ['cacheMemoryLimit' => $limit]); + $scan = $this->db($config + ['fieldIndexEnabled' => false, 'cacheMemoryLimit' => $limit]); + $this->assertSame([7], array_column($a->find('hm', ['u' => 'u7']), 'key'), "$label: first operation"); + $this->assertArrayNotHasKey($gfx . '#hot', Testing::get('staticFieldIndexCache'), "$label: no map at the first operation"); + foreach ($values as $value) { + $this->assertSame(array_column($scan->find('hm', ['u' => $value]), 'key'), array_column($a->find('hm', ['u' => $value]), 'key'), "$label: $value"); + } + $this->assertSame($label === 'an exact map', is_array(Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null), "$label: the map"); + // A value only newer lines hold (another process wrote it, no checkpoint), and a shard compacted by another process + $this->assertSame('{"n":1}', $this->runChild('$db = ' . $this->childDb($config) . '; echo json_encode($db->update("hm", [["key" => 5], ["set" => ["u" => "moved"]]]));')); + $this->assertSame([5], array_column($a->find('hm', ['u' => 'moved']), 'key'), "$label: a newer line"); + $this->assertSame([], $a->find('hm', ['u' => 'u5']), "$label: replaced by the newer line"); + $this->assertSame('true', $this->runChild('$db = ' . $this->childDb($config) . '; echo json_encode($db->compact("hm")["success"]);')); + foreach (array_merge($values, ['moved', 'u5']) as $value) { + $this->assertSame(array_column($scan->find('hm', ['u' => $value]), 'key'), array_column($a->find('hm', ['u' => $value]), 'key'), "$label, compacted: $value"); + } + $this->assertSame(1, $this->db($config)->update('hm', [['key' => 5], ['set' => ['u' => 'u5']]])['n']); + } + } + + /** + * @test §6.5.2: a global index is made from the shards in parts of whole pages, each within a quarter of the + * cache limit (docs/CONFIGURATION.md "Memory of large databases"). The file made in four parts is the file made + * at once, byte for byte after the write token; lookups answer as a scan. + */ + public function aGlobalIndexIsMadeInPartsUnderACacheLimit(): void + { + // 20 shards of 100 unique values: 4096 bits, width 32 (4-byte rows), pages of 1024 rows (4 KiB) + $config = ['shardSize' => 100]; + $this->db($config)->insert('gp', array_map(function ($n) { return ['u' => 'u' . $n]; }, range(0, 1999))); + $this->db($config)->createFieldIndex('gp', 'u'); + $gfx = $this->dbPath('gp', 'gfx', 'u'); + $made = []; + foreach (['at once' => 0, 'in parts' => 16384] as $label => $limit) { // 16 KiB: a part of one page + unlink($gfx); + \noneDB::clearStaticCache(); + $db = $this->db($config + ['cacheMemoryLimit' => $limit]); + $this->assertSame([1234], array_column($db->find('gp', ['u' => 'u1234']), 'key'), "$label: the lookup that makes it"); + [$header] = $this->readGlobalFieldFile($gfx); + $this->assertSame([4096, 32], [$header['bits'], $header['width']], "precondition, $label"); + $raw = (string)file_get_contents($gfx); + $made[$label] = [array_diff_key($header, ['w' => 0]), substr($raw, strpos($raw, "\n"))]; + } + $this->assertSame($made['at once'], $made['in parts']); + $scan = $this->db($config + ['fieldIndexEnabled' => false]); + foreach (['u0', 'u99', 'u100', 'u1999', 'none'] as $value) { + $this->assertSame(array_column($scan->find('gp', ['u' => $value]), 'key'), array_column($db->find('gp', ['u' => $value]), 'key'), $value); + } + } + + /** + * @test §6.5.1, §6.5.2 "Exact map": a shard's field index file that doesn't fit its data (its h is not the crc of + * the bytes it says it covers) is never used; the exact map gives that shard no entry and the shard is read. + * Within one lock epoch the map's entries are trusted as they are, so a file taken in would rule the shard + * out for a value the file lacks. + */ + public function anExactMapLeavesOutAShardFileThatDoesNotFitItsData(): void + { + $config = ['shardSize' => 100]; + $db = $this->db($config); + $db->insert('nf', array_map(function ($n) { return ['u' => 'u' . $n]; }, range(0, 399))); // 4 shards + $db->createFieldIndex('nf', 'u'); + // Shard 2's file without u250 (counters to match) and with another h: it doesn't fit the data + $fx = $this->dbPath('nf', 'fx', 'u', 2); + [$header, $map] = $this->readFieldIndexFile($fx); + unset($map[$this->bucketIdOf('u250')]); + $header['h'] = substr($header['h'], 0, -1) . ($header['h'][7] === '0' ? '1' : '0'); + $header['c']--; + $header['n']--; + $this->writeFieldIndexFile($fx, $header, $map); + $gfx = $this->dbPath('nf', 'gfx', 'u'); + \noneDB::clearStaticCache(); + $a = $this->db($config + ['cacheMemoryLimit' => 0]); + $this->assertSame([7], array_column($a->find('nf', ['u' => 'u7']), 'key'), 'first operation: the filters'); + // The map is built without shard 2, so the lookup reads shard 2, whose file is made again from the data + $this->assertSame([150], array_column($a->find('nf', ['u' => 'u150']), 'key'), 'second operation: the map is built'); + $hot = Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null; + $this->assertIsArray($hot, 'precondition: an exact map'); + [$made] = $this->readFieldIndexFile($fx); + $this->assertNotSame($header['h'], $made['h'], 'shard 2\'s file was made again'); + $this->assertSame(['s' => $made['s'], 'e' => $made['e'], 'h' => $made['h']], $hot['sh'][2], 'the map\'s entry for shard 2 is the new file\'s'); + $this->assertSame([250], array_column($a->find('nf', ['u' => 'u250']), 'key'), 'a value the old file lacked'); + } + + /** + * @test §6.5.2 "Exact map": the map is of the bucket ids of the shards' files. A file whose buckets this process + * holds exactly as they are in it (read whole, nothing changed since) is not decoded again; the others are, a + * held index that changed since its file too. The map and its entries are the same in every case. + */ + public function anExactMapTakesTheBucketsThisProcessHoldsOfAFile(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; // room for the map whatever the process holds + $this->db($config)->insert('mh', array_map(function ($n) { return ['g' => 'g' . ($n % 7) . '-' . intdiv($n, 100)]; }, range(0, 299))); + $this->db($config)->createFieldIndex('mh', 'g'); + $gfx = $this->dbPath('mh', 'gfx', 'g'); + $model = []; + $entries = []; + foreach ([0, 1, 2] as $shard) { + [$header, $map] = $this->readFieldIndexFile($this->dbPath('mh', 'fx', 'g', $shard)); + foreach (array_keys($map) as $id) { + $model[$id] = $shard; + } + $entries[$shard] = ['s' => $header['s'], 'e' => $header['e'], 'h' => $header['h']]; + } + foreach (['none held' => [[], false], 'shard 1 held' => [[1], false], 'shard 1 held, changed since' => [[1], true]] as $label => [$held, $changed]) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(300, $db->count('mh'), $label); // the process's first operation + foreach ($held as $shard) { + // a file of at most 64 KiB is read whole at its first use after the first operation (§6.5.1) + $this->assertCount(14, $db->getKeysFromFieldIndex('mh', 'g', 'g0-' . $shard, $shard), "$label: shard $shard's index"); + } + if ($changed) { + // a new value in shard 1 (too little for a new checkpoint), which the held index takes at its next read + $this->assertSame(1, $db->update('mh', [['key' => 150], ['set' => ['g' => 'extra']]])['n'], $label); + $this->assertSame([150], $db->getKeysFromFieldIndex('mh', 'g', 'extra', 1), $label); + } + $decoded = Testing::get('indexStats')['bodiesDecoded']; + // A value no shard holds: the filters, then the map, answer without reading any shard + $this->assertSame([], $db->find('mh', ['g' => 'none']), $label); + $this->assertSame([], $db->find('mh', ['g' => 'none']), "$label: the map is built"); + $hot = Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null; + $this->assertIsArray($hot, "$label: the map"); + $this->assertSame([$model, $entries], [$hot['map'], $hot['sh']], $label); + $this->assertSame($changed ? 3 : 3 - count($held), Testing::get('indexStats')['bodiesDecoded'] - $decoded, "$label: bodies decoded for it"); + } + } + + /** + * @test §6.5.2 "Exact map": the file's sizes bound the map (at most `bits / 32` values per shard entry, 32 bytes a + * pair). A map whose bound is over the room left under half of `cacheMemoryLimit` is not built, and no shard's + * `.fx` is read for it: a process never spends seconds reading files for a map it then drops. Answers stay a scan's. + */ + public function anExactMapThatCannotFitIsNotBuiltAndReadsNoShard(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + // 8 shards, every value in every shard: 100 per shard (bits 4096: at most 8 × 128 pairs, 32 KiB) + $this->db($config)->insert('nf', array_map(function ($n) { return ['g' => 'g' . ($n % 100)]; }, range(0, 799))); + $this->db($config)->createFieldIndex('nf', 'g'); + $gfx = $this->dbPath('nf', 'gfx', 'g'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(800, $db->count('nf')); // the process's first operation + try { + // Room for 16 KiB at the global index's second operation: half the limit is this process's memory and 16 KiB + Testing::set('cacheMemoryLimit', 2 * (memory_get_usage() + 16384)); + $decoded = Testing::get('indexStats')['bodiesDecoded']; + $this->assertSame([], $db->find('nf', ['g' => 'none'])); + $this->assertSame([], $db->find('nf', ['g' => 'none']), 'the second operation of the global index'); + $this->assertSame(0, Testing::get('indexStats')['bodiesDecoded'] - $decoded, 'no shard read for a map that cannot fit'); + $this->assertNull(Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null, 'no map'); + $this->assertSame(range(7, 799, 100), array_column($db->find('nf', ['g' => 'g7']), 'key'), 'the answer'); + } finally { + Testing::set('cacheMemoryLimit', null); + } + } + + /** + * @test §6.5.2 "Exact map": a process keeps the map only with room for it once it is built. The file's sizes bound + * it at about 32 bytes a pair, which long values pass by far, so the bound alone doesn't keep that promise: the + * build stops at the shard that fills the room, reads no further shard and leaves no map. Answers stay a scan's. + */ + public function anExactMapThatOutgrowsItsRoomIsDroppedWhileItIsBuilt(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + $value = function (int $n): string { return str_pad('v' . $n, 2048, '.'); }; + // 8 shards of 100 values of 2 KiB: the bound is 8 × 4096 bytes, one shard's values alone are 200 KiB + $this->db($config)->insert('ov', array_map(function ($n) use ($value) { return ['v' => $value($n)]; }, range(0, 799))); + $this->db($config)->createFieldIndex('ov', 'v'); + $gfx = $this->dbPath('ov', 'gfx', 'v'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(800, $db->count('ov')); // the process's first operation + try { + // Room for 64 KiB at the global index's second operation: the bound of 32 KiB fits + Testing::set('cacheMemoryLimit', 2 * (memory_get_usage() + 65536)); + $decoded = Testing::get('indexStats')['bodiesDecoded']; + $this->assertSame([], $db->find('ov', ['v' => 'none'])); + $this->assertSame([], $db->find('ov', ['v' => 'none']), 'the second operation of the global index'); + $this->assertSame(1, Testing::get('indexStats')['bodiesDecoded'] - $decoded, 'the first shard fills the room, no other is read'); + $cache = Testing::get('staticFieldIndexCache'); + $this->assertArrayHasKey($gfx . '#hot', $cache, 'the build ran'); + $this->assertNull($cache[$gfx . '#hot'], 'no map'); + foreach ([0, 99, 100, 450, 799] as $n) { + $this->assertSame([$n], array_column($db->find('ov', ['v' => $value($n)]), 'key'), "the answer for $n"); + } + } finally { + Testing::set('cacheMemoryLimit', null); + } + } + + /** + * @test §6.5.2 "Exact map": made only for a database of at most a million records. Making it reads every shard's + * `.fx` whole, which takes seconds at ten million (in a process's second operation: a web request with two queries). + * Above the limit the filters answer and nothing is read for a map. Shown with the limit lowered (a test seam): 800 + * records over a limit of 799, then at one of 800 + */ + public function anExactMapIsMadeOnlyForADatabaseOfAtMostAMillionRecords(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + $this->db($config)->insert('mr', array_map(function ($n) { return ['g' => 'g' . ($n % 7) . '-' . intdiv($n, 100)]; }, range(0, 799))); + $this->db($config)->createFieldIndex('mr', 'g'); + $gfx = $this->dbPath('mr', 'gfx', 'g'); + try { + foreach ([799 => false, 800 => true] as $limit => $made) { + \noneDB::clearStaticCache(); + Testing::set('exactMapRecords', $limit); + $db = $this->db($config); + $this->assertSame(800, $db->count('mr'), "limit $limit"); + $decoded = Testing::get('indexStats')['bodiesDecoded']; + $this->assertSame([], $db->find('mr', ['g' => 'none'])); + $this->assertSame([], $db->find('mr', ['g' => 'none']), "limit $limit: the second operation of the global index"); + $this->assertSame($made, is_array(Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null), "limit $limit: the map"); + if (!$made) { + $this->assertSame(0, Testing::get('indexStats')['bodiesDecoded'] - $decoded, "limit $limit: nothing read for a map"); + } + $this->assertSame(range(0, 99, 7), array_column($db->find('mr', ['g' => 'g0-0']), 'key'), "limit $limit: the answer"); + } + } finally { + Testing::set('exactMapRecords', null); + } + } + + /** + * @test §6.5.2 "Exact map": made only in PHP's command line. A web server's request ends with its statics, so a map + * made for it dies before it can pay: the second indexed query of a request reads no shard's `.fx` for one. Every other + * SAPI behaves the same; 'cli' makes it. The answers don't depend on it. Shown with the SAPI set by a test seam + */ + public function aProcessOfAWebServerMakesNoExactMap(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + $this->db($config)->insert('sa', array_map(function ($n) { return ['g' => 'g' . ($n % 7) . '-' . intdiv($n, 100)]; }, range(0, 799))); + $this->db($config)->createFieldIndex('sa', 'g'); + $gfx = $this->dbPath('sa', 'gfx', 'g'); + try { + foreach (['fpm-fcgi' => false, 'apache2handler' => false, 'cgi-fcgi' => false, 'cli-server' => false, 'frankenphp' => false, 'cli' => true] as $sapi => $made) { + \noneDB::clearStaticCache(); + Testing::set('sapi', $sapi); + $db = $this->db($config); + $this->assertSame(800, $db->count('sa'), $sapi); + $decoded = Testing::get('indexStats')['bodiesDecoded']; + $this->assertSame([], $db->find('sa', ['g' => 'none'])); + $this->assertSame([], $db->find('sa', ['g' => 'none']), "$sapi: the second operation of the global index"); + $this->assertSame($made, is_array(Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null), "$sapi: the map"); + if (!$made) { + $this->assertSame(0, Testing::get('indexStats')['bodiesDecoded'] - $decoded, "$sapi: nothing read for a map"); + } + $this->assertSame(range(0, 99, 7), array_column($db->find('sa', ['g' => 'g0-0']), 'key'), "$sapi: the answer"); + } + } finally { + Testing::set('sapi', null); + } + } + + /** + * @test §6.5.2 "Lookup": a process with no map to come, holding the file's body, answers lookups from the body, in + * every case as a scan: values of one shard, of many, of none, written later by this process and by another one. A + * filter this session replaced (`cols`) goes before the body's bits. A process whose map is still possible (its + * second operation was a join's) doesn't take this way: the body held is no answer to "no map" + */ + public function aHeldBodyAnswersLookupsAsAScanWhenNoMapWillCome(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + $db = $this->db($config); + $db->insert('hb', array_map(function ($n) { return ['u' => 'u' . $n, 'g' => 'g' . ($n % 300)]; }, range(0, 799))); + $db->createFieldIndex('hb', 'u'); + $db->createFieldIndex('hb', 'g'); + $gfx = $this->dbPath('hb', 'gfx', 'u'); + $values = [['u', 'u0'], ['u', 'u450'], ['u', 'none'], ['g', 'g7'], ['g', 'g299'], ['g', 'none']]; + try { + \noneDB::clearStaticCache(); + Testing::set('sapi', 'fpm-fcgi'); + $a = $this->db($config); + $scan = $this->db($config + ['fieldIndexEnabled' => false]); + $a->find('hb', ['u' => 'u7']); + $a->find('hb', ['u' => 'u8']); + $held = Testing::get('staticFieldIndexCache')[$gfx] ?? null; + $this->assertArrayHasKey('body', (array)$held, 'precondition: the body is held from the second operation'); + $before = Testing::get('indexStats')['bodyLookups']; + $this->assertSame([9], array_column($a->find('hb', ['u' => 'u9']), 'key')); + $this->assertSame(1, Testing::get('indexStats')['bodyLookups'] - $before, 'the lookup was answered from the body'); + $points = Testing::get('indexStats')['pointLookups']; + $this->assertSame([], $a->find('hb', ['u' => 'none'])); + $this->assertSame($points, Testing::get('indexStats')['pointLookups'], 'a miss answered from the body reads no index'); + foreach ($values as [$field, $value]) { + $this->assertSame(array_column($scan->find('hb', [$field => $value]), 'key'), array_column($a->find('hb', [$field => $value]), 'key'), "$field $value"); + } + // Written by another process, then by this one: newer lines, no checkpoint + $this->assertSame('{"n":1}', $this->runChild('$db = ' . $this->childDb($config) . '; echo json_encode($db->update("hb", [["key" => 5], ["set" => ["u" => "theirs"]]]));')); + $this->assertSame([5], array_column($a->find('hb', ['u' => 'theirs']), 'key'), 'a line of another process'); + $this->assertSame(1, $a->update('hb', [['key' => 6], ['set' => ['u' => 'mine']]])['n']); + $this->assertSame([6], array_column($a->find('hb', ['u' => 'mine']), 'key'), 'a line of this process'); + // A filter of this session: the value is in shard 3 by `cols` alone + \noneDB::clearStaticCache(); + $a = $this->db($config); + $a->find('hb', ['u' => 'u7']); + $a->find('hb', ['u' => 'u8']); + $this->assertSame([0], Testing::call($a, 'targetShardsFromGlobalIndex', 'hb', ['u' => 'u8'])); + $cache = Testing::get('staticFieldIndexCache'); + $this->assertArrayHasKey('body', $cache[$gfx], 'precondition: the body is held'); + $cache[$gfx]['cols'][3] = $this->shardFilterOfIds([$this->bucketIdOf('u8')], $cache[$gfx]['bits']); + Testing::set('staticFieldIndexCache', $cache); + $this->assertSame([0, 3], Testing::call($a, 'targetShardsFromGlobalIndex', 'hb', ['u' => 'u8']), 'the filter of this session'); + } finally { + Testing::set('sapi', null); + } + // A second operation that builds nothing (as checkIntegrity() and a snapshot run) loads the body and decides + // nothing about the map: the next operation makes it + \noneDB::clearStaticCache(); + $a = $this->db($config); + $a->find('hb', ['u' => 'u7']); + Testing::set('noWrites', 1); + try { + $this->assertSame([8], array_column($a->find('hb', ['u' => 'u8']), 'key')); + } finally { + Testing::set('noWrites', 0); + } + $cache = Testing::get('staticFieldIndexCache'); + $this->assertArrayHasKey('body', $cache[$gfx], 'precondition: the body is held'); + $this->assertArrayNotHasKey($gfx . '#hot', $cache, 'precondition: nothing decided about the map'); + $this->assertSame([9], array_column($a->find('hb', ['u' => 'u9']), 'key')); + $this->assertIsArray(Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null, 'a body held does not stop the map'); + } + + /** + * @test §6.5.2 "Sizes": `bits` follows the largest `c` of the shards' `.fx`, which their headers hold. Building the + * global index reads those headers for it and decodes each shard's file once (for its bucket ids), also when the + * cache keeps none of them (a limit of 1 byte); before, each was decoded twice + */ + public function theGlobalIndexDecodesEachShardOnce(): void + { + $db = $this->db(['shardSize' => 100, 'cacheMemoryLimit' => 1]); + $db->insert('go', array_map(function ($n) { return ['g' => 'g' . ($n % 50)]; }, range(0, 399))); + $decoded = Testing::get('indexStats')['bodiesDecoded']; + $this->assertTrue($db->createFieldIndex('go', 'g')['success']); + $this->assertSame(4, Testing::get('indexStats')['bodiesDecoded'] - $decoded, 'four shards, one decode each'); + [$header, $filters] = $this->readGlobalFieldFile($this->dbPath('go', 'gfx', 'g')); + // 50 values in every shard: the smallest power of two of at least 32 × 50 + $this->assertSame([2048, 8], [$header['bits'], $header['width']]); + $ids = array_map(function ($n) { return $this->bucketIdOf('g' . $n); }, range(0, 49)); + foreach ([0, 1, 2, 3] as $shard) { + $this->assertSame($this->shardFilterOfIds($ids, 2048), $filters[$shard], "shard $shard's filter"); + // Its entry describes the shard's .fx: that file's s, e and h + [$fx] = $this->readFieldIndexFile($this->dbPath('go', 'fx', 'g', $shard)); + $this->assertSame(['s' => $fx['s'], 'e' => $fx['e'], 'h' => $fx['h']], $header['sh'][$shard], "shard $shard's entry"); + } + foreach (['g7', 'g49', 'none'] as $value) { + $this->assertSame($value === 'none' ? [] : range((int)substr($value, 1), 399, 50), array_column($db->find('go', ['g' => $value]), 'key'), $value); + } + } + + /** + * @test §6.5.2 "Exact map": when this process replaces a shard's filter, its map takes the shard's new entry + * and values; a value the shard no longer holds may stay listed. Answers stay those of a scan, in key order. + */ + public function anExactMapFollowsTheShardsThisProcessUpdates(): void + { + $config = ['shardSize' => 100, 'cacheMemoryLimit' => 0]; + $db = $this->db($config); + $db->insert('hm', array_map(function ($n) { return ['u' => 'u' . $n]; }, range(0, 799))); + $db->createFieldIndex('hm', 'u'); + $gfx = $this->dbPath('hm', 'gfx', 'u'); + \noneDB::clearStaticCache(); + $a = $this->db($config); + $scan = $this->db($config + ['fieldIndexEnabled' => false]); + $a->find('hm', ['u' => 'u7']); + $a->find('hm', ['u' => 'u8']); + $map = function () use ($gfx) { return Testing::get('staticFieldIndexCache')[$gfx . '#hot'] ?? null; }; + $this->assertIsArray($map(), 'precondition: the map exists from the second operation'); + // u450 (shard 4) gains shards 1 and 0, u700 (shard 7) gains shard 2; the old values leave shards 0-2 + foreach ([150 => 'u450', 5 => 'u450', 250 => 'u700'] as $key => $value) { + $this->assertSame(1, $a->update('hm', [['key' => $key], ['set' => ['u' => $value]]])['n']); + } + $this->assertTrue($a->compact('hm')['success']); + $values = ['u450', 'u700', 'u5', 'u150', 'u250', 'u0', 'u799', 'none']; + foreach ([1, 2] as $round) { + foreach ($values as $value) { + $this->assertSame(array_column($scan->find('hm', ['u' => $value]), 'key'), array_column($a->find('hm', ['u' => $value]), 'key'), "round $round: $value"); + } + } + $hot = $map(); + $this->assertIsArray($hot, 'the map stays'); + foreach ([0, 1, 2] as $shard) { + $header = $this->readFieldIndexFile($this->dbPath('hm', 'fx', 'u', $shard))[0]; + $this->assertSame(['s' => $header['s'], 'e' => $header['e'], 'h' => $header['h']], $hot['sh'][$shard], "shard $shard: the new entry"); + } + $this->assertSame([0, 1, 4], $hot['map'][$this->bucketIdOf('u450')], 'u450: its holders, in shard order'); + $this->assertSame([2, 7], $hot['map'][$this->bucketIdOf('u700')], 'u700: its holders, in shard order'); + $this->assertSame(0, $hot['map'][$this->bucketIdOf('u5')], 'u5: still listed for shard 0 (read for it, answers nothing)'); + } + + /** + * @test §6.5.1 "Whole read": a segment index file above 64 KiB in repeated use is loaded whole once its + * operations paid for the load (at least two, one per 20 entries), never before, in one segment or in a + * shard of many; a process without room under its cache limit keeps the pages + */ + public function aPagedIndexIsLoadedWholeOnceItsLookupsPayForIt(): void + { + foreach (['one segment' => ['shardingEnabled' => false], '20 shards' => ['shardSize' => 400]] as $layout => $base) { + $this->cleanTestDirectory(); + $sharded = isset($base['shardSize']); + $db = $this->db($base + ['cacheMemoryLimit' => 0]); + // 8000 records, 400 entries: due at the 20th operation (20 × 20 >= 400). One segment: 400 values + // of 20 keys each. 20 shards: every value its own, so shard 3 (keys 1200-1599) holds 400 entries. + // Values of about 200 bytes put each file above 64 KiB (a smaller one is loaded at its first use). + $pad = str_repeat('-', 200); + for ($start = 0; $start < 8000; $start += 2000) { // in batches: the suite's memory + $db->insert('fw', array_map(function ($i) use ($sharded, $pad) { + return ['c' => ($sharded ? 'c' . $i : 'c' . ($i % 400)) . $pad]; + }, range($start, $start + 1999))); + } + $db->createFieldIndex('fw', 'c'); + $fx = $sharded ? $this->dbPath('fw', 'fx', 'c', 3) : $this->dbPath('fw', 'fx', 'c'); + $this->assertSame(400, $this->readFieldIndexFile($fx)[0]['c']); + clearstatcache(); + $this->assertGreaterThan(65536, filesize($fx), "precondition, $layout: read by pages"); + $due = 20; + foreach ([true, false] as $room) { + \noneDB::clearStaticCache(); + // The setting is process-wide: each instance sets it (0: no limit) + $db = $this->db($base + ['cacheMemoryLimit' => $room ? 0 : memory_get_usage() + 8 * 1048576]); + $label = ($room ? 'with room' : 'no room') . ", $layout"; + $loaded = function () use (&$db, $fx) { return is_array(Testing::getOf($db, 'fieldIndexCache')[$fx] ?? null); }; + for ($op = 1; $op <= $due + 1; $op++) { + if ($sharded) { + $value = 'c' . (1200 + $op) . $pad; + $expected = [1200 + $op]; + } else { + $value = 'c' . $op . $pad; + $expected = range($op, 7999, 400); + } + $this->assertSame($expected, array_column($db->find('fw', ['c' => $value]), 'key'), "$label: operation $op"); + $this->assertSame($room && $op >= $due, $loaded(), "$label: loaded whole after operation $op"); + } + } + } + } + + /** + * @test §6.5.1 "Body kept as bytes": an .fx above 64 KiB whose body is at most 256 KiB is read whole at a + * page miss from its second operation (after the header check), and every later page comes from those + * bytes, crc checked at its first use. The process's first operation, a larger body and a process without + * room read pages; a replaced file is never read at the old offsets. None of it loads the index whole. + */ + public function aBodyInUseIsReadWholeAsBytes(): void + { + $pad = str_repeat('-', 40); + $value = function (int $i) use ($pad): string { return 'v' . $i . $pad; }; + // One key per value (2000 entries: the rent-or-buy load waits for 100 operations) + $build = function (int $count) use ($value): string { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]); + for ($start = 0; $start < $count; $start += 2000) { // in batches: the suite's memory + $db->insert('fb', array_map(function ($i) use ($value) { return ['c' => $value($i)]; }, range($start, min($count, $start + 2000) - 1))); + } + $db->createFieldIndex('fb', 'c'); + return $this->dbPath('fb', 'fx', 'c'); + }; + $bodyOf = function (string $fx): string { + $raw = (string)file_get_contents($fx); + return substr($raw, strpos($raw, "\n") + 1); + }; + $member = function (int $i) use ($value): string { return json_encode('s' . $value($i)) . ':' . $i; }; + // One find per operation, in a process that starts now: [keys, pages read, bodies read] of each. Each + // must leave the index unloaded, or with $whole, whether it did goes there. + $run = function (array $values, $limit, ?array &$whole = null) use ($value): array { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => $limit === 'room' ? 0 : memory_get_usage() + 8 * 1048576]); + $fx = $this->dbPath('fb', 'fx', 'c'); + $steps = []; + foreach ($values as $step => $i) { + if (is_callable($i)) { + $i(); // between two operations + continue; + } + $before = $this->stats(); + $keys = array_column($db->find('fb', ['c' => $value($i)]), 'key'); + $after = $this->stats(); + $steps[] = [$keys, $after['pagesRead'] - $before['pagesRead'], $after['bodiesRead'] - $before['bodiesRead']]; + $loaded = is_array(Testing::getOf($db, 'fieldIndexCache')[$fx] ?? null); + if ($whole === null) { + $this->assertFalse($loaded, "operation $step: not loaded whole"); + } else { + $whole[] = $loaded; + } + } + return $steps; + }; + + $fx = $build(2000); + [$header] = $this->readFieldIndexFile($fx); + $body = $bodyOf($fx); + $this->assertGreaterThan(65536, filesize($fx), 'precondition: read by pages'); + $this->assertLessThanOrEqual(262144, strlen($body), 'precondition: a body of at most 256 KiB'); + $this->assertSame(1, $header['pg']['levels'], 'precondition: data pages under the root'); + $at = array_map(function ($i) use ($body, $member) { return strpos($body, $member($i)); }, [7, 1500, 1999, 1000]); + sort($at); + for ($n = 1; $n < count($at); $n++) { + $this->assertGreaterThan(8192, $at[$n] - $at[$n - 1], 'precondition: the values lie on four different data pages'); + } + + // The first operation: the root and a data page. The second: the body, once. Then nothing is read: a + // page of the body, the page kept before it was read, a value that isn't there. + $this->assertSame([[[7], 2, 0], [[1500], 0, 1], [[1999], 0, 0], [[1000], 0, 0], [[7], 0, 0], [[], 0, 0]], + $run([7, 1500, 1999, 1000, 7, 2500], 'room'), 'with room'); + // Without room: a page per miss, never the body + $this->assertSame([[[7], 2, 0], [[1500], 1, 0], [[1999], 1, 0], [[7], 0, 0]], + $run([7, 1500, 1999, 7], 'no room'), 'no room'); + + // A damaged page inside the body: its crc is checked at its first use, so the index is never trusted + // (the data answers, the file is written again) + $damaged = function () use ($fx, $member): void { + $raw = (string)file_get_contents($fx); + $at = strpos($raw, $member(1999)); + $this->assertNotFalse($at); + $raw = substr_replace($raw, substr($member(1999), 0, -1) . '8', $at, strlen($member(1999))); // :1999 → :1998 + file_put_contents($fx, $raw); + }; + $whole = []; + $steps = $run([7, $damaged, 1500, 1999, 1000], 'room', $whole); + $this->assertSame([false, false], array_slice($whole, 0, 2), 'damaged: pages until the damage is found'); + $this->assertSame([[7], 0], [$steps[0][0], $steps[0][2]], 'damaged: first operation'); + $this->assertSame([[1500], 1], [$steps[1][0], $steps[1][2]], 'damaged: the body read, its page of 1500 sound'); + $this->assertSame([1999], $steps[2][0], 'damaged: the page of 1999 is never trusted'); + $this->assertSame([1000], $steps[3][0]); + $this->assertSame($this->modelMap([$this->dbPath('fb', 'data')], 'c'), $this->readFieldIndexFile($fx)[1], 'damaged: written again'); + + // A file replaced between two operations (another write token, the same offsets): its header doesn't + // match the one held, so nothing of it is read at the old offsets + $fx = $build(2000); + $replaced = function () use ($fx): void { + $raw = (string)file_get_contents($fx); + $line = (string)strstr($raw, "\n", true); + $header = $this->openHeaderLine($fx, $line); + do { // the same length: the same offsets + $header['w'] = bin2hex(random_bytes(8)); + $new = $this->sealHeaderLine(json_encode($header)); + } while (strlen($new) !== strlen($line)); + file_put_contents($fx, $new . substr($raw, strlen($line))); + }; + $steps = $run([7, $replaced, 1500, 1999, 1000, 7], 'room'); + $this->assertSame([[7], [1500], [1999], [1000], [7]], array_column($steps, 0), 'replaced: every answer'); + $this->assertSame(0, $steps[1][2], 'replaced: the old offsets are never read from the new file'); + + // A body above 256 KiB: pages, one per level at the first lookup, the missing ones of each later path + $fx = $build(6000); + $this->assertGreaterThan(262144, strlen($bodyOf($fx)), 'precondition: a body above 256 KiB'); + $levels = $this->readFieldIndexFile($fx)[0]['pg']['levels']; + $steps = $run([7, 1500, 5999, 7], 'room'); + $this->assertSame([[7], [1500], [5999], [7]], array_column($steps, 0), 'larger body: every answer'); + $this->assertSame([0, 0, 0, 0], array_column($steps, 2), 'larger body: never the body'); + $this->assertSame($levels + 1, $steps[0][1], 'larger body: one page per level'); + $this->assertGreaterThanOrEqual(1, min($steps[1][1], $steps[2][1]), 'larger body: a page per miss'); + $this->assertSame(0, $steps[3][1], 'larger body: the kept pages'); + } + + /** + * @test §6.5.1 lookup step 3: a name is found only where a member starts (after `{"` or `,"`), never inside + * another name or a value that holds its bytes: names with quotes, backslashes, commas, braces, colons, + * control characters, line terminators and other names' text, in the pages of a process with room (the + * body kept as bytes from the second operation) and without (pages) + */ + public function namesThatLookLikeOtherMembersAreNeverMistaken(): void + { + $special = ['a', 'sa', 'ssa', 'a":1,"sa', 'x,"sa":', '{"sa":', 'sa":', '"', '""', '"a', '\\', '\\"', 'a\\', 'a\\"', + '",', '","sa":', "\n", "a\nb", "\0", "\x1f", "\u{2028}", 'ş', '/', 'a/b', '[1,2]', '1', ':', '"sa":[1]', + 'i1', 'n', "\\u0000", 'sa":[3,4],"sb', + // Right before "b" in bucket id order, on its page: its name's bytes after an escaped quote, after a + // comma and a letter + 'b', 'a"sb', 'a,ysb']; + $values = array_merge($special, $special, [1, -1, true, null]); // two keys each: lists of keys + // 5000 more entries: two rounds of probes (two operations each) stay below the whole load (c / 20) + for ($i = 0; $i < 5000; $i++) { + $values[] = sprintf('filler-%04d-', $i) . str_repeat('-', 10); + } + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]); + $db->insert('fe', array_map(function ($v) { return ['v' => $v]; }, $values)); + $db->createFieldIndex('fe', 'v'); + $fx = $this->dbPath('fe', 'fx', 'v'); + clearstatcache(); + $this->assertGreaterThan(65536, filesize($fx), 'precondition: read by pages'); + $this->assertLessThanOrEqual(262144, filesize($fx), 'precondition: a body kept as bytes from the second operation'); + $this->assertSame($this->modelMap([$this->dbPath('fe', 'data')], 'v'), $this->readFieldIndexFile($fx)[1]); + $raw = (string)file_get_contents($fx); + $body = substr($raw, strpos($raw, "\n") + 1); + $firstPage = strpos($body, '}'); // no name here holds a brace: the end of the first page's object + foreach (['"sa\\"sb":', '"sa,ysb":', '"sb":'] as $member) { + $at = strpos($body, $member); + $this->assertTrue($at !== false && $at < $firstPage, "precondition: $member on the first page"); + } + $this->assertLessThan(strpos($body, '"sb":'), strpos($body, '"sa\\"sb":'), 'precondition: the decoys come first'); + // The model: the keys of each value (===), in insert order + $expected = function ($value) use ($values): array { return array_keys(array_filter($values, function ($v) use ($value) { return $v === $value; })); }; + $probes = array_merge($special, [1, -1, true, null, 'filler-0007-' . str_repeat('-', 10), '"sa', 'zz"', '\\\\', 'sa":1', ',"sa":', "\x1e"]); + foreach (['room' => 0, 'pages' => null] as $mode => $limit) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => $limit ?? memory_get_usage() + 8 * 1048576]); + $before = $this->stats()['bodiesRead']; + foreach ([1, 2] as $round) { + foreach ($probes as $value) { + $label = "$mode, round $round: " . json_encode($value); + $this->assertSame($expected($value), array_column($db->find('fe', ['v' => $value]), 'key'), $label); + $candidates = Testing::call($db, 'withLock', 'fe', false, function () use ($db, $value) { + return Testing::call($db, 'fieldIndexCandidates', 'fe', null, ['v' => $value]); + }, 'read'); + $this->assertSame($expected($value), $candidates, "$label (index candidates)"); + $this->assertFalse(is_array(Testing::getOf($db, 'fieldIndexCache')[$fx] ?? null), "$label: answered by pages"); + } + } + $this->assertSame($mode === 'room' ? 1 : 0, $this->stats()['bodiesRead'] - $before, "$mode: the body kept as bytes"); + } + } + + /** @test §6.5.1: an index written from the specification by the test is read as it is */ + public function anIndexWrittenFromTheSpecificationIsRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('fw', [['c' => 'a'], ['c' => 'b'], ['c' => 'a']]); + $db->createFieldIndex('fw', 'c'); + [$header] = $this->readFieldIndexFile($this->dbPath('fw', 'fx', 'c')); + // Fits the data (s, e, h) but says key 2 holds "b": the index is the index of the bytes it covers + $this->writeFieldIndexFile($this->dbPath('fw', 'fx', 'c'), $header, ['sa' => 0, 'sb' => [1, 2]]); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([1], array_column($db->find('fw', ['c' => 'b']), 'key'), 'candidates are checked against the records'); + $this->assertSame([0], array_column($db->find('fw', ['c' => 'a']), 'key'), 'key 2 is not a candidate of "a" any more'); + } + + /** + * The pages of a body, cut after the members are encoded at once, are those §6.5.1 describes, for every id an + * encoder can make hard to cut: ids that end in a comma or a backslash, hold quotes and colons, and a bucket + * larger than a page (under an id that ends in a comma too) + * @test + */ + public function pagesAreCutBetweenMembersWhateverTheIds(): void + { + $values = []; + for ($i = 0; $i < 700; $i++) { + foreach (["$i,", "a,\"b$i", "c\":d$i", "e$i\\", "f$i\\,", "g,\":$i", "ü$i,", "\"$i", ",\"$i\",\"", "h$i\":", "x\ny$i", "k/$i\u{2028}"] as $value) { + $values[] = $value; + } + } + foreach ([7, -3, 1.5, true, false, null, ''] as $value) { + $values[] = $value; + } + $rows = array_map(function ($value) { return ['v' => $value]; }, $values); + for ($i = 0; $i < 1500; $i++) { + $rows[] = ['v' => 'big,']; + $rows[] = ['v' => 'large']; + } + $header = $this->assertPagesAsTheFormatSays('pc', $rows, ['0,', '699,', 'a,"b5', 'c":d5', "e5\\", "f5\\,", 'g,":5', 'ü5,', '"5', ',"5","', 'h5":', "x\ny5", "k/5\u{2028}", 7, -3, 1.5, true, false, null, '', 'big,', 'large', 'none', 'none,']); + $this->assertGreaterThanOrEqual(1, $header['pg']['levels'], 'precondition: a directory'); + } + + /** + * Index one database of $rows and compare its .fx body with the same map written by the format's description, + * page after page; lookups answer as a scan does. Returns the header. + */ + private function assertPagesAsTheFormatSays(string $dbname, array $rows, array $values): array + { + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(count($rows), $db->insert($dbname, $rows)['n']); + $db->createFieldIndex($dbname, 'v'); + $fx = $this->dbPath($dbname, 'fx', 'v'); + [$header, $map] = $this->readFieldIndexFile($fx); + $model = $fx . '.model'; + $this->writeFieldIndexFile($model, $header, $map); + $this->assertSame($this->fxBody($model), $this->fxBody($fx)); + unlink($model); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $scan = $this->db(['shardingEnabled' => false, 'fieldIndexEnabled' => false]); + foreach ($values as $value) { + $this->assertSame(array_column($scan->find($dbname, ['v' => $value]), 'key'), array_column($db->find($dbname, ['v' => $value]), 'key'), json_encode($value)); + } + return $header; + } + + private function fxBody(string $path): string + { + $bytes = (string)file_get_contents($path); + return substr($bytes, strpos($bytes, "\n") + 1); + } + + /** + * Records of one distinct string each, ascending in id order and in key order from $key, whose .fx members + * `"s{value}":{key}` take exactly $bytes between a page's braces + */ + private function rowsOfBytes(int $bytes, string $prefix, int $key = 0): array + { + $rows = []; + $left = $bytes + 1; // no comma before the first + for (;; $key++) { + $name = $prefix . sprintf('%05d', $key); + $left -= strlen(',"s' . $name . '":' . $key); + $last = $left <= 100; + $pad = $last ? $left : 40; + $rows[] = ['v' => $name . str_repeat('.', $pad)]; + $left -= $pad; + if ($last) { + return $rows; + } + } + } + + /** + * Space in a data page: `{`, members, `}`, the crc: 4090 bytes of members fill it to its last byte + * @test + * @testWith [4090, 0, 4096] + * [4091, 1, 4102] + */ + public function theOnlyMembersFillAPageToItsLastByte(int $bytes, int $levels, int $data): void + { + $rows = $this->rowsOfBytes($bytes, 'a'); + $header = $this->assertPagesAsTheFormatSays('pf', $rows, array_column($rows, 'v')); + $this->assertSame($levels, $header['pg']['levels'], 'one data page needs no directory'); + $this->assertSame($data, $header['pg']['data']); + } + + /** + * With more members behind them: the page takes what fits to its last byte, and a byte too many leaves the + * member for the next page + * @test + * @testWith [4090] + * [4091] + */ + public function aPageIsCutAtTheLastMemberThatFits(int $bytes): void + { + $rows = $this->rowsOfBytes($bytes, 'a'); + $key = count($rows) - 1; + $lastMember = strlen(',"s' . $rows[$key]['v'] . '":' . $key); + $first = strlen('{') + ($bytes > 4090 ? $bytes - $lastMember : $bytes) + strlen('}') + 4; + $more = $this->rowsOfBytes(500, 'b', count($rows)); + $this->assertPagesAsTheFormatSays('pn', array_merge($rows, $more), array_column(array_merge($rows, $more), 'v')); + $body = $this->fxBody($this->dbPath('pn', 'fx', 'v')); + $this->assertSame('}', $body[$first - 5], 'the first page ends here'); + $this->assertSame('{', $body[$first], 'the next page starts here'); + $this->assertSame(pack('V', crc32(substr($body, 0, $first - 4))), substr($body, $first - 4, 4)); + } + + /** + * A member over a page as the last one, alone or behind others, is a page of its own and is not lost + * @test + * @testWith [0, 0] + * [40, 1] + */ + public function aLastMemberOverAPageIsAPageOfItsOwn(int $before, int $levels): void + { + $rows = []; + for ($i = 0; $i < $before; $i++) { + $rows[] = ['v' => 'a' . $i]; + } + for ($i = 0; $i < 1500; $i++) { + $rows[] = ['v' => 'zz']; + } + $header = $this->assertPagesAsTheFormatSays('pl', $rows, ['a0', 'a39', 'zz', 'none']); + $this->assertSame($levels, $header['pg']['levels'], 'one data page needs no directory'); + $this->assertGreaterThan(4090, strlen('"szz":' . json_encode(range($before, $before + 1499))), 'precondition: the member is over a page'); + } + + /** + * Lookups of a new process, through the file (the process that built an index holds it): the keys of each value + * @param array $values key => value + */ + private function assertANewProcessFinds(string $db, string $field, array $values): void + { + $out = $this->runChild('$db = ' . $this->childDb() . '; $seen = [];' + . ' foreach (' . var_export($values, true) . ' as $key => $value) {' + . ' $seen[$key] = array_column($db->find(' . var_export($db, true) . ', [' . var_export($field, true) . ' => $value]), "key"); }' + . ' echo json_encode($seen, JSON_FORCE_OBJECT), "\n", \noneDB\Tests\Support\Testing::get("indexStats")["pagesRead"];'); + $expected = []; + foreach ($values as $key => $value) { + $expected[$key] = [$key]; + } + [$seen, $pages] = array_pad(explode("\n", $out), 2, 0); + $this->assertSame($expected, json_decode($seen, true), substr($out, 0, 300)); + $this->assertGreaterThan(0, (int)$pages, 'answered by the file: a scan would give the same keys'); + } + + /** + * @test A string value has no length limit, so neither has a bucket id: ids over 64 KiB, side by side, one + * sharing 70 000 bytes with the next, are found through directory pages that pass 64 KiB themselves + * (§6.5.1: u32 lengths and starts; u16 ones made every lookup of the field fail in every new process) + * @runInSeparateProcess + */ + public function idsLongerThanSixtyFourKiBAreFoundThroughTheDirectory(): void + { + $values = []; + for ($i = 0; $i < 1500; $i++) { + $values[] = sprintf('short-%04d', $i); + } + $shared = str_repeat('x', 70000); + foreach (['A', 'B', 'C', 'D', 'E', 'F'] as $letter) { + $values[] = $letter . $shared . 'one'; + $values[] = $letter . $shared . 'two'; + $values[] = 'z' . $letter . str_repeat('y', 66000); + $values[] = str_pad($letter, 65535, '-'); + $values[] = str_pad($letter, 65536, '+'); + } + $db = $this->db(['shardingEnabled' => false]); + $db->insert('lg', array_map(function ($value) { return ['v' => $value]; }, $values)); + $this->assertTrue($db->createFieldIndex('lg', 'v')['success']); + + [$header, $map] = $this->readFieldIndexFile($this->dbPath('lg', 'fx', 'v')); + $this->assertGreaterThan(1, $header['pg']['levels'], 'precondition: a directory above the data pages'); + $this->assertCount(count($values), $map); + + // Every directory page reads as what it holds: a lookup that can't read one answers by a scan, as rightly + $raw = (string)file_get_contents($this->dbPath('lg', 'fx', 'v')); + $body = substr($raw, strpos($raw, "\n") + 1); + $walk = function (int $offset, int $length, int $level) use (&$walk, $body): void { + if ($level > 0) { + $children = Testing::callStatic('PagedFile::directory', substr($body, $offset, $length - 4)); + $this->assertIsArray($children, "the directory page at $offset"); + foreach ($children as [, $childOffset, $childLength]) { + $walk($childOffset, $childLength, $level - 1); + } + } + }; + $walk($header['pg']['root'][0], $header['pg']['root'][1], $header['pg']['levels']); + + $probe = array_slice($values, 1490, 10, true) + array_slice($values, 1500, null, true); + $this->assertANewProcessFinds('lg', 'v', $probe); + $this->assertTrue($db->checkIntegrity('lg')['ok']); + } + + /** + * @test An index is built in time linear in its ids, whatever they hold. Two values with many `":`, a text of + * JSON among them: the first id of a page, found by decoding ever longer prefixes, took 20 s for the first + */ + public function anIdOfManyQuotesIsBuiltInLinearTime(): void + { + $json = []; + for ($i = 0; strlen(json_encode($json)) < 160000; $i++) { + $json["field$i"] = $i; + } + $values = ['quotes' => str_repeat('":', 80000), 'json' => json_encode($json)]; + foreach ($values as $name => $value) { + $out = $this->runChild('$db = ' . $this->childDb() . '; $value = ' . var_export($value, true) . ';' + . ' $db->insert("q' . $name . '", array_merge([["v" => $value]], array_map(function ($i) { return ["v" => "s$i"]; }, range(1, 300))));' + . ' $start = microtime(true); $db->createFieldIndex("q' . $name . '", "v");' + . ' echo json_encode([microtime(true) - $start, count($db->find("q' . $name . '", ["v" => $value]))]);'); + $result = json_decode($out, true); + $this->assertIsArray($result, substr($out, 0, 300)); + $this->assertSame(1, $result[1], "$name: found"); + $this->assertLessThan(2.0, $result[0], "$name: seconds of the build"); + } + } + + /** + * @test The first id of every data page is that id, whatever it ends in: a quote or a backslash, escaped in the + * page, a `":` or a `,"` inside it (the test's codec reads the directory and compares it with the pages) + */ + public function theDirectoryNamesEveryPageWhateverItsFirstIdHolds(): void + { + $tails = ['"', '\\', '\\"', '"\\', '":', '\\":', '",', ',"', ',":', '":1,"', '\\\\', "\n", "\t", '/', 'ş', '€', '😀']; + $values = []; + foreach ($tails as $i => $tail) { + foreach ([700, 1500, 3000, 4090, 4200] as $length) { + $values[] = sprintf('%02d', $i) . str_repeat('p', $length) . $tail; + } + } + $db = $this->db(['shardingEnabled' => false]); + $db->insert('tl', array_map(function ($value) { return ['v' => $value]; }, $values)); + $this->assertTrue($db->createFieldIndex('tl', 'v')['success']); + + [$header, $map] = $this->readFieldIndexFile($this->dbPath('tl', 'fx', 'v')); + $this->assertGreaterThan(0, $header['pg']['levels'], 'precondition: more than one data page'); + $this->assertCount(count($values), $map); + $this->assertANewProcessFinds('tl', 'v', $values); + } + + /** @test A field name ending in a line feed is no plain name: its file token is the hash, not a name with a line feed */ + public function aFieldNameWithALineFeedGivesNoFileNameWithOne(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('nl', [['abc' => 1, "abc\n" => 2], ['abc' => 3, "abc\n" => 4]]); + $this->assertTrue($db->createFieldIndex('nl', "abc\n")['success']); + $this->assertTrue($db->createFieldIndex('nl', 'abc')['success']); + foreach ($this->dbFiles('nl') as $file) { + $this->assertStringNotContainsString("\n", $file); + } + $this->assertSame(1, $db->find('nl', ["abc\n" => 4])[0]['key']); + $this->assertSame(1, $db->find('nl', ['abc' => 3])[0]['key']); + } + + /** + * @test §6.5.1 fixes the bytes: a writer made from the format alone gives the same body, directory levels included, + * every page as full as a page of about 4096 bytes lets it be + * @runInSeparateProcess + */ + public function aWriterOfTheFormatGivesTheSameBytes(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('by', array_map(function ($i) { return ['v' => sprintf('value-%06d', $i), 'w' => $i % 3]; }, range(0, 59999))); + $this->assertTrue($db->createFieldIndex('by', 'v')['success']); + $path = $this->dbPath('by', 'fx', 'v'); + [$header, $map] = $this->readFieldIndexFile($path); + $this->assertGreaterThan(1, $header['pg']['pages'] - intdiv($header['pg']['data'], 4096), 'precondition: more than one directory page'); + $copy = sys_get_temp_dir() . '/nonedb_fx_' . bin2hex(random_bytes(6)); + $this->writeFieldIndexFile($copy, $header, $map); + $body = function (string $file): string { + $raw = (string)file_get_contents($file); + return substr($raw, strpos($raw, "\n") + 1); + }; + $same = $body($path) === $body($copy); + unlink($copy); + $this->assertTrue($same); + } + + /** + * @test §6.5.1: a file of another version is not read, by a lookup or as a whole; it answers as no file does. + * The same file with its own version is read, in the same process state + * @runInSeparateProcess + */ + public function aFileOfAnotherVersionIsNotRead(): void + { + $count = function (): int { + $stats = $this->stats(); + return $stats['pagesRead'] + $stats['bodiesRead'] + $stats['bodiesDecoded']; + }; + $reads = ['lookup' => function ($db) { return $db->find('vr', ['v' => 'b7777']); }, + 'whole read' => function ($db) { return $db->query('vr')->where(['v' => ['$gte' => 'b7777', '$lte' => 'b7777']])->get(); }]; + foreach ($reads as $read => $run) { + foreach ([false, true] as $relabel) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('vr', array_map(function ($i) { return ['v' => 'b' . $i]; }, range(0, 19999))); + $this->assertTrue($db->createFieldIndex('vr', 'v')['success']); + if ($relabel) { + $path = $this->dbPath('vr', 'fx', 'v'); + $raw = (string)file_get_contents($path); + $line = (string)strstr($raw, "\n", true); + $header = $this->openHeaderLine($path, $line); + $this->assertSame(1, $header['v'], 'precondition'); + $header['v'] = 2; + file_put_contents($path, $this->sealHeaderLine(json_encode($header)) . substr($raw, strlen($line))); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->find('vr', ['key' => 0]); // past the process's first lock session + $before = $count(); + $this->assertSame([7777], array_column($run($db), 'key'), $read); + if ($relabel) { + $this->assertSame($before, $count(), "$read: nothing of the file was read"); + } else { + $this->assertGreaterThan($before, $count(), "$read: precondition, the file of its version is read"); + } + } + } + } + + /** + * @test §6.5.1 a page whose crc fits but whose entry leaves it makes the file invalid: never an error of PHP, the + * query is answered as without the file. Lengths past the table, past the page, at the end of the page. + * @runInSeparateProcess + */ + public function aDirectoryEntryThatLeavesItsPageMakesTheFileInvalid(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('dp', array_map(function ($i) { return ['u' => sprintf('value-%06d', $i)]; }, range(0, 29999))); + $this->assertTrue($db->createFieldIndex('dp', 'u')['success']); + $path = $this->dbPath('dp', 'fx', 'u'); + $intact = (string)file_get_contents($path); + $line = strpos($intact, "\n") + 1; + $header = json_decode(substr($intact, 0, $line), true); + $this->assertGreaterThan(0, $header['pg']['levels'], 'precondition: a directory'); + [$offset, $length] = $header['pg']['root']; + $table = $length - 6 - 4 * unpack('v', $intact, $line + $offset + $length - 6)[1]; + foreach ([$table - 12 + 1, $table - 3, $table, $length, 70000, 4294967295] as $bad) { + $page = substr_replace(substr($intact, $line + $offset, $length - 4), pack('V', $bad), 0, 4); + file_put_contents($path, substr_replace($intact, $page . pack('V', crc32($page)), $line + $offset, $length)); + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . ';' + . ' echo json_encode(array_column($db->find("dp", ["u" => "value-012345"]), "key"));'); + $this->assertSame('[12345]', $out, "entry length $bad"); + } + } +} diff --git a/tests/Regression/FieldIndexRegressionTest.php b/tests/Integration/FieldIndexRegressionTest.php similarity index 64% rename from tests/Regression/FieldIndexRegressionTest.php rename to tests/Integration/FieldIndexRegressionTest.php index 9ade5cd..b39e8d7 100644 --- a/tests/Regression/FieldIndexRegressionTest.php +++ b/tests/Integration/FieldIndexRegressionTest.php @@ -1,6 +1,8 @@ assertSame('u2', $db->find('fx', ['e_mail' => 'u2'])[0]['e_mail']); $this->assertSame('b@x', $db->find('fx', ['şehir' => 'Bursa'])[0]['e-mail']); $this->assertSame('Bob', $db->find('fx', ['first.name' => 'Bob'])[0]['first.name']); + $this->assertSame([1], array_column($db->find('fx', ['şehir' => 'Bursa']), 'key')); + $this->assertSame([0], array_column($db->find('fx', ['e-mail' => 'a@x']), 'key')); $this->assertEqualsCanonicalizing(['e-mail', 'e_mail', 'şehir', 'first.name'], $db->getFieldIndexes('fx')['fields']); - $this->assertCount(4, glob($this->dbBasePath('fx') . '.nonedb.fidx.*'), 'one file per field, no collisions'); + $this->assertCount(4, glob($this->dbPath('fx', 'gen') . 'indexes/main.*.fx'), 'one file per field, no collisions'); + // Tokens (STORAGE_FORMAT.md §5.1): "e_mail" is itself; the others get a lower-case prefix and 12 hex of + // their sha256; the file is indexes/{segment}.{token}.fx + $base = $this->dbPath('fx', 'gen') . 'indexes/main.'; + $this->assertFileExists($base . 'e_mail.fx'); + foreach (['e-mail' => 'e_mail', 'şehir' => '__ehir', 'first.name' => 'first_name'] as $field => $prefix) { + $this->assertFileExists($base . $prefix . '-' . substr(hash('sha256', $field), 0, 12) . '.fx', $field); + } } /** @test Y-12 */ @@ -38,7 +49,10 @@ public function nullAndTypeTransitionsAreIndexed(): void $db->update('nt', [['key' => 0], ['set' => ['x' => ['now-array']]]]); // scalar -> array $this->assertSame([1, 3], array_column($db->find('nt', ['x' => null]), 'key')); + $this->assertSame([1, 3], $db->getKeysFromFieldIndex('nt', 'x', null)); $this->assertSame([2], array_column($db->find('nt', ['x' => 'b']), 'key')); + $this->assertSame([0], array_column($db->find('nt', ['x' => ['now-array']]), 'key'), 'an array value: found by a scan'); + $this->assertNull($db->getKeysFromFieldIndex('nt', 'x', ['now-array']), 'arrays are not indexed'); $this->assertSame([], $db->find('nt', ['x' => 'a'])); $db->delete('nt', ['x' => null]); $this->assertSame([], $db->find('nt', ['x' => null])); @@ -56,6 +70,8 @@ public function shardedIndexesCoverNullsAndNewShards(): void $db->insert('sn', $rows); $db->createFieldIndex('sn', 'v'); $this->assertCount(5, $db->find('sn', ['v' => null])); + $this->assertSame([1, 3, 5, 7, 9], array_column($db->find('sn', ['v' => null]), 'key')); + $this->assertSame([0, 2, 4, 6, 8], array_column($db->find('sn', ['v' => 'x']), 'key')); // New shards created after the index $more = []; @@ -66,11 +82,15 @@ public function shardedIndexesCoverNullsAndNewShards(): void $db->update('sn', [['i' => 19], ['set' => ['v' => 'Z']]]); $this->assertSame([19], array_column($db->find('sn', ['v' => 'Z']), 'i')); $this->assertCount(9, $db->find('sn', ['v' => 'late'])); + $this->assertSame(range(10, 18), array_column($db->find('sn', ['v' => 'late']), 'key')); + $this->assertSame([1, 3, 5, 7, 9], array_column($db->find('sn', ['v' => null]), 'key'), 'the new shards hold no null'); $this->assertSame(['v'], $db->getFieldIndexes('sn')['fields']); $this->assertTrue($db->getFieldIndexes('sn')['sharded']); $this->assertTrue($db->dropFieldIndex('sn', 'v')['success']); - $this->assertSame([], glob($this->dbBasePath('sn') . '*fidx*')); + // the files of every shard and the global one are gone + $this->assertSame([], glob($this->dbPath('sn', 'gen') . 'indexes/*.v.fx')); + $this->assertFileDoesNotExist($this->dbPath('sn', 'gfx', 'v')); $this->assertFalse($db->dropFieldIndex('sn', 'v')['success']); } @@ -84,9 +104,13 @@ public function negativeZeroAndMixedTypesMatchLikeAScan(): void foreach ($values as $v) { $scan[] = array_column($db->find('mz', ['v' => $v]), 'key'); } + // By hand (=== semantics): -0.0 is 0.0; 0, "0", 1.0, 1, "1", true and "__null__" are each only themselves + $byHand = [[0], [0], [1], [2], [4], [3], [5], [6], [], [8], [7]]; + $this->assertSame($byHand, $scan, 'the scan agrees with ==='); $db->createFieldIndex('mz', 'v'); foreach ($values as $i => $v) { $this->assertSame($scan[$i], array_column($db->find('mz', ['v' => $v]), 'key'), 'value ' . var_export($v, true)); + $this->assertSame($byHand[$i], $db->getKeysFromFieldIndex('mz', 'v', $v), 'value ' . var_export($v, true)); } } @@ -130,17 +154,20 @@ public function randomizedIndexedResultsEqualScan(): void public function leftoverJournalRebuildsIndexes(): void { $db = $this->db(); - $db->insert('j', [['city' => 'A'], ['city' => 'B']]); + $db->insert('j', [['city' => 'A']]); $db->createFieldIndex('j', 'city'); - // Simulate a writer that appended data and died before updating the field index: it had - // bumped the generation and journaled the segment in the lock file's state record + $fx = $this->dbPath('j', 'fx', 'city'); + $checkpoint = file_get_contents($fx); // covers record 0 only + $db->insert('j', [['city' => 'B']]); + // Simulate a writer that appended data and died: it had bumped the generation and journaled the + // segment in the lock file's state record (STORAGE_FORMAT.md §4.2: "{gen}:{epoch} j=main"). (Writers + // never touch the field index; its checkpoint covers the bytes before B, the lines after it are + // replayed when it is loaded.) $state = $this->lockState('j'); - $this->assertContains('v', $state['flags']); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', (string)$state['epoch']); $this->assertSame([], preg_grep('/^(j=|J$)/', $state['flags']), 'no journal entry after a completed write'); - file_put_contents($this->dbBasePath('j') . '.nonedb.lock', str_pad(($state['gen'] + 1) . ' v j=main', 128)); - $index = json_decode(file_get_contents($this->dbBasePath('j') . '.nonedb.fidx.city'), true); - $index['values'] = ['A' => [0]]; // B lost - file_put_contents($this->dbBasePath('j') . '.nonedb.fidx.city', json_encode($index)); + $this->assertSame($checkpoint, file_get_contents($fx)); + $this->writeLockState('j', ($state['gen'] + 1) . ':' . $state['epoch'] . ' j=main'); \noneDB::clearStaticCache(); $db = $this->db(); @@ -148,27 +175,14 @@ public function leftoverJournalRebuildsIndexes(): void $this->assertSame([0], array_column($db->find('j', ['city' => 'A']), 'key')); $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('j')['flags']), 'journal entry cleared after recovery'); // "J" (too many segments to list) rebuilds every segment - file_put_contents($this->dbBasePath('j') . '.nonedb.fidx.city', json_encode($index)); - file_put_contents($this->dbBasePath('j') . '.nonedb.lock', str_pad(($this->lockState('j')['gen'] + 1) . ' v J', 128)); + file_put_contents($fx, $checkpoint); + $state = $this->lockState('j'); + $this->writeLockState('j', ($state['gen'] + 1) . ':' . $state['epoch'] . ' J'); \noneDB::clearStaticCache(); $this->assertSame([1], array_column($this->db()->find('j', ['city' => 'B']), 'key')); $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('j')['flags'])); } - /** @test pre-3.2 index files are rebuilt, not trusted */ - public function legacyIndexFilesAreRebuilt(): void - { - $db = $this->db(); - $db->insert('lv', [['city' => 'A'], ['city' => 'B'], ['city' => null]]); - // A v1 file from an older version that missed records (and, like every pre-3.2 database, no lock file) - file_put_contents($this->dbBasePath('lv') . '.nonedb.fidx.city', json_encode(['v' => 1, 'field' => 'city', 'values' => ['A' => [0]]])); - $this->removeLockFile('lv'); - $db = $this->db(); - $this->assertSame(['city'], $db->getFieldIndexes('lv')['fields'], 'derived from the old file'); - $this->assertSame([1], array_column($db->find('lv', ['city' => 'B']), 'key')); - $this->assertSame([2], array_column($db->find('lv', ['city' => null]), 'key')); - $this->assertSame(2, json_decode(file_get_contents($this->dbBasePath('lv') . '.nonedb.fidx.city'), true)['v']); - } /** @test P-02 a batch writes each index file once */ public function batchInsertWritesIndexOnce(): void @@ -180,9 +194,23 @@ public function batchInsertWritesIndexOnce(): void for ($i = 0; $i < 3000; $i++) { $rows[] = ['c' => $i % 7]; } + $fx = $this->dbPath('bt', 'fx', 'c'); + $file = file_get_contents($fx); $start = microtime(true); $db->insert('bt', $rows); $this->assertLessThan(2.0, microtime(true) - $start); $this->assertCount(428, $db->find('bt', ['c' => 6])); // i % 7 === 6 for i < 3000 + $this->assertSame(range(7, 3000, 7), array_column($db->find('bt', ['c' => 6]), 'key'), 'key = i + 1'); + // At most one index write for the whole batch: none while the new data is under 64 KiB, else one + // checkpoint at the end of the session that covers all of it + clearstatcache(); + $data = $this->dbPath('bt', 'data'); + $covered = json_decode(strtok($file, "\n"), true)['s']; + $header = json_decode(strtok(file_get_contents($fx), "\n"), true); + if (filesize($data) - $covered < 65536) { + $this->assertSame($file, file_get_contents($fx)); + } else { + $this->assertSame(filesize($data), $header['s']); + } } } diff --git a/tests/Feature/FieldIndexTest.php b/tests/Integration/FieldIndexTest.php similarity index 62% rename from tests/Feature/FieldIndexTest.php rename to tests/Integration/FieldIndexTest.php index 6b9e932..9b242ba 100644 --- a/tests/Feature/FieldIndexTest.php +++ b/tests/Integration/FieldIndexTest.php @@ -1,38 +1,16 @@ cleanupTestFiles(); - } - - protected function tearDown(): void - { - $this->cleanupTestFiles(); - parent::tearDown(); - } - - private function cleanupTestFiles() - { - $files = glob($this->testDbDir . '*' . $this->testDbName . '*'); - foreach ($files as $file) { - @unlink($file); - } - noneDB::clearStaticCache(); - } - // ==================== CREATE INDEX TESTS ==================== public function testCreateFieldIndex() @@ -47,8 +25,11 @@ public function testCreateFieldIndex() // Create index on city field $result = $this->noneDB->createFieldIndex($this->testDbName, 'city'); - $this->assertTrue($result['success']); - $this->assertEquals(2, $result['values']); // 2 unique values: Istanbul, Ankara + // 3 records indexed, 2 distinct values: Istanbul, Ankara + $this->assertSame(['success' => true, 'indexed' => 3, 'error' => null], $result); + $this->assertTrue($this->noneDB->hasFieldIndex($this->testDbName, 'city')); + $this->assertFileExists($this->dbPath($this->testDbName, 'fx', 'city')); + $this->assertSame([0, 2], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Istanbul')); } public function testCreateFieldIndexOnEmptyDatabase() @@ -60,8 +41,8 @@ public function testCreateFieldIndexOnEmptyDatabase() // Create index should work but have 0 values $result = $this->noneDB->createFieldIndex($this->testDbName, 'city'); - $this->assertTrue($result['success']); - $this->assertEquals(0, $result['values']); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $result); + $this->assertSame([], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'temp')); } public function testCreateFieldIndexOnNonExistentField() @@ -73,8 +54,13 @@ public function testCreateFieldIndexOnNonExistentField() // Create index on field that doesn't exist $result = $this->noneDB->createFieldIndex($this->testDbName, 'nonexistent'); - $this->assertTrue($result['success']); - $this->assertEquals(0, $result['values']); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $result); + $this->assertSame(['nonexistent'], $this->noneDB->getFieldIndexes($this->testDbName)['fields']); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['nonexistent' => null]), 'a missing field is not null'); + + // A record that gets the field later is found through the index + $this->noneDB->insert($this->testDbName, ['name' => 'Jane', 'nonexistent' => 'now']); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'nonexistent', 'now')); } public function testCreateFieldIndexWithNullValues() @@ -87,8 +73,10 @@ public function testCreateFieldIndexWithNullValues() $result = $this->noneDB->createFieldIndex($this->testDbName, 'city'); - $this->assertTrue($result['success']); - $this->assertEquals(2, $result['values']); // Istanbul and null + // Istanbul and null are indexed (2 values); Bob's missing field is not indexed at all + $this->assertSame(['success' => true, 'indexed' => 2, 'error' => null], $result); + $this->assertSame([1], array_column($this->noneDB->find($this->testDbName, ['city' => null]), 'key'), 'missing is not null'); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', null)); } public function testCreateFieldIndexWithBooleanValues() @@ -101,8 +89,10 @@ public function testCreateFieldIndexWithBooleanValues() $result = $this->noneDB->createFieldIndex($this->testDbName, 'active'); - $this->assertTrue($result['success']); - $this->assertEquals(2, $result['values']); // true and false + $this->assertSame(['success' => true, 'indexed' => 3, 'error' => null], $result); // true and false + $this->assertSame([0, 2], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'active', true)); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'active', false)); + $this->assertSame([], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'active', 1), 'true is not 1'); } // ==================== DROP INDEX TESTS ==================== @@ -117,7 +107,12 @@ public function testDropFieldIndex() $result = $this->noneDB->dropFieldIndex($this->testDbName, 'city'); - $this->assertTrue($result['success']); + $this->assertSame(['success' => true, 'error' => null], $result); + $this->assertFalse($this->noneDB->hasFieldIndex($this->testDbName, 'city')); + $this->assertSame(['fields' => [], 'sharded' => false], $this->noneDB->getFieldIndexes($this->testDbName)); + $this->assertFileDoesNotExist($this->dbPath($this->testDbName, 'fx', 'city')); + $this->assertNull($this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Istanbul')); + $this->assertSame([0], array_column($this->noneDB->find($this->testDbName, ['city' => 'Istanbul']), 'key'), 'found without the index'); } public function testDropNonExistentIndex() @@ -128,7 +123,7 @@ public function testDropNonExistentIndex() $result = $this->noneDB->dropFieldIndex($this->testDbName, 'nonexistent'); - $this->assertFalse($result['success']); + $this->assertSame(['success' => false, 'error' => 'Index does not exist'], $result); } // ==================== GET INDEXES TESTS ==================== @@ -144,10 +139,10 @@ public function testGetFieldIndexes() $result = $this->noneDB->getFieldIndexes($this->testDbName); - $this->assertArrayHasKey('fields', $result); + $this->assertSame(['fields', 'sharded'], array_keys($result)); + $this->assertFalse($result['sharded']); $this->assertCount(2, $result['fields']); - $this->assertContains('city', $result['fields']); - $this->assertContains('age', $result['fields']); + $this->assertEqualsCanonicalizing(['city', 'age'], $result['fields']); } public function testGetFieldIndexesEmpty() @@ -158,8 +153,7 @@ public function testGetFieldIndexesEmpty() $result = $this->noneDB->getFieldIndexes($this->testDbName); - $this->assertArrayHasKey('fields', $result); - $this->assertEmpty($result['fields']); + $this->assertSame(['fields' => [], 'sharded' => false], $result); } // ==================== FIND WITH INDEX TESTS ==================== @@ -183,12 +177,16 @@ public function testFindUsesFieldIndex() // Find with indexed field $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); - $this->assertCount(20, $result); // Every 5th record = 20 records + $this->assertSame(range(0, 95, 5), array_column($result, 'key')); // Every 5th record = 20 records // Verify all results have correct city foreach ($result as $record) { $this->assertEquals('Istanbul', $record['city']); } + + // ...and that they were read through the index: only the 20 candidates + $explain = $this->noneDB->query($this->testDbName)->where(['city' => 'Istanbul'])->explain(true); + $this->assertSame(['index', 20, 20, 20], [$explain['access'], $explain['estimatedRead'], $explain['actual']['recordsRead'], $explain['actual']['returned']]); } public function testFindWithoutIndex() @@ -203,6 +201,7 @@ public function testFindWithoutIndex() $this->assertCount(1, $result); $this->assertEquals('John', $result[0]['name']); + $this->assertSame('scan', $this->noneDB->query($this->testDbName)->where(['city' => 'Istanbul'])->explain()['access']); } public function testFindWithMultipleIndexedFields() @@ -221,7 +220,9 @@ public function testFindWithMultipleIndexedFields() // Find with both indexed fields (intersection) $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul', 'dept' => 'IT']); - $this->assertCount(2, $result); // John and Alice + $this->assertSame([[0, 'John'], [3, 'Alice']], array_map(function ($r) { return [$r['key'], $r['name']]; }, $result)); + $explain = $this->noneDB->query($this->testDbName)->where(['city' => 'Istanbul', 'dept' => 'IT'])->explain(); + $this->assertSame(['index', 2], [$explain['access'], $explain['estimatedRead']], 'the intersection of both indexes'); } public function testFindWithNoMatchingRecords() @@ -234,7 +235,8 @@ public function testFindWithNoMatchingRecords() $result = $this->noneDB->find($this->testDbName, ['city' => 'NonExistent']); - $this->assertEmpty($result); + $this->assertSame([], $result); + $this->assertSame([], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'NonExistent')); } // ==================== INDEX MAINTENANCE ON INSERT ==================== @@ -256,6 +258,9 @@ public function testInsertUpdatesFieldIndex() $this->assertCount(1, $result); $this->assertEquals('Jane', $result[0]['name']); + $this->assertSame(1, $result[0]['key']); + noneDB::clearStaticCache(); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Ankara'), 'also for a new process'); } public function testBulkInsertUpdatesFieldIndex() @@ -275,7 +280,8 @@ public function testBulkInsertUpdatesFieldIndex() // Index should be updated for all $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); - $this->assertCount(2, $result); // John and Alice + $this->assertSame([0, 3], array_column($result, 'key')); // John and Alice + $this->assertSame([2], array_column($this->noneDB->find($this->testDbName, ['city' => 'Izmir']), 'key')); } // ==================== INDEX MAINTENANCE ON DELETE ==================== @@ -297,6 +303,7 @@ public function testDeleteUpdatesFieldIndex() $this->assertCount(1, $result); $this->assertEquals('Jane', $result[0]['name']); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Istanbul')); } public function testDeleteAllWithSameValueUpdatesIndex() @@ -314,11 +321,12 @@ public function testDeleteAllWithSameValueUpdatesIndex() // Index should be updated - Istanbul value should have no keys $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); - $this->assertEmpty($result); + $this->assertSame([], $result); + $this->assertSame([], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Istanbul')); // Ankara should still work $result = $this->noneDB->find($this->testDbName, ['city' => 'Ankara']); - $this->assertCount(1, $result); + $this->assertSame([2], array_column($result, 'key')); } // ==================== INDEX MAINTENANCE ON UPDATE ==================== @@ -340,12 +348,14 @@ public function testUpdateUpdatesFieldIndex() // Old value should not find John $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); - $this->assertEmpty($result); + $this->assertSame([], $result); // New value should find John $result = $this->noneDB->find($this->testDbName, ['city' => 'Izmir']); $this->assertCount(1, $result); $this->assertEquals('John', $result[0]['name']); + $this->assertSame([0], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Izmir')); + $this->assertSame([1], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Ankara'), 'Jane is untouched'); } public function testUpdateNonIndexedFieldDoesNotAffectIndex() @@ -365,7 +375,8 @@ public function testUpdateNonIndexedFieldDoesNotAffectIndex() // Index should still work $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); $this->assertCount(1, $result); - $this->assertEquals(31, $result[0]['age']); + $this->assertSame(0, $result[0]['key']); + $this->assertSame(31, $result[0]['age']); } // ==================== REBUILD INDEX TESTS ==================== @@ -379,11 +390,18 @@ public function testRebuildFieldIndex() $this->noneDB->createFieldIndex($this->testDbName, 'city'); + // Damage the index: wrong buckets under a header that still fits the data + $fx = $this->dbPath($this->testDbName, 'fx', 'city'); + $header = strtok(file_get_contents($fx), "\n"); + file_put_contents($fx, $header . "\n" . json_encode(['sIstanbul' => 1, 'sAnkara' => 0]) . "\n"); + noneDB::clearStaticCache(); + // Rebuild index $result = $this->noneDB->rebuildFieldIndex($this->testDbName, 'city'); - $this->assertTrue($result['success']); - $this->assertEquals(2, $result['values']); + $this->assertSame(['success' => true, 'indexed' => 2, 'error' => null], $result); + $this->assertSame([0], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'city', 'Istanbul'), 'rebuilt from the data'); + $this->assertSame('Jane', $this->noneDB->find($this->testDbName, ['city' => 'Ankara'])[0]['name']); } // ==================== SPECIAL VALUE TESTS ==================== @@ -401,6 +419,8 @@ public function testFindWithNullValue() $this->assertCount(1, $result); $this->assertEquals('John', $result[0]['name']); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['city' => '']), 'null is not ""'); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['city' => false]), 'null is not false'); } public function testFindWithBooleanValue() @@ -414,7 +434,8 @@ public function testFindWithBooleanValue() $this->noneDB->createFieldIndex($this->testDbName, 'active'); $result = $this->noneDB->find($this->testDbName, ['active' => true]); - $this->assertCount(2, $result); + $this->assertSame([0, 2], array_column($result, 'key')); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['active' => 1]), 'true is not 1'); $result = $this->noneDB->find($this->testDbName, ['active' => false]); $this->assertCount(1, $result); @@ -427,13 +448,19 @@ public function testFindWithNumericValue() ['name' => 'John', 'score' => 100], ['name' => 'Jane', 'score' => 85], ['name' => 'Bob', 'score' => 100], + ['name' => 'Ann', 'score' => '100'], + ['name' => 'Eve', 'score' => 100.0], ]); $this->noneDB->createFieldIndex($this->testDbName, 'score'); $result = $this->noneDB->find($this->testDbName, ['score' => 100]); - $this->assertCount(2, $result); + // Matched with ===: the string "100" and the float 100.0 are other values + $this->assertSame([0, 2], array_column($result, 'key')); + $this->assertSame([3], array_column($this->noneDB->find($this->testDbName, ['score' => '100']), 'key')); + $this->assertSame([4], array_column($this->noneDB->find($this->testDbName, ['score' => 100.0]), 'key')); + $this->assertSame([0, 2], $this->noneDB->getKeysFromFieldIndex($this->testDbName, 'score', 100)); } // ==================== PERFORMANCE TESTS ==================== @@ -465,10 +492,15 @@ public function testFieldIndexPerformance() $result = $this->noneDB->find($this->testDbName, ['category' => 'cat5']); $timeWithIndex = (microtime(true) - $start) * 1000; - // With index should be faster (at least not slower) - $this->assertCount(100, $result); // 1000/10 = 100 records per category + // 1000/10 = 100 records per category: keys 5, 15, ..., 995 + $this->assertSame(range(5, 995, 10), array_column($result, 'key')); - // Note: In small datasets, the difference might be minimal - // This test ensures the feature works correctly + // Timing is not asserted (too noisy); what makes the indexed lookup cheaper is: it reads only the + // 100 candidates, the unindexed one all 1000 records + $explain = $this->noneDB->query($this->testDbName)->where(['category' => 'cat5'])->explain(true); + $this->assertSame(['index', 100, 100], [$explain['access'], $explain['estimatedRead'], $explain['actual']['recordsRead']]); + $this->noneDB->dropFieldIndex($this->testDbName, 'category'); + $explain = $this->noneDB->query($this->testDbName)->where(['category' => 'cat5'])->explain(true); + $this->assertSame(['scan', 1000, 1000, 100], [$explain['access'], $explain['estimatedRead'], $explain['actual']['recordsRead'], $explain['actual']['returned']]); } } diff --git a/tests/Integration/FileSecurityTest.php b/tests/Integration/FileSecurityTest.php new file mode 100644 index 0000000..93e4c17 --- /dev/null +++ b/tests/Integration/FileSecurityTest.php @@ -0,0 +1,244 @@ +umask !== null) { + umask($this->umask); + $this->umask = null; + } + \noneDB::clearStaticCache(); + foreach ($this->outside as $path) { + if (is_link($path)) { + @unlink($path); + } else { + $this->removeDirectoryTree($path); + } + } + $this->outside = []; + parent::tearDown(); + } + + private function outsidePath(string $prefix): string + { + $path = realpath(sys_get_temp_dir()) . '/nonedb_' . $prefix . '_' . bin2hex(random_bytes(6)); + $this->outside[] = $path; + return $path; + } + + private function posixOnly(): void + { + if (DIRECTORY_SEPARATOR !== '/') { + $this->markTestSkipped('POSIX permissions and links'); + } + } + + /** relative path => permission bits, of a directory tree (the root itself under '.') */ + private function modes(string $dir): array + { + $out = ['.' => fileperms($dir) & 0777]; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $full = $path . '/' . $entry; + $out[$rel . $entry] = fileperms($full) & 0777; + if (is_dir($full)) { + $walk($full, $rel . $entry . '/'); + } + } + }; + $walk(rtrim($dir, '/'), ''); + return $out; + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + /** Everything noneDB writes in a store: records, indexes, buffer, recovery log and quarantine, store log */ + private function exercise(\noneDB $db): void + { + $db->insert('u', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n]]]; }, range(0, 9))); + $db->createFieldIndex('u', 'c'); + $db->createSpatialIndex('u', 'loc'); + $db->delete('u', ['n' => 3]); + $db->compact('u'); + $db->enableBuffering(true); + $db->insert('b', [['v' => 1]]); + $db->enableBuffering(false); + file_put_contents($this->dbPath('u', 'data', 2), '{"torn', FILE_APPEND); // the shard the next insert appends to + \noneDB::clearStaticCache(); + $db->insert('u', ['n' => 10]); // the torn tail goes to quarantine, the event to the recovery log + file_put_contents($this->testDbDir . 'STORE.2', 'x'); + \noneDB::clearStaticCache(); + $db->find('u', 0); // the store copy is repaired, logged in .nonedb-maintenance/log.jsonl + } + + /** @test STORAGE_FORMAT.md §3: files filePermissions, directories directoryPermissions, whatever the umask */ + public function everythingNoneDBCreatesHasTheConfiguredPermissions(): void + { + $this->posixOnly(); + foreach ([[0022, [], 0640, 0750], [0077, ['filePermissions' => 0660, 'directoryPermissions' => '0770'], 0660, 0770]] as [$umask, $config, $file, $dir]) { + $this->cleanTestDirectory(); + $this->umask = $this->umask ?? umask(); + umask($umask); + $location = $this->outsidePath('store'); + $this->testDbDir = $location . '/'; // created by noneDB + $db = $this->db(['shardSize' => 4] + $config); + $this->exercise($db); + $modes = $this->modes($location); + $this->assertArrayHasKey('.nonedb-maintenance/log.jsonl', $modes, 'the store log was written'); + $this->assertNotEmpty(glob($this->dbPath('u', 'quarantine') . 'q.*.bin'), 'a quarantine file was written'); + $this->assertFileExists($this->dbPath('b', 'buffer')); + foreach ($modes as $path => $mode) { + $isDir = $path === '.' || is_dir($location . '/' . $path); + $this->assertSame(sprintf('%o', $isDir ? $dir : $file), sprintf('%o', $mode), "$path (umask " . sprintf('%o', $umask) . ')'); + } + $this->testDbDir = TEST_DB_DIR; + } + } + + /** @test CONFIGURATION.md: owner access needed, nothing world-writable */ + public function invalidPermissionsAreRefused(): void + { + foreach ([['filePermissions', 0666], ['filePermissions', 0400], ['filePermissions', 01640], ['filePermissions', 'rw-r-----'], + ['filePermissions', '0648'], ['filePermissions', "0640\n"], ['filePermissions', -1], ['filePermissions', 0640.0], ['directoryPermissions', 0640], + ['directoryPermissions', 0777], ['directoryPermissions', 0600], ['directoryPermissions', true]] as [$key, $value]) { + try { + $this->db([$key => $value]); + $this->fail("$key " . var_export($value, true) . ': refused'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString($key, $e->getMessage()); + } + } + foreach ([[0600, 0700], [0644, 0755], ['0640', '0750'], ['640', '750'], [0660, 0770]] as [$file, $dir]) { + $db = $this->db(['filePermissions' => $file, 'directoryPermissions' => $dir]); + $this->assertSame([is_string($file) ? octdec($file) : $file, is_string($dir) ? octdec($dir) : $dir], + [Testing::getOf($db, 'filePermissions'), Testing::getOf($db, 'directoryPermissions')]); + } + } + + /** @test STORAGE_FORMAT.md §3: two spellings of dbDir are one store, with one lock per file */ + public function dbDirIsUsedByItsCanonicalPath(): void + { + $this->posixOnly(); + $real = $this->outsidePath('real'); + mkdir($real); + $link = $this->outsidePath('link'); + symlink($real, $link); + $spellings = [$real, $link, $real . '/./', dirname($real) . '/../' . basename(dirname($real)) . '/' . basename($real), $link . '/not-yet/..']; + foreach ($spellings as $spelling) { + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $spelling]); + $this->assertSame($real . '/', Testing::getOf($db, 'dbDir'), $spelling); + } + $missing = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $link . '/sub/dir']); + $this->assertSame($real . '/sub/dir/', Testing::getOf($missing, 'dbDir'), 'the nearest existing parent resolved'); + + // One lock per file: a write through one spelling inside a write through the other (one process) + $a = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $real]); + $b = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $link, 'lockTimeout' => 0.5]); + $a->insert('u', ['v' => 'a']); + Testing::call($a, 'withLock', 'u', true, function () use ($b) { + $b->insert('u', ['v' => 'b']); // the same lock: re-entered, not waited for + }); + $this->assertSame(['a', 'b'], array_column($a->find('u', 0), 'v')); + } + + /** @test STORAGE_FORMAT.md §3.4: a database directory that is a link is not a database */ + public function aDatabaseDirectoryThatIsALinkIsNotADatabase(): void + { + $this->posixOnly(); + $db = $this->db(); + $db->insert('u', [['v' => 1]]); + $db->insert('w', [['v' => 2]]); + $home = rtrim($this->dbPath('u', 'home'), '/'); + $elsewhere = $this->outsidePath('moved'); + rename($home, $elsewhere); + symlink($elsewhere, $home); + \noneDB::clearStaticCache(); + $before = $this->modes($elsewhere) + ['bytes' => md5(serialize(array_map('md5_file', glob($elsewhere . '/generations/*/segments/*'))))]; + $db = $this->db(); + foreach (['find' => function () use ($db) { $db->find('u', 0); }, + 'insert' => function () use ($db) { $db->insert('u', ['v' => 3]); }, + 'count' => function () use ($db) { $db->count('u'); }, + 'a missing database' => function () use ($db) { $db->find('missing', 0); }, + 'a new database' => function () use ($db) { $db->insert('new', ['v' => 1]); }] as $case => $fn) { + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code($fn), $case); + } + $after = $this->modes($elsewhere) + ['bytes' => md5(serialize(array_map('md5_file', glob($elsewhere . '/generations/*/segments/*'))))]; + $this->assertSame($before, $after, 'no link followed: nothing written there'); + $this->assertSame(['w'], $db->getDBs(), 'skipped'); + $this->assertSame([2], array_column($db->find('w', 0), 'v'), 'the other databases work'); + + // An entry named like a database directory that is a file + unlink($home); + file_put_contents($home, 'x'); + \noneDB::clearStaticCache(); + $db = $this->db(); + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code(function () use ($db) { $db->find('u', 0); })); + $this->assertSame(['w'], $db->getDBs()); + } + + /** @test errors, logs and reports hold no key and no record value */ + public function noKeyOrRecordValueIsReported(): void + { + $value = 'SENSITIVE-VALUE-91d2'; + $db = $this->db(['shardingEnabled' => false]); + $db->insert('u', [['v' => $value], ['v' => 'other']]); + // A damaged line holding the value: found, logged and quarantined by the next writer + file_put_contents($this->dbPath('u', 'data'), '{"v":"' . $value . '","key":7' . "\n" . '{"v":"' . $value . '",', FILE_APPEND); + unlink($this->dbPath('u', 'kix')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->rebuildIndex('u'); + $db->insert('u', ['v' => 'after']); + $texts = [file_get_contents($this->dbPath('u', 'recovery')), json_encode($db->getRecoveryLog('u')), json_encode($db->getMetrics('u')), + json_encode($db->getDurability()), json_encode($db->checkIntegrity('u'))]; + $messages = []; + $failing = [ + function () { (new \noneDB(['secretKey' => 'another-secret-key-77', 'dbDir' => $this->testDbDir]))->find('u', 0); }, + function () { $this->db(['secretKey' => 'your_key']); }, + function () { file_put_contents($this->dbPath('u', 'current'), 'x'); \noneDB::clearStaticCache(); $this->db()->find('u', 0); }, + function () { $this->db()->restore($this->testDbDir, 'u'); }, + ]; + foreach ($failing as $fn) { + try { + $fn(); + } catch (\Exception $e) { + $messages[] = get_class($e) . ': ' . $e->getMessage(); + } + } + $this->assertCount(4, $messages); + foreach (array_merge($texts, $messages) as $i => $text) { + $this->assertStringNotContainsString(self::SECRET, (string)$text, "text $i"); + $this->assertStringNotContainsString('another-secret-key-77', (string)$text, "text $i"); + $this->assertStringNotContainsString($value, (string)$text, "text $i"); + } + $this->assertNotEmpty(array_filter(array_map(function ($l) { return json_decode($l, true)['action'] ?? null; }, + explode("\n", trim($texts[0])))), 'events were logged'); + } +} diff --git a/tests/Feature/FindTest.php b/tests/Integration/FindTest.php similarity index 53% rename from tests/Feature/FindTest.php rename to tests/Integration/FindTest.php index f9e81bf..ec1201c 100644 --- a/tests/Feature/FindTest.php +++ b/tests/Integration/FindTest.php @@ -1,6 +1,6 @@ noneDB->insert($this->testDbName, $data); } + /** The records of setUp as find() returns them: in key order, with the key added */ + private function expectedAll(): array + { + return [ + ['username' => 'john', 'email' => 'john@test.com', 'age' => 25, 'active' => true, 'key' => 0], + ['username' => 'jane', 'email' => 'jane@test.com', 'age' => 30, 'active' => true, 'key' => 1], + ['username' => 'bob', 'email' => 'bob@test.com', 'age' => 25, 'active' => false, 'key' => 2], + ['username' => 'alice', 'email' => 'alice@test.com', 'age' => 35, 'active' => true, 'key' => 3], + ]; + } + + private function keys(array $records): array + { + return array_column($records, 'key'); + } + // ========================================== // FIND ALL RECORDS // ========================================== @@ -40,8 +56,7 @@ public function findAllWithInteger0(): void { $result = $this->noneDB->find($this->testDbName, 0); - $this->assertIsArray($result); - $this->assertCount(4, $result); + $this->assertSame($this->expectedAll(), $result); } /** @@ -51,10 +66,10 @@ public function findAllReturnsAllRecords(): void { $result = $this->noneDB->find($this->testDbName, 0); - $this->assertEquals('john', $result[0]['username']); - $this->assertEquals('jane', $result[1]['username']); - $this->assertEquals('bob', $result[2]['username']); - $this->assertEquals('alice', $result[3]['username']); + $this->assertSame(['john', 'jane', 'bob', 'alice'], array_column($result, 'username')); + // docs/API.md: 0 and null (and any integer) mean every record + $this->assertSame($result, $this->noneDB->find($this->testDbName, null)); + $this->assertSame($result, $this->noneDB->find($this->testDbName, 7)); } /** @@ -65,10 +80,8 @@ public function findAllAddsKeyField(): void // Key IS added for all find results for consistency $result = $this->noneDB->find($this->testDbName, 0); - // Each record should have 'key' field with its index - $this->assertArrayHasKey('key', $result[0]); - $this->assertEquals(0, $result[0]['key']); - $this->assertEquals(1, $result[1]['key']); + // Each record has its key, as an int, in key order + $this->assertSame([0, 1, 2, 3], $this->keys($result)); } // ========================================== @@ -82,10 +95,7 @@ public function findBySingleKey(): void { $result = $this->noneDB->find($this->testDbName, ['key' => 0]); - $this->assertIsArray($result); - $this->assertCount(1, $result); - $this->assertEquals('john', $result[0]['username']); - $this->assertEquals(0, $result[0]['key']); + $this->assertSame([$this->expectedAll()[0]], $result); } /** @@ -95,9 +105,40 @@ public function findByMultipleKeys(): void { $result = $this->noneDB->find($this->testDbName, ['key' => [0, 2]]); - $this->assertCount(2, $result); - $this->assertEquals('john', $result[0]['username']); - $this->assertEquals('bob', $result[1]['username']); + $this->assertSame([$this->expectedAll()[0], $this->expectedAll()[2]], $result); + } + + /** + * @test docs/API.md: records "in the order of a key list"; a key listed twice is returned once + */ + public function findByKeyListKeepsTheOrderOfTheList(): void + { + $this->assertSame([3, 0, 2], $this->keys($this->noneDB->find($this->testDbName, ['key' => [3, 0, 2, 0]]))); + } + + /** + * @test docs/API.md: key accepts an integer, a digit string or a list of them; anything else matches + * nothing; it is combined with the other filters (AND). "5\n" is not key 5. + */ + public function findByKeyAcceptsOnlyIntegersAndDigitStrings(): void + { + $this->assertSame([$this->expectedAll()[1]], $this->noneDB->find($this->testDbName, ['key' => '1'])); + $this->assertSame([1, 3], $this->keys($this->noneDB->find($this->testDbName, ['key' => ['1', 3]]))); + foreach ([1.0, "1\n", ' 1', '1 ', '1e0', '+1', true, null, -1, [], [[1]]] as $bad) { + $this->assertSame([], $this->noneDB->find($this->testDbName, ['key' => $bad]), var_export($bad, true)); + } + $this->assertSame([1], $this->keys($this->noneDB->find($this->testDbName, ['key' => [0, 1, 2], 'age' => 30])), 'AND with other fields'); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['key' => 0, 'age' => 30])); + } + + /** + * @test docs/API.md: find() returns false for filters that are neither arrays, integers nor null + */ + public function findWithAnInvalidFilterReturnsFalse(): void + { + foreach (['john', 1.5, true, new \stdClass()] as $filter) { + $this->assertFalse($this->noneDB->find($this->testDbName, $filter), gettype($filter)); + } } /** @@ -107,8 +148,7 @@ public function findByKeyReturnsKeyInResult(): void { $result = $this->noneDB->find($this->testDbName, ['key' => 1]); - $this->assertArrayHasKey('key', $result[0]); - $this->assertEquals(1, $result[0]['key']); + $this->assertSame([$this->expectedAll()[1]], $result); } /** @@ -118,8 +158,7 @@ public function findByNonExistentKey(): void { $result = $this->noneDB->find($this->testDbName, ['key' => 999]); - $this->assertIsArray($result); - $this->assertCount(0, $result); + $this->assertSame([], $result); } /** @@ -129,8 +168,8 @@ public function findByKeyArrayWithNonExistent(): void { $result = $this->noneDB->find($this->testDbName, ['key' => [0, 999, 1]]); - // Should return only existing keys - $this->assertCount(2, $result); + // Only existing keys, in the order of the list + $this->assertSame([0, 1], $this->keys($result)); } /** @@ -140,8 +179,7 @@ public function findByLastKey(): void { $result = $this->noneDB->find($this->testDbName, ['key' => 3]); - $this->assertCount(1, $result); - $this->assertEquals('alice', $result[0]['username']); + $this->assertSame([$this->expectedAll()[3]], $result); } // ========================================== @@ -155,8 +193,7 @@ public function findBySingleField(): void { $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertCount(1, $result); - $this->assertEquals('john', $result[0]['username']); + $this->assertSame([$this->expectedAll()[0]], $result); } /** @@ -166,8 +203,7 @@ public function findBySingleFieldReturnsKeyField(): void { $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertArrayHasKey('key', $result[0]); - $this->assertEquals(0, $result[0]['key']); + $this->assertSame(0, $result[0]['key']); } /** @@ -178,8 +214,7 @@ public function findByMultipleFieldsWithANDLogic(): void $result = $this->noneDB->find($this->testDbName, ['age' => 25, 'active' => true]); // Only john has age=25 AND active=true - $this->assertCount(1, $result); - $this->assertEquals('john', $result[0]['username']); + $this->assertSame([$this->expectedAll()[0]], $result); } /** @@ -189,8 +224,7 @@ public function findNoMatchReturnsEmptyArray(): void { $result = $this->noneDB->find($this->testDbName, ['username' => 'nonexistent']); - $this->assertIsArray($result); - $this->assertCount(0, $result); + $this->assertSame([], $result); } /** @@ -200,8 +234,8 @@ public function findMultipleMatches(): void { $result = $this->noneDB->find($this->testDbName, ['age' => 25]); - // john and bob both have age=25 - $this->assertCount(2, $result); + // john and bob both have age=25, in key order + $this->assertSame([0, 2], $this->keys($result)); } /** @@ -211,8 +245,7 @@ public function findByBooleanField(): void { $result = $this->noneDB->find($this->testDbName, ['active' => false]); - $this->assertCount(1, $result); - $this->assertEquals('bob', $result[0]['username']); + $this->assertSame([$this->expectedAll()[2]], $result); } /** @@ -222,8 +255,9 @@ public function findByIntegerField(): void { $result = $this->noneDB->find($this->testDbName, ['age' => 35]); - $this->assertCount(1, $result); - $this->assertEquals('alice', $result[0]['username']); + $this->assertSame([$this->expectedAll()[3]], $result); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['age' => 35.0]), 'identical (===): 35.0 is not 35'); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['age' => '35'])); } /** @@ -233,7 +267,8 @@ public function findByNonExistentField(): void { $result = $this->noneDB->find($this->testDbName, ['nonexistent' => 'value']); - $this->assertCount(0, $result); + $this->assertSame([], $result); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['nonexistent' => null]), 'null matches only an existing field'); } /** @@ -249,7 +284,8 @@ public function findUsesStrictTypeComparison(): void $result = $this->noneDB->find('typetest', ['value' => 25]); // Only integer 25 should match - $this->assertCount(1, $result); + $this->assertSame([['value' => 25, 'key' => 1]], $result); + $this->assertSame([['value' => '25', 'key' => 0]], $this->noneDB->find('typetest', ['value' => '25'])); } // ========================================== @@ -266,12 +302,12 @@ public function findSkipsNullRecords(): void $result = $this->noneDB->find($this->testDbName, 0); - // Should still return 4 items (null is included in raw data) - // But find with filter should skip null + // A deleted record is gone: no placeholder, the others keep their keys + $this->assertSame([0, 2, 3], $this->keys($result)); $filtered = $this->noneDB->find($this->testDbName, ['active' => true]); // jane was deleted, so only john and alice should match - $this->assertCount(2, $filtered); + $this->assertSame(['john', 'alice'], array_column($filtered, 'username')); } /** @@ -284,8 +320,7 @@ public function findAfterDeleteReturnsCorrectRecords(): void $result = $this->noneDB->find($this->testDbName, ['age' => 25]); // Only john should remain with age=25 - $this->assertCount(1, $result); - $this->assertEquals('john', $result[0]['username']); + $this->assertSame([$this->expectedAll()[0]], $result); } // ========================================== @@ -299,9 +334,10 @@ public function findNonExistentDB(): void { $result = $this->noneDB->find('nonexistent-db', 0); - // Should auto-create and return empty - $this->assertIsArray($result); - $this->assertCount(0, $result); + // docs/API.md: [] for a database that doesn't exist; reading never creates a database + $this->assertSame([], $result); + $this->assertDatabaseNotExists('nonexistent-db'); + $this->assertDirectoryDoesNotExist($this->dbPath('nonexistent-db', 'home'), 'no file at all, not even a lock file'); } /** @@ -313,8 +349,7 @@ public function findEmptyDB(): void $result = $this->noneDB->find('emptydb', 0); - $this->assertIsArray($result); - $this->assertCount(0, $result); + $this->assertSame([], $result); } /** @@ -324,10 +359,7 @@ public function findPreservesAllFields(): void { $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertArrayHasKey('username', $result[0]); - $this->assertArrayHasKey('email', $result[0]); - $this->assertArrayHasKey('age', $result[0]); - $this->assertArrayHasKey('active', $result[0]); + $this->assertSame([$this->expectedAll()[0]], $result); } /** @@ -338,7 +370,7 @@ public function findWithEmptyFilter(): void $result = $this->noneDB->find($this->testDbName, []); // Empty array filter should return all records (same as filter 0) - $this->assertCount(4, $result); + $this->assertSame($this->expectedAll(), $result); } /** @@ -347,10 +379,13 @@ public function findWithEmptyFilter(): void public function findWithNullValue(): void { $this->noneDB->insert('nulltest', ['field' => null, 'other' => 'test']); + $this->noneDB->insert('nulltest', ['other' => 'no field']); + $this->noneDB->insert('nulltest', ['field' => false, 'other' => 'false']); $result = $this->noneDB->find('nulltest', ['field' => null]); - $this->assertCount(1, $result); + // docs/API.md: null matches only a field that exists with the value null + $this->assertSame([['field' => null, 'other' => 'test', 'key' => 0]], $result); } /** @@ -359,10 +394,12 @@ public function findWithNullValue(): void public function findWithEmptyStringValue(): void { $this->noneDB->insert('emptytest', ['field' => '', 'other' => 'test']); + $this->noneDB->insert('emptytest', ['field' => null, 'other' => 'null']); + $this->noneDB->insert('emptytest', ['field' => 0, 'other' => 'zero']); $result = $this->noneDB->find('emptytest', ['field' => '']); - $this->assertCount(1, $result); + $this->assertSame([['field' => '', 'other' => 'test', 'key' => 0]], $result); } /** @@ -372,8 +409,7 @@ public function findResultsAreIndexedFromZero(): void { $result = $this->noneDB->find($this->testDbName, ['age' => 25]); - $this->assertArrayHasKey(0, $result); - $this->assertArrayHasKey(1, $result); + $this->assertSame([0, 1], array_keys($result), 'a list, not keyed by record key'); } /** @@ -381,7 +417,7 @@ public function findResultsAreIndexedFromZero(): void */ public function findWithSpecialCharsInDBName(): void { - // v3.2.0: invalid names are rejected on every operation + // Invalid names are rejected on every operation $this->expectException(\InvalidArgumentException::class); $this->noneDB->find('test<>db', 0); } @@ -392,10 +428,12 @@ public function findWithSpecialCharsInDBName(): void public function findWithZeroAsFieldValue(): void { $this->noneDB->insert('zerotest', ['value' => 0]); + $this->noneDB->insert('zerotest', ['value' => false]); + $this->noneDB->insert('zerotest', ['value' => '0']); $result = $this->noneDB->find('zerotest', ['value' => 0]); - $this->assertCount(1, $result); + $this->assertSame([['value' => 0, 'key' => 0]], $result); } /** @@ -405,8 +443,19 @@ public function findWithFalseAsFieldValue(): void { $result = $this->noneDB->find($this->testDbName, ['active' => false]); - $this->assertCount(1, $result); - $this->assertEquals('bob', $result[0]['username']); + $this->assertSame([$this->expectedAll()[2]], $result); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['active' => 0]), 'false is not 0'); + } + + /** + * @test docs/API.md: arrays must be equal as a whole + */ + public function findByArrayValueMatchesTheWholeArray(): void + { + $this->noneDB->insert('arrays', [['tags' => ['a', 'b']], ['tags' => ['b', 'a']], ['tags' => ['a']], ['tags' => ['a', 'b', 'c']]]); + $this->assertSame([0], $this->keys($this->noneDB->find('arrays', ['tags' => ['a', 'b']]))); + $this->assertSame([2], $this->keys($this->noneDB->find('arrays', ['tags' => ['a']]))); + $this->assertSame([], $this->noneDB->find('arrays', ['tags' => 'a'])); } /** diff --git a/tests/Integration/GetDBsTest.php b/tests/Integration/GetDBsTest.php new file mode 100644 index 0000000..891dc2d --- /dev/null +++ b/tests/Integration/GetDBsTest.php @@ -0,0 +1,318 @@ +created[0] = time(); + $this->noneDB->createDB('testdb1'); + $this->noneDB->createDB('testdb2'); + $this->noneDB->createDB('testdb3'); + $this->created[1] = time(); + } + + private function sortedNames(array $names): array + { + sort($names); + return $names; + } + + /** Independent model of the documented size format: B/KB/MB/GB/TB, 2 decimals, comma separator ("2,5 KB") */ + private function sizeOf(int $bytes): string + { + foreach (['TB' => 1024 ** 4, 'GB' => 1024 ** 3, 'MB' => 1024 ** 2, 'KB' => 1024, 'B' => 1] as $unit => $factor) { + if ($bytes >= $factor) { + return str_replace('.', ',', (string)round($bytes / $factor, 2)) . ' ' . $unit; + } + } + return '0 B'; + } + + /** Creation time of a database as FORMAT stores it (docs/STORAGE_FORMAT.md §4.1 createdAt) */ + private function formatCreatedAt(string $name): int + { + $format = json_decode((string)file_get_contents($this->dbPath($name, 'format')), true); + $this->assertIsInt($format['createdAt']); + return $format['createdAt']; + } + + /** The documented entry of a database created in setUp() and still empty */ + private function assertEmptyDbInfo(string $name, $info): void + { + $this->assertIsArray($info); + $this->assertSame(['name', 'createdTime', 'size'], array_keys($info)); + $this->assertSame($name, $info['name']); + $this->assertGreaterThanOrEqual($this->created[0], $info['createdTime']); + $this->assertLessThanOrEqual($this->created[1], $info['createdTime']); + $this->assertSame('0 B', $info['size'], 'an empty data file'); + } + + // ========================================== + // GET ALL DATABASES + // ========================================== + + /** + * @test + */ + public function getDBsWithoutInfoReturnsArray(): void + { + $result = $this->noneDB->getDBs(false); + + $this->assertSame(['testdb1', 'testdb2', 'testdb3'], $this->sortedNames($result)); + } + + /** + * @test + */ + public function getDBsWithoutInfoReturnsNames(): void + { + $result = $this->noneDB->getDBs(false); + + $this->assertCount(3, $result, 'each database once'); + $this->assertSame(['testdb1', 'testdb2', 'testdb3'], $this->sortedNames($result)); + // Index files, lock files and the store files are no databases + $this->noneDB->createFieldIndex('testdb1', 'a'); + $this->assertSame(['testdb1', 'testdb2', 'testdb3'], $this->sortedNames($this->noneDB->getDBs(false))); + } + + /** + * @test + */ + public function getDBsWithInfoReturnsArray(): void + { + $result = $this->noneDB->getDBs(true); + + $this->assertCount(3, $result); + usort($result, function ($a, $b) { return strcmp($a['name'], $b['name']); }); + foreach (['testdb1', 'testdb2', 'testdb3'] as $i => $name) { + $this->assertEmptyDbInfo($name, $result[$i]); + } + } + + /** + * @test + */ + public function getDBsWithInfoReturnsMetadata(): void + { + $this->noneDB->insert('testdb2', ['data' => str_repeat('x', 3000)]); + $result = $this->noneDB->getDBs(true); + usort($result, function ($a, $b) { return strcmp($a['name'], $b['name']); }); + + $this->assertSame(['testdb1', 'testdb2', 'testdb3'], array_column($result, 'name')); + $this->assertEmptyDbInfo('testdb1', $result[0]); + $this->assertSame($this->sizeOf(filesize($this->getDbFilePath('testdb2'))), $result[1]['size']); + $this->assertSame($result[1], $this->noneDB->getDBs('testdb2'), 'the same entry as for the single database'); + } + + /** + * @test + */ + public function getDBsCreatedTimeIsInteger(): void + { + $result = $this->noneDB->getDBs(true); + + foreach ($result as $info) { + $this->assertIsInt($info['createdTime']); + // The creation time stored in FORMAT + $this->assertSame($this->formatCreatedAt($info['name']), $info['createdTime']); + } + } + + /** + * @test + */ + public function getDBsSizeIsString(): void + { + $result = $this->noneDB->getDBs(true); + + $this->assertSame(['0 B', '0 B', '0 B'], array_column($result, 'size')); + } + + // ========================================== + // GET SINGLE DATABASE + // ========================================== + + /** + * @test + */ + public function getDBsSingleDatabaseReturnsInfo(): void + { + $result = $this->noneDB->getDBs('testdb1'); + + $this->assertEmptyDbInfo('testdb1', $result); + } + + /** + * @test + */ + public function getDBsSingleDatabaseHasCreatedTime(): void + { + $result = $this->noneDB->getDBs('testdb1'); + + $this->assertSame($this->formatCreatedAt('testdb1'), $result['createdTime']); + } + + /** + * @test + */ + public function getDBsSingleDatabaseHasSize(): void + { + $result = $this->noneDB->getDBs('testdb1'); + + $this->assertSame('0 B', $result['size']); + } + + /** + * @test + */ + public function getDBsNonExistentDBReturnsFalse(): void + { + // docs/API.md: "false if it doesn't exist; never creates it" (also with autoCreateDB true) + $before = glob($this->testDbDir . '*'); + $this->assertFalse($this->noneDB->getDBs('nonexistent')); + $this->setPrivateProperty('autoCreateDB', false); + $this->assertFalse($this->noneDB->getDBs('nonexistent')); + $this->assertSame($before, glob($this->testDbDir . '*'), 'nothing created'); + // Invalid names are rejected like everywhere else + $this->expectException(\InvalidArgumentException::class); + $this->noneDB->getDBs('bad/name'); + } + + // ========================================== + // MULTIPLE CALLS TEST + // ========================================== + + /** + * @test + */ + public function getDBsMultipleCallsNoError(): void + { + // This tests the fix for "Cannot redeclare FileSizeConvert()" + $result1 = $this->noneDB->getDBs(true); + $result2 = $this->noneDB->getDBs(true); + $result3 = $this->noneDB->getDBs(true); + + $this->assertCount(3, $result1); + $this->assertSame($result1, $result2); + $this->assertSame($result1, $result3); + } + + /** + * @test + */ + public function getDBsWithAndWithoutInfoAlternating(): void + { + $withInfo = $this->noneDB->getDBs(true); + $withoutInfo = $this->noneDB->getDBs(false); + $withInfo2 = $this->noneDB->getDBs(true); + + $this->assertSame($withoutInfo, array_column($withInfo, 'name'), 'the same databases in the same order'); + $this->assertSame($withInfo, $withInfo2); + $this->assertSame(['testdb1', 'testdb2', 'testdb3'], $this->sortedNames($withoutInfo)); + } + + // ========================================== + // FILE SIZE TESTS + // ========================================== + + /** + * @test + */ + public function getDBsFileSizeIncreasesWithData(): void + { + $before = $this->noneDB->getDBs('testdb1'); + $this->assertSame('0 B', $before['size']); + + $this->noneDB->insert('testdb1', array_fill(0, 100, ['data' => 'test value'])); + + // The size of the data file, formatted as documented + clearstatcache(); + $bytes = filesize($this->getDbFilePath('testdb1')); + $this->assertGreaterThan(1024, $bytes); + $this->assertSame($this->sizeOf($bytes), $this->noneDB->getDBs('testdb1')['size']); + } + + /** + * @test + */ + public function getDBsEmptyDBHasMinimalSize(): void + { + $result = $this->noneDB->getDBs('testdb1'); + + // An empty database is an empty JSONL file + $this->assertSame('0 B', $result['size']); + } + + // ========================================== + // EDGE CASES + // ========================================== + + /** + * @test + */ + public function getDBsWithSpecialCharsInName(): void + { + $this->noneDB->createDB('test-special db'); + + $result = $this->noneDB->getDBs('test-special db'); + + $this->assertSame('test-special db', $result['name']); + $this->assertContains('test-special db', $this->noneDB->getDBs(false)); + } + + /** + * @test + */ + public function getDBsWithNumericName(): void + { + $this->noneDB->createDB('12345'); + + $result = $this->noneDB->getDBs('12345'); + + $this->assertSame('12345', $result['name']); + $this->assertSame($result, $this->noneDB->getDBs(12345), 'an integer names the same database'); + $this->assertContains('12345', $this->noneDB->getDBs(false)); + } + + /** + * @test + */ + public function getDBsCreatedTimeIsRecent(): void + { + $beforeCreate = time(); + $this->noneDB->createDB('timetest'); + $afterCreate = time(); + + $result = $this->noneDB->getDBs('timetest'); + + $this->assertGreaterThanOrEqual($beforeCreate, $result['createdTime']); + $this->assertLessThanOrEqual($afterCreate, $result['createdTime']); + } + + /** + * @test + */ + public function getDBsEmptyDirectoryReturnsEmptyArray(): void + { + // Clean all databases + $this->cleanTestDirectory(); + + $result = $this->noneDB->getDBs(false); + + $this->assertIsArray($result); + $this->assertCount(0, $result); + } +} diff --git a/tests/Integration/GlobalIndexTest.php b/tests/Integration/GlobalIndexTest.php new file mode 100644 index 0000000..4f58f58 --- /dev/null +++ b/tests/Integration/GlobalIndexTest.php @@ -0,0 +1,205 @@ +getValue(); + } + + private function globalFile(string $name, string $kind, string $field): string + { + return $this->dbPath($name, $kind, $field); + } + + private function db5(array $extra = []): \noneDB + { + return $this->db(['shardSize' => 50] + $extra); + } + + private function fill(\noneDB $db, string $name): void + { + $db->createFieldIndex($name, 'u'); + $db->createFieldIndex($name, 'g'); + $db->createSpatialIndex($name, 'loc'); + $db->insert($name, array_map(function ($n) { + // shard i holds longitudes 10 * i .. 10 * i + 4 + return ['u' => 'u' . $n, 'g' => 'g' . intdiv($n, 50), 'loc' => ['type' => 'Point', 'coordinates' => [10 * intdiv($n, 50) + ($n % 5), 1]]]; + }, range(0, 199))); + } + + /** @test */ + public function uniqueInsertsDontRewriteTheGlobalIndex(): void + { + $db = $this->db5(); + $this->fill($db, 'gu'); + clearstatcache(); + $gfx = file_get_contents($this->globalFile('gu', 'gfx', 'u')); + $grx = file_get_contents($this->globalFile('gu', 'grx', 'loc')); + for ($i = 0; $i < 10; $i++) { + $db->insert('gu', ['u' => 'new' . $i, 'g' => 'g3', 'loc' => ['type' => 'Point', 'coordinates' => [35, 1]]]); + } + clearstatcache(); + $this->assertSame($gfx, file_get_contents($this->globalFile('gu', 'gfx', 'u')), 'an insert never rewrites it'); + $this->assertSame($grx, file_get_contents($this->globalFile('gu', 'grx', 'loc'))); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db5(); + } + $this->assertSame([200], array_column($db->find('gu', ['u' => 'new0']), 'key'), 'a newer line of a shard is found'); + $this->assertSame([7], array_column($db->find('gu', ['u' => 'u7']), 'key')); + $this->assertSame([], $db->find('gu', ['u' => 'none'])); + $this->assertCount(10, $db->withinBBox('gu', 'loc', 34.5, 0, 36, 2)); + $inBox = array_column($db->withinBBox('gu', 'loc', 34.5, 0, 36, 2), 'key'); + sort($inBox); + $this->assertSame(range(200, 209), $inBox, 'exactly the ten new points at (35, 1)'); + } + } + + /** @test */ + public function shardsThatCantHoldTheValueAreSkipped(): void + { + $db = $this->db5(); + $this->fill($db, 'gs'); + \noneDB::clearStaticCache(); + $db = $this->db5(); + $db->find('gs', ['u' => 'u0']); // warm up: indexes, entries checked once + $before = $this->stat('segmentReads'); + $this->assertSame([123], array_column($db->find('gs', ['u' => 'u123']), 'key')); + $this->assertSame(1, $this->stat('segmentReads') - $before, 'only the shard that holds it'); + $before = $this->stat('segmentReads'); + $this->assertSame([], $db->find('gs', ['u' => 'missing'])); + $this->assertSame(0, $this->stat('segmentReads') - $before, 'no shard holds it'); + $before = $this->stat('segmentReads'); + $this->assertCount(50, $db->withinBBox('gs', 'loc', 20, 0, 24, 2)); + $this->assertSame(1, $this->stat('segmentReads') - $before, 'one shard box overlaps'); + } + + /** @test */ + public function missingOrBrokenEntriesNeverSkip(): void + { + $db = $this->db5(); + $this->fill($db, 'gm'); + $file = $this->globalFile('gm', 'gfx', 'u'); + $content = file_get_contents($file); + $newline = strpos($content, "\n"); + $header = json_decode(substr($content, 0, $newline), true); + // An entry of shard 2 missing: shard 2 is searched + unset($header['sh']['2']); + file_put_contents($file, json_encode($header) . substr($content, $newline)); + // The value moved into shard 2 behind the index's back: only the data knows + $db->update('gm', [['u' => 'u120'], ['set' => ['u' => 'u5']]]); + \noneDB::clearStaticCache(); + $db = $this->db5(); + $this->assertSame([5, 120], array_column($db->find('gm', ['u' => 'u5']), 'key')); + // A broken file: ignored and built again + file_put_contents($file, 'garbage'); + \noneDB::clearStaticCache(); + $db = $this->db5(); + $this->assertSame([5, 120], array_column($db->find('gm', ['u' => 'u5']), 'key')); + $this->assertSame([199], array_column($db->find('gm', ['u' => 'u199']), 'key')); + // A spatial entry with a box that misses its shard's points (stale): the shard's own index decides + $gfile = $this->globalFile('gm', 'grx', 'loc'); + $line = json_decode(trim(file_get_contents($gfile)), true); + $line['sh']['1']['e'] = 'deadbeef'; + $line['le'] = null; + $line['sh']['1']['b'] = [0, 0, 1, 1]; + file_put_contents($gfile, json_encode($line) . "\n"); + \noneDB::clearStaticCache(); + $this->assertCount(50, $this->db5()->withinBBox('gm', 'loc', 10, 0, 14, 2)); + $keys = array_column($this->db5()->withinBBox('gm', 'loc', 10, 0, 14, 2), 'key'); + sort($keys); + $this->assertSame(range(50, 99), $keys, "shard 1's points (longitudes 10..14)"); + } + + /** @test */ + public function lookupsFollowWritesMadeAfterTheShardListWasWorkedOut(): void + { + // A lookup works out once which shards it must check besides the value's holders; later writes of + // this process (an existing shard, a new shard) and of another process must still be seen + $db = $this->db5(); + $db->createFieldIndex('gl', 'u'); + $db->insert('gl', array_map(function ($n) { return ['u' => 'u' . $n]; }, range(0, 149))); + $keys = function () use (&$db) { return array_column($db->find('gl', ['u' => 'x']), 'key'); }; + $this->assertSame([], $keys()); + $db->update('gl', [['u' => 'u120'], ['set' => ['u' => 'x']]]); // shard 2: a newer line + $this->assertSame([120], $keys()); + $db->insert('gl', ['u' => 'x']); // key 150: new shard 3 + $this->assertSame([120, 150], $keys()); + $this->assertSame('', $this->runChild('$db = ' . $this->childDb(['shardSize' => 50]) . '; + $db->update("gl", [["u" => "u7"], ["set" => ["u" => "x"]]]);')); + $this->assertSame([7, 120, 150], $keys(), 'another process wrote shard 0'); + $this->assertSame([], array_column($db->find('gl', ['u' => 'u7']), 'key')); + \noneDB::disableStaticCache(); + try { + $a = $this->db5(); + $b = $this->db5(); + $this->assertSame([7, 120, 150], array_column($a->find('gl', ['u' => 'x']), 'key')); + $b->update('gl', [['u' => 'u60'], ['set' => ['u' => 'x']]]); // another instance, own caches + $this->assertSame([7, 60, 120, 150], array_column($a->find('gl', ['u' => 'x']), 'key')); + } finally { + \noneDB::enableStaticCache(); + } + } + + /** @test */ + public function rewritingTheGlobalIndexAfterACompactionDropsStaleEntries(): void + { + // Compaction replaces shards 0 and 1 (new lock epoch). Then a lookup on two fields builds only a + // new shard's index of "g" and rewrites g's global index under the new epoch: the old entries of + // shards 0 and 1 ("g0" / "g1"; the shards now hold "mv", same length, so no tail) must not ride + // along as if they had been checked under it + $db = $this->db5(); + $db->createFieldIndex('gr', 'g'); + $db->createFieldIndex('gr', 'h'); + $db->insert('gr', array_map(function ($n) { + return ['g' => 'g' . intdiv($n, 50), 'h' => 'h' . intdiv($n, 50)]; + }, range(0, 149))); + foreach ([0, 1] as $shard) { + $db->update('gr', [['g' => 'g' . $shard], ['set' => ['g' => 'mv']]]); // compacted afterwards + } + $this->assertSame([], $db->find('gr', ['h' => 'h9']), "h's global index checked again under the new epoch"); + $db->insert('gr', ['g' => 'g3', 'h' => 'h3']); // shard 3 + $this->assertSame([150], array_column($db->find('gr', ['h' => 'h3', 'g' => 'g3']), 'key'), "only shard 3's index of g is built"); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db5(); + } + $this->assertCount(100, $db->find('gr', ['g' => 'mv']), 'warm ' . var_export($warm, true)); + $this->assertSame([], $db->find('gr', ['g' => 'g0'])); + } + $this->assertTrue($db->checkIntegrity('gr')['ok']); + } + + /** @test */ + public function aCompactedShardIsNotJudgedByItsOldEntry(): void + { + $db = $this->db5(); + $this->fill($db, 'gc'); + for ($n = 50; $n < 70; $n++) { + $db->update('gc', [['u' => 'u' . $n], ['set' => ['u' => 'late' . $n]]]); + } + $db->compact('gc'); // new data files, new epochs: the global entries no longer describe them + \noneDB::clearStaticCache(); + $db = $this->db5(); + $this->assertSame([], $db->find('gc', ['u' => 'u55'])); + $this->assertSame([55], array_column($db->find('gc', ['u' => 'late55']), 'key')); + $this->assertCount(50, $db->withinBBox('gc', 'loc', 10, 0, 14, 2)); + $keys = array_column($db->withinBBox('gc', 'loc', 10, 0, 14, 2), 'key'); + sort($keys); + $this->assertSame(range(50, 99), $keys); + $this->assertTrue($db->checkIntegrity('gc')['ok']); + } +} diff --git a/tests/Integration/HardeningRegressionTest.php b/tests/Integration/HardeningRegressionTest.php new file mode 100644 index 0000000..4b79a32 --- /dev/null +++ b/tests/Integration/HardeningRegressionTest.php @@ -0,0 +1,541 @@ +db(['bufferEnabled' => true]); + $db->insert('up', ['v' => 1]); + $this->assertContains('b', $this->lockState('up')['flags']); + \noneDB::clearStaticCache(); + $start = microtime(true) + 1.0; + $outputs = $this->runParallel(' + $db = ' . $this->childDb(['lockTimeout' => 2]) . '; + // Both hold the shared lock before either upgrades + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { + if ($point === "upgrade:released") { usleep(300000); } + }); + while (microtime(true) < ' . $start . ') { usleep(1000); } + try { + echo count($db->find("up", 0)), " ok"; + } catch (Throwable $e) { + echo get_class($e), ": ", $e->getMessage(); + } + ', 2); + $this->assertSame(['1 ok', '1 ok'], array_values($outputs)); + $this->assertNotContains('b', $this->lockState('up')['flags']); + } + + /** @test */ + public function writerBetweenUpgradeStepsIsRecovered(): void + { + // A-4: while a reader upgrades, a writer that dies leaves a journal entry; the reader must + // recover it instead of clearing it as its own + $db = $this->db(['shardingEnabled' => false, 'bufferEnabled' => true]); + $db->createFieldIndex('a4', 'c'); + $db->insert('a4', ['c' => 'x']); + $db->flush('a4'); + $db->insert('a4', ['c' => 'y']); // buffered: the next read upgrades to flush it + $signal = $this->testDbDir . 'a4.signal'; + @unlink($signal); + \noneDB::clearStaticCache(); + + $script = $this->childScript(' + $db = ' . $this->childDb(['shardingEnabled' => false, 'lockTimeout' => 10]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { + static $done = false; + if ($point === "upgrade:released" && !$done) { + $done = true; + touch(' . var_export($signal, true) . '); + usleep(500000); // the "other writer" takes the lock now + } + }); + echo json_encode(array_column($db->find("a4", ["c" => "x"]), "c")); + '); + $proc = proc_open([PHP_BINARY, $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $deadline = microtime(true) + 10; + while (!file_exists($signal) && microtime(true) < $deadline) { + usleep(1000); + } + $this->assertFileExists($signal); + // A writer that died: field index damaged, journal entry left behind + $lock = fopen($this->dbPath('a4', 'lock'), 'c+b'); + $this->assertTrue(flock($lock, LOCK_EX)); + $fxPath = $this->dbPath('a4', 'fx', 'c'); + $this->assertFileExists($fxPath); + file_put_contents($fxPath, '{"kind":"fx","v":1,"field":"c"}' . "\n{}"); + // "{generation}:{epoch} b j=main": the writer bumped the generation, the epoch stays + $state = explode(' ', trim(fread($lock, 128))); + [$generation, $epoch] = explode(':', $state[0]); + fseek($lock, 0); + fwrite($lock, str_pad(((int)$generation + 1) . ':' . $epoch . ' b j=main', 128)); + fflush($lock); + flock($lock, LOCK_UN); + fclose($lock); + + $out = stream_get_contents($pipes[1]); + $err = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($proc); + $this->assertSame('', trim($err)); + $this->assertSame('["x"]', trim($out), 'the reader rebuilt the damaged field index'); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('a4')['flags'])); + \noneDB::clearStaticCache(); + $this->assertSame(['x'], array_column($this->db(['shardingEnabled' => false])->find('a4', ['c' => 'x']), 'c')); + } + + /** @test */ + public function fractionalLockTimeoutIsHonoured(): void + { + // A-10: 0.2 used to become 0 = wait forever + $db = $this->db(['lockTimeout' => 0.2]); + $db->insert('lt', ['v' => 1]); + $lock = fopen($this->dbPath('lt', 'lock'), 'c+b'); + $this->assertTrue(flock($lock, LOCK_EX)); + $out = $this->runChild(' + $db = ' . $this->childDb(['lockTimeout' => 0.2]) . '; + $t = microtime(true); + try { $db->find("lt", 0); echo "no timeout"; } catch (RuntimeException $e) { printf("%.3f %d", microtime(true) - $t, $e->getCode()); } + '); + flock($lock, LOCK_UN); + fclose($lock); + $this->assertMatchesRegularExpression('/^\d+\.\d+ \d+$/', $out, 'times out'); + [$elapsed, $code] = explode(' ', $out); + $this->assertSame(\noneDB::LOCK_FAILED, (int)$code, 'docs/CONFIGURATION.md: RuntimeException with code noneDB::LOCK_FAILED'); + $this->assertGreaterThanOrEqual(0.19, (float)$elapsed, 'it waited the 0.2 s'); + $this->assertLessThan(2.5, (float)$elapsed, 'after a fraction of a second, not after the 5 s default or never'); + } + + /** @test */ + public function invalidSettingsAreRejected(): void + { + foreach ([['lockTimeout', NAN], ['lockTimeout', INF], ['lockTimeout', -1], ['lockTimeout', 'abc'], ['lockTimeout', [1]], + ['bufferEnabled', 'yes'], ['autoCreateDB', 2], ['fieldIndexEnabled', 'on'], ['shardingEnabled', 'off'], + ['spatialIndexEnabled', 'enabled'], ['shardSize', -1], ['durability', 'sometimes']] as $case) { + try { + $this->db([$case[0] => $case[1]]); + $this->fail("{$case[0]} = " . var_export($case[1], true) . ' must be rejected'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString($case[0], $e->getMessage()); + } + } + foreach (['false', 'FALSE', '0', 0, false] as $off) { + $db = $this->db(['bufferEnabled' => $off]); + $this->assertFalse($db->isBufferingEnabled(), var_export($off, true) . ' means off'); + } + foreach (['true', 'True', '1', 1, true] as $on) { + $this->assertTrue($this->db(['bufferEnabled' => $on])->isBufferingEnabled(), var_export($on, true) . ' means on'); + } + $timeout = new \noneDB\Tests\InternalValue('lockTimeout'); + $this->assertSame(1.5, $timeout->getValue($this->db(['lockTimeout' => '1.5'])), 'a numeric string, fractions kept'); + $this->assertSame(0.0, $timeout->getValue($this->db(['lockTimeout' => 0])), '0 = wait forever'); + $this->assertSame(0.2, $timeout->getValue($this->db(['lockTimeout' => 0.2]))); + } + + /** @test */ + public function serializePrecisionDoesNotChangeStoredFloats(): void + { + // A-12: json_encode() rounds to serialize_precision digits + $before = ini_get('serialize_precision'); + try { + ini_set('serialize_precision', '5'); + $db = $this->db(); + $db->insert('prec', ['p' => 28.980345, 'q' => 0.30000000000000004, 'loc' => ['type' => 'Point', 'coordinates' => [28.980345, 41.008611]]]); + $db->update('prec', [['p' => 28.980345], ['set' => ['r' => 1.0000001]]]); + $this->assertSame('5', ini_get('serialize_precision'), 'the setting is restored'); + \noneDB::clearStaticCache(); + $row = $this->db()->find('prec', 0)[0]; + $this->assertSame(28.980345, $row['p']); + $this->assertSame(0.30000000000000004, $row['q']); + $this->assertSame(1.0000001, $row['r']); + $this->assertSame([28.980345, 41.008611], $row['loc']['coordinates']); + } finally { + ini_set('serialize_precision', (string)$before); + } + } + + /** Files of a database (relative to its directory) except its LOCK; $dir: another dbDir */ + private function filesOf(string $name, ?string $dir = null): array + { + $home = $dir === null ? $this->dbPath($name, 'home') + : rtrim($dir, '/') . '/' . substr(hash_hmac('sha256', 'nonedb4/db/' . $name, self::SECRET), 0, 32) . '.ndb4/'; + $files = []; + $walk = function (string $path) use (&$walk, &$files, $home) { + foreach (@scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($path . $entry)) { + $walk($path . $entry . '/'); + } elseif ($path . $entry !== $home . 'LOCK') { + $files[] = substr($path . $entry, strlen($home)); + } + } + }; + $walk($home); + sort($files); + return $files; + } + + /** @test */ + public function orphanShardFilesAreNeverTakenOver(): void + { + // A-5: a new database growing into shard 1 must not load an old shard 1 (deleted records back). + // Shard files that shards.json does not list are set aside as {name}.orphan.{YmdHis} before a shard of + // that number is created (STORAGE_FORMAT.md 6.3) + $db = $this->db(['shardSize' => 10]); + $db->insert('orph', array_map(function ($n) { return ['n' => $n, 'old' => true]; }, range(1, 30))); + $segments = $this->dbPath('orph', 'gen') . 'segments/'; + $leftovers = []; + foreach (scandir($segments) as $file) { + if (preg_match('/^s[12]\./', $file)) { + $leftovers[$file] = file_get_contents($segments . $file); + } + } + $this->assertArrayHasKey('s1.jsonl', $leftovers); + $this->assertTrue($db->dropDB('orph')['success']); + // The new database: sharded, shard 0 only; the old files of shards 1 and 2 are put back + $db = $this->db(['shardSize' => 10]); + $db->insert('orph', array_map(function ($n) { return ['n' => $n, 'old' => false]; }, range(1, 5))); + $this->assertSame('converted', $db->convertToSharded('orph')['status']); + $segments = $this->dbPath('orph', 'gen') . 'segments/'; + foreach ($leftovers as $file => $content) { + file_put_contents($segments . $file, $content); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 10]); + $db->insert('orph', array_map(function ($n) { return ['n' => $n, 'old' => false]; }, range(6, 15))); + $all = $db->find('orph', 0); + $this->assertCount(15, $all); + $this->assertSame([], array_values(array_filter(array_column($all, 'old'))), 'no old record is visible'); + $info = $db->getDBs('orph'); + $this->assertNotEmpty($info['orphanFiles'] ?? [], 'the set-aside files are reported'); + foreach ($info['orphanFiles'] as $file) { + $this->assertMatchesRegularExpression('/^s1\..*\.orphan\.\d+$/', $file); + } + $this->assertTrue($db->dropDB('orph')['success']); + $this->assertSame([], $this->filesOf('orph'), 'dropDB removes the orphans too'); + } + + /** @test */ + public function newShardsListTheDirectoryOncePerOperation(): void + { + // The leftover check of a new shard listed the whole directory for every new shard (a bulk insert + // into 2000 shards took 80% longer): the number of listings must not depend on the number of new + // shards, of an insert or of the switch to shards + $listings = function () { + $property = new \noneDB\Tests\InternalValue('directoryListings'); + return $property->getValue(); + }; + $rows = function ($from, $count) { + return array_map(function ($n) { return ['n' => $n]; }, range($from, $from + $count - 1)); + }; + $db = $this->db(['shardSize' => 10]); + // The first operation creates the store, which lists the directory (STORAGE_FORMAT.md 3.2): not + // part of what is measured + $db->insert('grow', $rows(0, 10)); + $counts = []; + foreach (['mig3' => 25, 'mig10' => 95] as $name => $count) { + $before = $listings(); + $db->insert($name, $rows(0, $count)); // switches into 3 / 10 shards + $counts[] = $listings() - $before; + $this->assertSame((int)ceil($count / 10), $db->getShardInfo($name)['shards']); + } + $this->assertSame($counts[0], $counts[1], 'the switch into 3 and into 10 shards'); + + // Another process leaves files of shards 3 and 5 that shards.json does not list (STORAGE_FORMAT.md + // 6.3) and writes: both new shards must set them aside + $base = $this->dbPath('grow', 'gen') . 'segments/'; + $leftovers = function (array $files, string $base) { + $this->assertSame('ok', $this->runChild(' + $d = ' . $this->childDb(['shardSize' => 10]) . '; + foreach (' . var_export($files, true) . ' as $suffix => $content) { + file_put_contents(' . var_export($base, true) . ' . $suffix, $content); + } + $d->update("grow", [["n" => 0], ["set" => ["seen" => true]]]); + echo "ok"; + ')); + }; + $leftovers(['s3.jsonl' => '{"n":"old","key":30}' . "\n", 's5.kix' => '{}'], $base); + $counts = []; + foreach ([20, 200] as $count) { // 2 and 20 new shards + $before = $listings(); + $this->assertSame($count, $db->insert('grow', $rows(1000 + $count, $count))['n']); + $counts[] = $listings() - $before; + } + $this->assertSame($counts[0], $counts[1], 'an insert into 2 and into 20 new shards'); + // Above every shard that had files, a new shard needs no listing at all + $before = $listings(); + $this->assertSame(100, $db->insert('grow', $rows(5000, 100))['n']); + $this->assertSame($before, $listings(), 'new shards above every known shard file'); + // A leftover above them, left by another process: found again + $leftovers(['s40.jsonl' => '{"n":"old","key":400}' . "\n"], $base); + $this->assertSame(200, $db->insert('grow', $rows(6000, 200))['n']); // shards 33 to 52 + $this->assertSame(530, $db->count('grow')); + $this->assertSame([], $db->find('grow', ['n' => 'old']), 'no leftover record is visible'); + $orphans = $db->getDBs('grow')['orphanFiles'] ?? []; + sort($orphans); + $this->assertCount(3, $orphans); + $this->assertMatchesRegularExpression('/^s3\.jsonl\.orphan\.\d+$/', $orphans[0]); + $this->assertMatchesRegularExpression('/^s40\.jsonl\.orphan\.\d+$/', $orphans[1]); + $this->assertMatchesRegularExpression('/^s5\.kix\.orphan\.\d+$/', $orphans[2]); + } + + /** @test */ + public function interruptedDropCanBeRepeatedAndNothingComesBack(): void + { + // A-5: the drop dies part-way (after its "d" record, after removing FORMAT; STORAGE_FORMAT.md 8.2): + // the buffered records must not reappear + foreach (['drop:flagged', 'drop:format'] as $stop) { + $this->cleanTestDirectory(); + $db = $this->db(['bufferEnabled' => true]); + $db->insert('rep', ['v' => 'stored']); + $db->flush('rep'); + $db->insert('rep', ['v' => 'buffered']); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + $db = ' . $this->childDb(['bufferEnabled' => true]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { + if ($point === ' . var_export($stop, true) . ') { exit(0); } + }); + $db->dropDB("rep"); + echo "not reached"; + '); + $this->assertSame('', $out, $stop); + $this->assertNotEmpty($this->filesOf('rep'), "$stop: the drop was interrupted"); + \noneDB::clearStaticCache(); + $db = $this->db(['bufferEnabled' => true]); + // The next access finishes the interrupted drop, the database is absent (STORAGE_FORMAT.md 8.3) + $this->assertSame(['success' => false, 'files' => 0, 'error' => 'Database does not exist'], $db->dropDB('rep'), "$stop: can be repeated"); + $this->assertSame([], $this->filesOf('rep'), $stop); + $db->insert('rep', ['v' => 'new']); + $this->assertSame(['new'], array_column($db->find('rep', 0), 'v'), $stop); + } + } + + /** @test */ + public function dbDirWithGlobCharactersWorks(): void + { + // A-7: glob() read [ab] as a pattern: interrupted flushes were never replayed, dropDB deleted nothing + $dir = $this->testDbDir . '[ab]*?/'; + @mkdir($dir, 0777, true); + try { + $config = ['secretKey' => self::SECRET, 'dbDir' => $dir, 'bufferEnabled' => true, 'shardSize' => 4]; + $db = new \noneDB($config); + $db->insert('g', ['v' => 1]); + $db->insert('g', ['v' => 2]); + // An interrupted flush: the buffer renamed, nothing inserted yet + $buffer = $dir . substr(hash_hmac('sha256', 'nonedb4/db/g', self::SECRET), 0, 32) . '.ndb4/generations/1/buffer/'; + $this->assertTrue(rename($buffer . 'pending.jsonl', $buffer . 'flushing.0.jsonl')); + \noneDB::clearStaticCache(); + $db = new \noneDB($config); + $this->assertSame([1, 2], array_column($db->find('g', 0), 'v')); + $db->insert('g', array_map(function ($n) { return ['v' => $n]; }, range(3, 10))); + $db->flush('g'); + $db->createFieldIndex('g', 'v'); + $this->assertSame([7], array_column($db->find('g', ['v' => 7]), 'v')); + $this->assertTrue($db->dropDB('g')['success']); + $this->assertSame([], $this->filesOf('g', $dir)); + } finally { + $remove = function (string $path) use (&$remove) { + if (is_dir($path) && !is_link($path)) { + foreach (scandir($path) as $entry) { + if ($entry !== '.' && $entry !== '..') { + $remove($path . '/' . $entry); + } + } + rmdir($path); + } else { + unlink($path); + } + }; + $remove(rtrim($dir, '/')); + } + } + + /** @test */ + public function compactionKeepsTheHighestUsedKey(): void + { + // A-9: after compaction and a lost index, deleted top keys were handed out again + foreach ([['shardingEnabled' => false], ['shardSize' => 5]] as $config) { + $name = isset($config['shardSize']) ? 'hk2' : 'hk'; + $db = $this->db($config); + $count = isset($config['shardSize']) ? 8 : 5; + $db->insert($name, array_map(function ($n) { return ['n' => $n]; }, range(0, $count - 1))); + $db->delete($name, ['n' => $count - 1]); + $db->delete($name, ['n' => $count - 2]); + $db->compact($name); + \noneDB::clearStaticCache(); + foreach (glob(rtrim($this->testDbDir, '/') . '/*.ndb4/generations/*/segments/*.kix') as $index) { + unlink($index); + } + if (isset($config['shardSize'])) { + // the shard checkpoint's n is what the shards.json recount falls back to + $meta = $this->dbPath($name, 'shards'); + $m = json_decode(file_get_contents($meta), true); + $m['nextKey'] = 0; + file_put_contents($meta, json_encode($m)); + } + $db = $this->db($config); + $db->rebuildIndex($name); + $db->insert($name, ['n' => 'new']); + $this->assertSame($count, $db->find($name, ['n' => 'new'])[0]['key'], json_encode($config)); + } + } + + /** @test */ + public function nestedWriteThroughAnotherInstanceKeepsItsSpatialChanges(): void + { + // A-15: dirty spatial indexes were tracked per instance; the outer instance cleared the journal + $a = $this->db(['shardingEnabled' => false]); + $b = $this->db(['shardingEnabled' => false]); + $a->createSpatialIndex('sp', 'loc'); + $nested = false; + $this->setFaultHook(function ($point) use ($b, &$nested) { + if ($point === 'index:written' && !$nested) { + $nested = true; + $b->insert('sp', ['loc' => ['type' => 'Point', 'coordinates' => [2, 2]]]); + } + }); + $a->insert('sp', ['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + $this->setFaultHook(null); + \noneDB::clearStaticCache(); + $got = array_column($this->db(['shardingEnabled' => false])->withinBBox('sp', 'loc', 0, 0, 3, 3), 'key'); + sort($got); + $this->assertSame([0, 1], $got); + } + + /** @test */ + public function unreadableMetaIsRebuiltFromTheShards(): void + { + // A-16: a zero-length meta (power loss after the rename) made find() throw and count() return 0 + $db = $this->db(['shardSize' => 7]); + $db->insert('meta', array_map(function ($n) { return ['n' => $n]; }, range(0, 19))); + $db->delete('meta', ['n' => 19]); + $meta = $this->dbPath('meta', 'shards'); + foreach (['', 'garbage{'] as $content) { + file_put_contents($meta, $content); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 10]); // configured size differs from the database's 7 + $this->assertSame(range(0, 18), array_column($db->find('meta', 0), 'n')); + $this->assertSame(19, $db->count('meta')); + $this->assertSame(7, json_decode(file_get_contents($meta), true)['shardSize']); + $this->assertSame('meta-rebuilt', array_column($db->getRecoveryLog('meta'), 'action')[0]); + } + $db->insert('meta', ['n' => 'new']); + $this->assertSame(20, $db->find('meta', ['n' => 'new'])[0]['key'], 'key 19 (deleted) is not reused'); + } + + /** @test */ + public function staleMetaNextKeyDoesNotReuseKeys(): void + { + // A-16: the meta rename may not reach the disk before the appended data + $db = $this->db(['shardSize' => 5]); + $db->insert('stalekey', array_map(function ($n) { return ['n' => $n]; }, range(0, 7))); + $meta = $this->dbPath('stalekey', 'shards'); + $m = json_decode(file_get_contents($meta), true); + $m['nextKey'] = 6; // keys 6 and 7 are stored already + file_put_contents($meta, json_encode($m)); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $db->insert('stalekey', ['n' => 'new']); + $this->assertSame(8, $db->find('stalekey', ['n' => 'new'])[0]['key']); + $this->assertCount(9, $db->find('stalekey', 0)); + } + + /** @test */ + public function zeroLengthIndexManifestKeepsTheIndexes(): void + { + // A-16: an empty manifest meant "no indexes": they silently stopped being maintained + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('man', 'c'); + $db->createSpatialIndex('man', 'loc'); + $db->insert('man', ['c' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + file_put_contents($this->dbPath('man', 'indexes'), ''); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('man', ['c' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [2, 2]]]); + $this->assertContains('c', $db->getFieldIndexes('man')['fields']); + $this->assertContains('loc', $db->getSpatialIndexes('man')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0, 1], array_column($db->find('man', ['c' => 'x']), 'key'), 'the index followed the insert'); + $got = array_column($db->withinBBox('man', 'loc', 0, 0, 3, 3), 'key'); + sort($got); + $this->assertSame([0, 1], $got); + } + + /** @test */ + public function updatingAnotherFieldLeavesTheFieldIndexAlone(): void + { + // T2-6: every update rewrote every field index file of the segment + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('t26', 'c'); + $db->insert('t26', [['c' => 'x', 'v' => 1], ['c' => 'y', 'v' => 2]]); + $fxPath = $this->dbPath('t26', 'fx', 'c'); + clearstatcache(); + // (Docker Desktop's file sharing renumbers inodes by itself: there the mtime stands in) + $identity = function ($f) { return file_exists('/.dockerenv') ? filemtime($f) : fileinode($f); }; + $before = [$identity($fxPath), file_get_contents($fxPath)]; + $db->update('t26', [['c' => 'x'], ['set' => ['v' => 10]]]); + clearstatcache(); + $this->assertSame($before, [$identity($fxPath), file_get_contents($fxPath)], 'not rewritten'); + $db->update('t26', [['c' => 'x'], ['set' => ['c' => 'z']]]); + $this->assertSame([0], array_column($db->find('t26', ['c' => 'z']), 'key')); + $this->assertSame([], $db->find('t26', ['c' => 'x'])); + } + + /** @test */ + public function interruptedMigrateKeyCanBeRepeated(): void + { + // T2-13: a migrateKey that died half-way left databases nobody could open or migrate again. The + // rotation (STORAGE_FORMAT.md 9.2, 9.3) dies after marking its targets, between and after the + // renames of the two database directories, after its commit and after the STORE copies: the next + // run finishes it + $newKey = 'a-new-secret-key-for-the-test-0123456789'; + $setup = function () { + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 3]); + $db->createFieldIndex('mk', 'c'); + $db->createSpatialIndex('mk', 'loc'); + $db->insert('mk', array_map(function ($n) { + return ['n' => $n, 'c' => 'c' . ($n % 2), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n]]]; + }, range(0, 7))); + $db->insert('mk2', [['v' => 'a'], ['v' => 'b']]); + \noneDB::clearStaticCache(); + }; + foreach ([['rotate:marked', 1], ['rotate:renamed', 1], ['rotate:renamed', 2], ['rotate:committed', 1], + ['maintenance:store-written', 1]] as [$stopAt, $count]) { + $stopAfter = "$stopAt #$count"; + $setup(); + $out = $this->runChild(' + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($point) use (&$n) { + if ($point === ' . var_export($stopAt, true) . ' && ++$n === ' . $count . ') { exit(0); } + }); + noneDB::migrateKey(' . var_export($this->testDbDir, true) . ', ' . var_export(self::SECRET, true) . ', ' . var_export($newKey, true) . '); + echo "completed"; + '); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $newKey); + $this->assertSame('', $out, "stopped at $stopAfter"); + $this->assertTrue($result['success'], "stopped at $stopAfter: " . json_encode($result) . $out); + $db = new \noneDB(['secretKey' => $newKey, 'dbDir' => $this->testDbDir, 'shardSize' => 3]); + $this->assertSame(range(0, 7), array_column($db->find('mk', 0), 'n'), "stop $stopAfter"); + $this->assertSame([1, 3, 5, 7], array_column($db->find('mk', ['c' => 'c1']), 'n'), "stop $stopAfter: field index"); + $this->assertCount(3, $db->withinBBox('mk', 'loc', 0, 0, 2.5, 2.5), "stop $stopAfter: spatial index"); + $this->assertSame(['a', 'b'], array_column($db->find('mk2', 0), 'v'), "stop $stopAfter"); + $this->assertSame([], $this->filesOf('mk'), 'nothing left under the old key'); + } + } +} diff --git a/tests/Integration/HeldDataTest.php b/tests/Integration/HeldDataTest.php new file mode 100644 index 0000000..2b7cf72 --- /dev/null +++ b/tests/Integration/HeldDataTest.php @@ -0,0 +1,503 @@ + 10, 'cacheMemoryLimit' => $this->limit()]; + $db = $this->db($config); + $db->insert('hd', array_map(function ($n) { return ['u' => 'u' . $n, 'n' => $n]; }, range(0, $records - 1))); + $db->createFieldIndex('hd', 'u'); + \noneDB::clearStaticCache(); + Testing::set('cacheMemoryLimit', $config['cacheMemoryLimit'] === 'auto' ? null : $config['cacheMemoryLimit']); + return $this->db($config); + } + + private function stat(string $name): int + { + return Testing::get('indexStats')[$name]; + } + + private function dataFile(int $shard): string + { + return $this->dbPath('hd', 'data', $shard); + } + + private function held(int $shard): ?array + { + return Testing::get('staticFieldIndexCache')[$this->dataFile($shard)] ?? null; + } + + /** @test */ + public function aShardInRepeatedUseAnswersFromItsHeldBytesAsTheDiskDoes(): void + { + $db = $this->shards(); + $before = $this->stat('dataHeld'); + $this->assertSame(['u' => 'u35', 'n' => 35, 'key' => 35], $db->find('hd', ['u' => 'u35'])[0]); + $this->assertSame($before, $this->stat('dataHeld'), 'the first operation on a shard reads its record only'); + $this->assertNull($this->held(3)); + $this->assertSame(['u' => 'u36', 'n' => 36, 'key' => 36], $db->find('hd', ['u' => 'u36'])[0]); + $this->assertSame($before + 1, $this->stat('dataHeld'), 'its second operation holds the file'); + $this->assertSame((string)file_get_contents($this->dataFile(3)), $this->held(3)['raw'], 'the bytes are the file\'s'); + // Nothing of the file is needed now: the record comes from what is held + $file = $this->dataFile(3); + rename($file, $file . '.away'); + $reads = $this->stat('heldReads'); + try { + $this->assertSame(['u' => 'u37', 'n' => 37, 'key' => 37], $db->find('hd', ['u' => 'u37'])[0]); + $this->assertSame([['u' => 'u30', 'n' => 30, 'key' => 30], ['u' => 'u31', 'n' => 31, 'key' => 31]], + array_values($db->find('hd', ['key' => [30, 31]])), 'several keys of the shard too'); + } finally { + rename($file . '.away', $file); + } + $this->assertSame($reads + 2, $this->stat('heldReads')); + $this->assertSame($before + 1, $this->stat('dataHeld'), 'held once'); + } + + /** @test */ + public function fewSegmentsHoldNothing(): void + { + $db = $this->shards(['shardSize' => 10], 160); // 16 shards: the limit of "few" (Segments::FEW_SEGMENTS) + $before = $this->stat('dataHeld'); + for ($i = 0; $i < 12; $i++) { + $this->assertCount(1, $db->find('hd', ['u' => 'u' . (30 + $i % 10)])); + } + $this->assertSame($before, $this->stat('dataHeld')); + } + + /** @test */ + public function aFewSegmentIndexThatStaysABodyHoldsNothing(): void + { + // Two shards. Lines of about 20 bytes: the arrays of a shard (80 bytes a key) don't fit between what the process + // uses and half of the limit, twice the file does. The body stays, its operations count on, and still nothing is held + $config = ['shardSize' => 8000, 'cacheMemoryLimit' => $this->limit()]; + $db = $this->db($config); + foreach (array_chunk(range(0, 15999), 4000) as $part) { + $db->insert('hd', array_map(function ($n) { return ['u' => $n % 7]; }, $part)); + } + \noneDB::clearStaticCache(); + $config['cacheMemoryLimit'] = 2 * (memory_get_usage() + 640000); + Testing::set('cacheMemoryLimit', $config['cacheMemoryLimit']); + $db = $this->db($config); + $size = filesize($this->dataFile(0)); + $this->assertLessThan(240000, $size, 'precondition: short lines'); + $before = $this->stat('dataHeld'); + $fits = []; + for ($i = 0; $i < 8; $i++) { + $this->assertSame([$i, $i + 7], array_column($db->find('hd', ['key' => [$i, $i + 7]]), 'key')); + $fits[] = Testing::callStatic('CacheManager::decodeFits', 2 * $size); // here: PHPUnit's own classes load below + } + $this->assertSame($before, $this->stat('dataHeld'), 'nothing held'); + $this->assertSame(array_fill(0, 8, true), $fits, 'precondition: the bytes would fit at every operation'); + $index = Testing::get('staticIndexCache')[$this->dbPath('hd', 'kix', 0)]; + $this->assertNotSame('', $index['kb'], 'precondition: the index stays a body'); + $this->assertGreaterThanOrEqual($size, $index['uses'] * 131072, 'precondition: its operations paid for the file'); + } + + /** @test */ + public function aProcessWithoutALimitHoldsNothing(): void + { + $db = $this->shards(['cacheMemoryLimit' => 0]); + $before = $this->stat('dataHeld'); + for ($i = 0; $i < 12; $i++) { + $this->assertCount(1, $db->find('hd', ['u' => 'u' . (30 + $i % 10)])); + } + $this->assertSame($before, $this->stat('dataHeld')); + } + + /** @test */ + public function bytesThatDontFitTheBudgetAreNotHeld(): void + { + // Under the limit, so nothing is evicted and the shard's uses count; over half of it, where nothing more is loaded + $db = $this->shards(['cacheMemoryLimit' => (int)(1.5 * memory_get_usage())]); + $this->assertSame('', Testing::get('staticIndexCache') === [] ? '' : 'precondition: a new process'); + $before = $this->stat('dataHeld'); + for ($i = 0; $i < 12; $i++) { + $this->assertCount(1, $db->find('hd', ['u' => 'u' . (30 + $i % 10)])); + } + $this->assertSame($before, $this->stat('dataHeld')); + } + + /** @test */ + public function aLargerFileWaitsForMoreUses(): void + { + // Lines of about 35 KB: a shard of 10 is about 350 KB, more than two uses pay for (131072 bytes each), less than three + $db = $this->db(['shardSize' => 10, 'cacheMemoryLimit' => $this->limit()]); + $pad = str_repeat('x', 35000); + $db->insert('hd', array_map(function ($n) use ($pad) { return ['u' => 'u' . $n, 'p' => $pad]; }, range(0, 199))); + $db->createFieldIndex('hd', 'u'); + \noneDB::clearStaticCache(); + Testing::set('cacheMemoryLimit', $this->limit()); + $db = $this->db(['shardSize' => 10, 'cacheMemoryLimit' => $this->limit()]); + $size = filesize($this->dataFile(3)); + $this->assertTrue($size > 2 * 131072 && $size <= 3 * 131072, 'precondition: the size of the file'); + $before = $this->stat('dataHeld'); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertSame($before, $this->stat('dataHeld'), 'two uses'); + $db->find('hd', ['u' => 'u37']); + $this->assertSame($before + 1, $this->stat('dataHeld'), 'three uses'); + } + + /** @test */ + public function aWriterHoldsNothingAndUsesWhatIsCurrent(): void + { + $db = $this->shards(); + $before = $this->stat('dataHeld'); + $db->find('hd', ['u' => 'u35']); + // A write by filter reads records under the exclusive lock: bytes a rollback may take back aren't held + $this->assertSame(1, $db->update('hd', [['n' => 36], ['set' => ['m' => 1]]])['n']); + $this->assertSame(1, $db->update('hd', [['n' => 37], ['set' => ['m' => 1]]])['n']); + $this->assertSame($before, $this->stat('dataHeld')); + } + + /** @test */ + public function newerLinesComeFromTheFile(): void + { + $config = ['shardSize' => 10, 'cacheMemoryLimit' => $this->limit()]; + $db = $this->shards(); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertNotNull($this->held(3)); + // This process, then another one: same epoch, lines after the bytes held + $kix = $this->dbPath('hd', 'kix', 3); + $epoch = Testing::get('staticIndexCache')[$kix]['e']; + $this->assertSame(1, $db->update('hd', [['key' => 35], ['set' => ['m' => 'mine']]])['n']); + $this->assertSame('mine', $db->find('hd', ['key' => 35])[0]['m']); + $both = array_values($db->find('hd', ['key' => [30, 35]])); + $this->assertSame([30, 35], array_column($both, 'key'), 'a line held and a newer one in one read'); + $this->assertSame('mine', $both[1]['m']); + $this->assertSame('{"n":1}', $this->runChild('$db = ' . $this->childDb($config) . '; echo json_encode(["n" => $db->update("hd", [["key" => 36], ["set" => ["m" => "theirs"]]])["n"]]);')); + $this->assertSame('theirs', $db->find('hd', ['key' => 36])[0]['m']); + $this->assertSame('mine', $db->find('hd', ['u' => 'u35'])[0]['m']); + $this->assertSame(37, $db->find('hd', ['key' => 37])[0]['n'], 'the lines held still answer'); + $this->assertSame($epoch, Testing::get('staticIndexCache')[$kix]['e'], 'no repair: nothing looked wrong'); + } + + /** @test */ + public function aDataFileReplacedByThisProcessDropsTheBytesHeld(): void + { + $db = $this->shards(); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $stale = $this->held(3)['raw']; + $this->assertSame(1, $db->update('hd', [['key' => 30], ['set' => ['m' => str_repeat('y', 40)]]])['n']); + $db->compact('hd'); + $this->assertNotSame($stale, (string)file_get_contents($this->dataFile(3)), 'precondition: the file is another'); + $epoch = $this->readKeyIndexFile($this->dbPath('hd', 'kix', 3))['header']['e']; + $this->assertSame(39, $db->find('hd', ['key' => 39])[0]['n']); + $this->assertSame(['u' => 'u38', 'n' => 38, 'key' => 38], $db->find('hd', ['u' => 'u38'])[0]); + $held = $this->held(3); + if ($held !== null) { + $this->assertSame((string)file_get_contents($this->dataFile(3)), $held['raw'], 'what is held is the new file'); + } + $this->assertSame($epoch, $this->readKeyIndexFile($this->dbPath('hd', 'kix', 3))['header']['e'], 'no repair: nothing looked wrong'); + } + + /** + * @test CONFIGURATION.md: after disableStaticCache() each new instance keeps its own cache. Bytes held are its + * own too, and what another instance writes reaches it through the lock state as it reaches another process. + */ + public function anInstanceWithItsOwnCacheHoldsItsOwnBytesAndAnswersWhatIsCurrent(): void + { + $config = ['shardSize' => 10, 'cacheMemoryLimit' => $this->limit()]; + $this->shards(); + \noneDB::clearStaticCache(); + \noneDB::disableStaticCache(); + try { + $db = $this->db($config); + $other = $this->db($config); + $before = $this->stat('dataHeld'); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertSame($before + 1, $this->stat('dataHeld')); + $file = $this->dataFile(3); + $this->assertSame((string)file_get_contents($file), Testing::getOf($db, 'fieldIndexCache')[$file]['raw']); + $this->assertArrayNotHasKey($file, Testing::getOf($other, 'fieldIndexCache')); + $this->assertArrayNotHasKey($file, Testing::get('staticFieldIndexCache')); + $this->assertSame(1, $other->update('hd', [['key' => 35], ['set' => ['n' => -35]]])['n']); + $other->compact('hd'); + $this->assertSame(['u' => 'u35', 'n' => -35, 'key' => 35], $db->find('hd', ['u' => 'u35'])[0]); + $this->assertSame(['u' => 'u36', 'n' => 36, 'key' => 36], $db->find('hd', ['u' => 'u36'])[0]); + } finally { + \noneDB::enableStaticCache(); + } + } + + /** @test */ + public function aDataFileReplacedByAnotherProcessDropsTheBytesHeld(): void + { + $config = ['shardSize' => 10, 'cacheMemoryLimit' => $this->limit()]; + $db = $this->shards(); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertNotNull($this->held(3)); + $this->assertSame('1', $this->runChild('$db = ' . $this->childDb($config) . '; $db->update("hd", [["key" => 30], ["set" => ["m" => "' . str_repeat('y', 40) . '"]]]); echo (int)$db->compact("hd")["success"];')); + $epoch = $this->readKeyIndexFile($this->dbPath('hd', 'kix', 3))['header']['e']; + $this->assertSame(['u' => 'u38', 'n' => 38, 'key' => 38], $db->find('hd', ['u' => 'u38'])[0]); + $this->assertSame(str_repeat('y', 40), $db->find('hd', ['key' => 30])[0]['m']); + $this->assertSame($epoch, $this->readKeyIndexFile($this->dbPath('hd', 'kix', 3))['header']['e'], 'no repair: nothing looked wrong'); + } + + /** @test */ + public function checkIntegrityReadsTheDiskNotTheBytesHeld(): void + { + $db = $this->shards(); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertNotNull($this->held(3)); + $file = $this->dataFile(3); + $bytes = (string)file_get_contents($file); + $at = strpos($bytes, '"u37"'); + $bytes[$at + 1] = 'z'; + $bytes[$at + 2] = 'z'; + $bytes = substr_replace($bytes, '{', strpos($bytes, '"u38"') - 5, 1); // a line that doesn't decode + file_put_contents($file, $bytes); + $this->assertFalse($db->checkIntegrity('hd')['ok'], 'the damage on disk is reported'); + } + + /** @test */ + public function evictedBytesAreReadAgainAndAnswersStayTheSame(): void + { + $db = $this->shards(); + $db->find('hd', ['u' => 'u35']); + $db->find('hd', ['u' => 'u36']); + $this->assertNotNull($this->held(3)); + Testing::set('cacheMemoryLimit', 1); + $db->find('hd', ['u' => 'u105']); // loads something: the cache is looked at when the lock is released + $this->assertNull($this->held(3), 'dropped like any cached index'); + Testing::set('cacheMemoryLimit', $this->limit()); + $this->assertSame(['u' => 'u38', 'n' => 38, 'key' => 38], $db->find('hd', ['u' => 'u38'])[0]); + } + + /** @test */ + public function aWholeReadOfASegmentWithoutGarbageSplitsItsLines(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => $n, 'p' => str_repeat('x', $n % 300)]; }, range(0, 4999))); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stat('liveReads'); + $records = $db->find('hd', 0); + $this->assertSame(1, $this->stat('liveReads') - $before); + $this->assertSame(range(0, 4999), array_column($records, 'key')); + foreach ([0, 1, 299, 300, 4999] as $n) { + $this->assertSame(['n' => $n, 'p' => str_repeat('x', $n % 300), 'key' => $n], $records[$n]); + } + } + + /** @test */ + public function aReadOfSomeKeysLeavesTheOtherLinesOfTheFileAlone(): void + { + $db = $this->shards(); + $before = $this->stat('liveTries'); + $this->assertSame(['u' => 'u35', 'n' => 35, 'key' => 35], $db->find('hd', ['u' => 'u35'])[0]); + $this->assertSame([30, 31], array_column($db->find('hd', ['key' => [30, 31]]), 'key')); + $this->assertSame($before, $this->stat('liveTries')); + } + + /** @test */ + public function anIndexIsDecodedOnceHoweverOftenItIsUsed(): void + { + $db = $this->shards(); // more segments than "few": its operations go on counting for the data bytes + $before = $this->stat('keyDecodes'); + for ($i = 0; $i < 12; $i++) { + $this->assertCount(1, $db->find('hd', ['u' => 'u' . (30 + $i % 10)])); + } + $this->assertSame($before + 1, $this->stat('keyDecodes'), 'shard 3, the only one read'); + } + + /** @test */ + public function aSegmentWithGarbageIsReadByPosition(): void + { + foreach ([ + 'an update' => function ($db) { $db->update('hd', [['key' => 3], ['set' => ['m' => 1]]]); }, + 'a delete' => function ($db) { $db->delete('hd', ['key' => 3]); }, + ] as $case => $change) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + $change($db); + $expected = $case === 'an update' ? range(0, 9) : array_values(array_diff(range(0, 9), [3])); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stat('liveTries'); + $this->assertSame($expected, array_column($db->find('hd', 0), 'key'), $case); + $this->assertSame(0, $this->stat('liveTries') - $before, $case . ': the index says so, no line is looked at'); + } + } + + /** @test */ + public function aTombstoneAloneIsAReasonNotToSplit(): void + { + // Compaction keeps the highest key by a tombstone: no garbage line, one that is no record + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => $n]; }, range(0, 99))); + $db->delete('hd', ['key' => 99]); + $db->compact('hd'); + $header = $this->readKeyIndexFile($this->dbPath('hd', 'kix'))['header']; + $this->assertSame([0, 1], [$header['d'], $header['t']], 'precondition: a tombstone, no garbage'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stat('liveTries'); + $this->assertSame(range(0, 98), array_column($db->find('hd', 0), 'key')); + $this->assertSame(0, $this->stat('liveTries') - $before); + } + + /** @test */ + public function aKeyTheLinesDontHoldIsLeftToTheReadByPosition(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => $n]; }, range(0, 99))); + $kix = $this->dbPath('hd', 'kix'); + $checkpoint = $this->readKeyIndexFile($kix); + \noneDB::clearStaticCache(); + // One key more than there are lines, the bytes of the file unchanged: every line is where the index says + $entries = $checkpoint['entries']; + $entries[100] = $entries[99]; + $this->writeKeyIndexFile($kix, $checkpoint['header'], $entries); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(range(0, 99), array_column($db->find('hd', 0), 'key')); + $this->assertArrayNotHasKey(100, $this->readKeyIndexFile($kix)['entries'], 'the read by position repaired the index'); + } + + /** @test */ + public function aShardWithoutCandidatesIsNotRead(): void + { + $db = $this->db(['shardSize' => 10]); + $db->insert('hd', array_map(function ($n) { return ['a' => $n % 2, 'b' => ($n + 1) % 2]; }, range(0, 39))); + $db->createFieldIndex('hd', 'a'); + $db->createFieldIndex('hd', 'b'); + // Both fields are in every shard, the two values never in one record: the filters let every shard through, its candidates are none + $reads = Testing::get('segmentReads'); + $this->assertSame([], $db->find('hd', ['a' => 0, 'b' => 0])); + $this->assertSame($reads, Testing::get('segmentReads')); + } + + /** @test */ + public function aFieldWithoutAnIndexGetsNone(): void + { + $db = $this->db(['shardSize' => 10]); + $db->insert('hd', array_map(function ($n) { return ['a' => $n % 2, 'plain' => 'p' . $n]; }, range(0, 39))); + $db->createFieldIndex('hd', 'a'); + $this->assertSame([3], array_column($db->find('hd', ['a' => 1, 'plain' => 'p3']), 'key')); + $this->assertSame([3], array_column($db->find('hd', ['plain' => 'p3']), 'key')); + $this->assertSame([], glob(dirname($this->dbPath('hd', 'fx', 'a', 0)) . '/*plain*'), 'a filter builds no index of its own'); + } + + /** @test */ + public function linesOutOfKeyOrderAreSorted(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => 100 + $n]; }, range(0, 99))); // past the checkpoint threshold + $data = $this->dbPath('hd', 'data'); + $kix = $this->dbPath('hd', 'kix'); + $checkpoint = $this->readKeyIndexFile($kix); + \noneDB::clearStaticCache(); + // Keys 1 and 2 swapped, each line where the index now says: every line live and in its place, out of key order + $lines = explode("\n", rtrim((string)file_get_contents($data), "\n")); + [$lines[1], $lines[2]] = [$lines[2], $lines[1]]; + file_put_contents($data, implode("\n", $lines) . "\n"); + $entries = $checkpoint['entries']; + [$entries[1], $entries[2]] = [$entries[2], $entries[1]]; + $this->writeKeyIndexFile($kix, $checkpoint['header'], $entries); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->stat('liveReads'); + $records = $db->find('hd', 0); + $this->assertSame(1, $this->stat('liveReads') - $before); + $this->assertSame(range(0, 99), array_column($records, 'key')); + $this->assertSame(range(100, 199), array_column($records, 'n')); + } + + /** @test */ + public function aLineThatIsNotWhereTheIndexSaysIsLeftToTheReadByPosition(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => 100 + $n]; }, range(0, 99))); + $data = $this->dbPath('hd', 'data'); + $this->assertFileExists($this->dbPath('hd', 'kix'), 'precondition: an index on disk the swap goes behind'); + \noneDB::clearStaticCache(); + // Two lines of the same length swapped behind the index: each is a record, neither where the index says + $lines = explode("\n", rtrim((string)file_get_contents($data), "\n")); + [$lines[1], $lines[2]] = [$lines[2], $lines[1]]; + file_put_contents($data, implode("\n", $lines) . "\n"); + $before = strlen($lines[0]) + 1; + $db = $this->db(['shardingEnabled' => false]); + $records = $db->find('hd', 0); + $this->assertSame(range(0, 99), array_column($records, 'key')); + $this->assertSame(range(100, 199), array_column($records, 'n')); + $index = $this->readKeyIndexFile($this->dbPath('hd', 'kix')); + $this->assertSame($before + strlen($lines[2]) + 1, $index['entries'][1][0], 'the read by position found the index stale and repaired it: key 1 is one line later now'); + $this->assertSame($before, $index['entries'][2][0]); + } + + /** @test */ + public function anUnterminatedLastLineIsNoRecordOfAWholeRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', [['n' => 0], ['n' => 1]]); + \noneDB::clearStaticCache(); + file_put_contents($this->dbPath('hd', 'data'), '{"n":2,"key":2', FILE_APPEND); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0, 1], array_column($db->find('hd', 0), 'key')); + } + + /** @test */ + public function aShardWhoseIndexCantAnswerIsScanned(): void + { + $db = $this->shards(); + unlink($this->dbPath('hd', 'fx', 'u', 3)); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 10]); + // Nothing may be built here (as in a snapshot or checkIntegrity()): the shard has no index, its records are read + Testing::set('noWrites', 1); + try { + $this->assertSame(['u' => 'u35', 'n' => 35, 'key' => 35], $db->find('hd', ['u' => 'u35'])[0] ?? null); + } finally { + Testing::set('noWrites', 0); + } + } + + /** + * @test The read of lines compares offsets, not lengths: each line's end is the next one's offset. A wrong length in + * an index gives no wrong record, whichever read comes first + */ + public function aWrongLengthInTheIndexGivesNoWrongRecord(): void + { + foreach (['all records' => function ($db) { return $db->find('hd', 0); }, + 'by key' => function ($db) { return array_merge($db->find('hd', ['key' => 50]), $db->find('hd', ['key' => 99])); }] as $read => $run) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hd', array_map(function ($n) { return ['n' => 100 + $n]; }, range(0, 99))); + $kix = $this->dbPath('hd', 'kix'); + $checkpoint = $this->readKeyIndexFile($kix); + \noneDB::clearStaticCache(); + $entries = $checkpoint['entries']; + $entries[50][1] += 7; + $entries[99][1] -= 3; + $this->writeKeyIndexFile($kix, $checkpoint['header'], $entries); + $records = $run($this->db(['shardingEnabled' => false])); + foreach ($records as $record) { + $this->assertSame(100 + $record['key'], $record['n'], $read); + } + $this->assertSame(range(0, 99), array_column($this->db(['shardingEnabled' => false])->find('hd', 0), 'key'), $read); + } + } +} diff --git a/tests/Integration/IndexHardeningTest.php b/tests/Integration/IndexHardeningTest.php new file mode 100644 index 0000000..e422965 --- /dev/null +++ b/tests/Integration/IndexHardeningTest.php @@ -0,0 +1,328 @@ +db($config); + $scan = $this->db($config + ['spatialIndexEnabled' => false]); + $polygon = ['type' => 'Polygon', 'coordinates' => [[[-10, -10], [10, -10], [10, 10], [-10, 10], [-10, -10]]]]; + foreach ([$indexed, $scan] as $i => $db) { + $results[$i] = [ + 'bbox' => $this->keysOf($db->withinBBox($name, 'loc', -10, -10, 10, 10)), + 'world' => $this->keysOf($db->withinBBox($name, 'loc', -180, -90, 180, 90)), + 'distance' => $this->keysOf($db->withinDistance($name, 'loc', 0, 0, 2000000)), + 'polygon' => $this->keysOf($db->withinPolygon($name, 'loc', $polygon)), + 'nearest' => array_column($db->nearest($name, 'loc', 1, 1, 3), 'key'), + 'withDistance' => array_column($db->query($name)->withDistance('loc', 0, 0)->sort('key')->get(), '_distance', 'key'), + ]; + } + $this->assertSame($results[1], $results[0]); + } + + /** @test */ + public function malformedGeometryIsNotAPoisonPill(): void + { + // B-1: a TypeError in the validator during index maintenance left the journal set; every later + // lock ran the same failing recovery: find, delete and even dropDB failed + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('poison', 'loc'); + $db->insert('poison', ['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + $bad = [ + ['type' => 'LineString', 'coordinates' => [1, 2]], + ['type' => 'LineString', 'coordinates' => ['a' => [1, 2], 'b' => [3, 4]]], + ['type' => 'Polygon', 'coordinates' => ['a' => [[0, 0], [1, 0], [1, 1], [0, 0]]]], + ['type' => true, 'coordinates' => [1, 1]], + ['type' => 0, 'coordinates' => [1, 1]], + ['type' => ['Point'], 'coordinates' => [1, 1]], + ['type' => 'GeometryCollection', 'geometries' => ['x' => ['type' => 'Point', 'coordinates' => [1, 1]]]], + ['type' => 'MultiPolygon', 'coordinates' => [[[1, 2]]]], + ]; + foreach ($bad as $geometry) { + $this->assertFalse($db->validateGeoJSON($geometry)['valid'], json_encode($geometry)); + $this->assertSame(1, $db->insert('poison', ['loc' => $geometry])['n'], json_encode($geometry)); + } + $this->assertCount(1 + count($bad), $db->find('poison', 0)); + $this->assertSame([0], $this->keysOf($db->withinBBox('poison', 'loc', 0, 0, 2, 2))); + $this->assertSpatialIndexEqualsScan('poison', ['shardingEnabled' => false]); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('poison')['flags'])); + $this->assertTrue($db->dropDB('poison')['success']); + } + + /** @test */ + public function leftoverJournalWithMalformedGeometryRecovers(): void + { + // B-1: records with malformed geometry in the data and the journal still set (a writer that died: + // its generation bumped, "j=main" in the state record, STORAGE_FORMAT.md §4.2, §7.3) + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('hurt', 'loc'); + $db->insert('hurt', [['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['loc' => ['type' => 'LineString', 'coordinates' => [1, 2]]]]); + $state = $this->lockState('hurt'); + $this->writeLockState('hurt', ($state['gen'] + 1) . ':' . $state['epoch'] . ' j=main'); + $db = $this->db(['shardingEnabled' => false]); + $this->assertCount(2, $db->find('hurt', 0)); + $this->assertSame([0], $this->keysOf($db->withinBBox('hurt', 'loc', 0, 0, 2, 2))); + $this->assertSame([], preg_grep('/^(j=|J$)/', $this->lockState('hurt')['flags'])); + } + + /** @test */ + public function emptyGeometryCollectionHasNoLocation(): void + { + // B-2: it got the box [0,0,0,0], was "within" every polygon, and nearest() put INF into _distance + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $config) { + $name = isset($config['shardSize']) ? 'gc2' : 'gc'; + $db = $this->db($config); + $db->createSpatialIndex($name, 'loc'); + $db->insert($name, [ + ['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], + ['loc' => ['type' => 'GeometryCollection', 'geometries' => []]], + ['loc' => ['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => []]]]], + ['loc' => ['type' => 'GeometryCollection', 'geometries' => [['type' => 'Point', 'coordinates' => [2, 2]], ['type' => 'GeometryCollection', 'geometries' => []]]]], + ['loc' => ['type' => 'Point', 'coordinates' => [50, 50]]], + ]); + $this->assertSame([0, 3], $this->keysOf($db->withinPolygon($name, 'loc', ['type' => 'Polygon', 'coordinates' => [[[-10, -10], [10, -10], [10, 10], [-10, 10], [-10, -10]]]]))); + $nearest = $db->nearest($name, 'loc', 0, 0, 10); + $this->assertSame([0, 3, 4], array_column($nearest, 'key')); + $this->assertNotFalse(json_encode($nearest), 'no INF distances'); + $this->assertNotFalse(json_encode($db->query($name)->withDistance('loc', 0, 0)->get())); + $this->assertSpatialIndexEqualsScan($name, $config); + } + } + + /** @test */ + public function nanAndInfiniteSpatialArgumentsTerminate(): void + { + // B-5: maxDistance NAN with fewer geometries than k (and k < count) looped forever + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('nan', 'loc'); + $db->insert('nan', [['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['x' => 1], ['x' => 2]]); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + set_time_limit(20); + $db = ' . $this->childDb(['shardingEnabled' => false]) . '; + echo count($db->nearest("nan", "loc", 1, 1, 2, ["maxDistance" => NAN])), ","; + echo count($db->nearest("nan", "loc", NAN, 1, 2)), ","; + echo count($db->withinDistance("nan", "loc", 1, 1, NAN)), ","; + echo count($db->withinBBox("nan", "loc", NAN, 0, 2, 2)), ","; + echo count($db->nearest("nan", "loc", 1, 1, 2, ["maxDistance" => INF])), ","; + echo count($db->withinDistance("nan", "loc", 179, -89, INF)), ","; + echo count($db->withinBBox("nan", "loc", -INF, -INF, INF, INF)); + '); + $this->assertSame('0,0,0,0,1,1,1', $out); + } + + /** @test */ + public function floatLookupsDoNotDependOnPrecision(): void + { + // B-6: the bucket key was (string)$float, which depends on the precision setting + $before = ini_get('precision'); + try { + $db = $this->db(['shardSize' => 3]); + $db->createFieldIndex('fl', 'p'); + $db->insert('fl', [['p' => 0.1], ['p' => 0.2], ['p' => 1.0], ['p' => 1], ['p' => '1'], ['p' => 0.1]]); + ini_set('precision', '17'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 3]); + $this->assertSame([0, 5], array_column($db->find('fl', ['p' => 0.1]), 'key')); + $this->assertSame([2], array_column($db->find('fl', ['p' => 1.0]), 'key')); + $this->assertSame([0, 5], $db->getKeysFromFieldIndex('fl', 'p', 0.1)); + } finally { + ini_set('precision', (string)$before); + } + } + + /** @test */ + public function fieldsDifferingOnlyInCaseGetTheirOwnIndexFiles(): void + { + // B-7: on case-insensitive file systems, files named after the raw field names "city" and "City" + // would be one file (I1): names other than [a-z0-9_] get a hash, so "city" and "City" have their own + // files on any file system + $db = $this->db(['shardingEnabled' => false]); + $db->insert('case', [['city' => 'A', 'City' => 'B'], ['city' => 'B', 'City' => 'A']]); + $this->assertTrue($db->createFieldIndex('case', 'city')['success']); + $this->assertTrue($db->createFieldIndex('case', 'City')['success']); + $this->assertCount(2, glob($this->dbPath('case', 'gen') . 'indexes/main.*.fx')); + // "city" is its own token; "City" is not [a-z0-9_]: lower-case prefix + "-" + 12 hex of its sha256 + // (STORAGE_FORMAT.md §5.1; the file is indexes/{segment}.{token}.fx) + $this->assertFileExists($this->dbPath('case', 'gen') . 'indexes/main.city.fx'); + $this->assertFileExists($this->dbPath('case', 'gen') . 'indexes/main.city-' . substr(hash('sha256', 'City'), 0, 12) . '.fx'); + $this->assertSame([1], array_column($db->find('case', ['City' => 'A']), 'key')); + $this->assertSame([0], array_column($db->find('case', ['city' => 'A']), 'key')); + $db->insert('case', ['city' => 'A', 'City' => 'C']); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0, 2], array_column($db->find('case', ['city' => 'A']), 'key')); + $this->assertSame([2], array_column($db->find('case', ['City' => 'C']), 'key')); + } + + /** @test */ + public function rebuildIndexAlsoRebuildsFieldAndSpatialIndexes(): void + { + // B-8: after a manual restore of the data, rebuildIndex() left the other indexes stale + $db = $this->db(['shardingEnabled' => false]); + $db->createFieldIndex('rb', 'c'); + $db->createSpatialIndex('rb', 'loc'); + $db->insert('rb', ['c' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); + $data = $this->dbPath('rb', 'data'); + file_put_contents($data, '{"c":"y","loc":{"type":"Point","coordinates":[5,5]},"key":0}' . "\n"); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->rebuildIndex('rb'); + $this->assertSame([0], array_column($db->find('rb', ['c' => 'y']), 'key')); + $this->assertSame([], $db->find('rb', ['c' => 'x'])); + $this->assertSame([0], $this->keysOf($db->withinBBox('rb', 'loc', 4, 4, 6, 6))); + $this->assertSame([], $db->withinBBox('rb', 'loc', 0, 0, 2, 2)); + } + + /** @test */ + public function missingGlobalSpatialEntryNeverSkipsAShard(): void + { + // B-9: a shard without an entry in the global spatial index was skipped + $db = $this->db(['shardSize' => 3]); + $db->createSpatialIndex('gs', 'loc'); + $db->insert('gs', array_map(function ($i) { + return ['loc' => ['type' => 'Point', 'coordinates' => [$i * 10 - 80, $i * 5 - 40]]]; + }, range(0, 11))); + $global = $this->dbPath('gs', 'grx', 'loc'); + $g = json_decode(trim(file_get_contents($global)), true); + $this->assertArrayHasKey('1', $g['sh']); + unset($g['sh']['1']); + file_put_contents($global, json_encode($g) . "\n"); + \noneDB::clearStaticCache(); + $this->assertCount(12, $this->db(['shardSize' => 3])->withinBBox('gs', 'loc', -180, -90, 180, 90)); + unlink($global); + \noneDB::clearStaticCache(); + $this->assertCount(12, $this->db(['shardSize' => 3])->withinBBox('gs', 'loc', -180, -90, 180, 90)); + $this->assertSpatialIndexEqualsScan('gs', ['shardSize' => 3]); + } + + /** @test */ + public function indexFileNamesNeverExceedTheFileSystemLimit(): void + { + // B-10: the manifest listed the index before it was built; a too long name failed after that. + // (I1): a field takes at most 40 bytes of a file name, so any field name works + $db = $this->db(['shardingEnabled' => false]); + $db->insert('long', ['v' => 1]); + $field = str_repeat('ü', 110); // 220 bytes + $this->assertTrue($db->createFieldIndex('long', $field)['success']); + $this->assertTrue($db->createSpatialIndex('long', $field)['success']); + $this->assertSame([$field], $db->getFieldIndexes('long')['fields']); + $this->assertSame([$field], $db->getSpatialIndexes('long')); + $this->assertSame(1, $db->insert('long', ['v' => 2, $field => 'x'])['n']); + $this->assertSame([1], array_column($db->find('long', [$field => 'x']), 'key')); + foreach ($this->dbFiles('long') as $file) { + $this->assertLessThanOrEqual(255, strlen(basename($file)), $file); + } + + // Index file names hold no database name (STORAGE_FORMAT.md §4, §5.1: {locator}.ndb4/.../indexes/ + // {segment}.{token}.fx): a long database name gets its indexes like any other + $longName = str_repeat('a', 200); + $db->insert($longName, ['v' => 1]); + $this->assertSame(['success' => true, 'indexed' => 1, 'error' => null], $db->createFieldIndex($longName, 'v')); + $this->assertTrue($db->createSpatialIndex($longName, 'loc')['success']); + $this->assertSame(['v'], $db->getFieldIndexes($longName)['fields']); + $this->assertSame(['loc'], $db->getSpatialIndexes($longName)); + $this->assertSame(1, $db->insert($longName, ['v' => 2, 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]])['n']); + $this->assertSame([1], array_column($db->find($longName, ['v' => 2]), 'key')); + $this->assertSame([1], $db->getKeysFromFieldIndex($longName, 'v', 2)); + $this->assertSame([1], array_column($db->withinBBox($longName, 'loc', 0, 0, 2, 2), 'key')); + foreach ($this->dbFiles($longName) as $file) { + $this->assertLessThanOrEqual(255, strlen(basename($file)), $file); + } + $this->assertTrue($db->createFieldIndex(str_repeat('a', 150), 'v')['success']); + } + + /** @test */ + public function selfIntersectionCheckIsNotQuadratic(): void + { + // B-11: every query validated every candidate polygon with a test of all edge pairs + $counter = new \noneDB\Tests\InternalValue('segmentPairTests'); + $ring = []; + $n = 2000; + for ($i = 0; $i < $n; $i++) { + $ring[] = [cos(2 * M_PI * $i / $n) * 10, sin(2 * M_PI * $i / $n) * 10]; + } + $ring[] = $ring[0]; + $polygon = ['type' => 'Polygon', 'coordinates' => [$ring]]; + $db = $this->db(); + $counter->setValue(null, 0); + $this->assertTrue($db->validateGeoJSON($polygon)['valid']); + $this->assertLessThan(20 * $n, $counter->getValue(), 'pairs compared (all pairs: about 2,000,000)'); + + $bowtie = ['type' => 'Polygon', 'coordinates' => [[[0, 0], [2, 2], [2, 0], [0, 2], [0, 0]]]]; + $this->assertFalse($db->validateGeoJSON($bowtie)['valid']); + $touching = ['type' => 'Polygon', 'coordinates' => [[[0, 0], [4, 0], [4, 4], [2, 0], [0, 4], [0, 0]]]]; + $this->assertFalse($db->validateGeoJSON($touching)['valid'], 'a vertex on another edge'); + } + + /** @test */ + public function objectValuesAreIndexedLikeTheirStoredForm(): void + { + // B-12: a stdClass/JsonSerializable value was stored as JSON but not indexed + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $config) { + $name = isset($config['shardSize']) ? 'obj2' : 'obj'; + $db = $this->db($config); + $db->createFieldIndex($name, 'c'); + $db->createSpatialIndex($name, 'loc'); + $db->insert($name, [ + ['c' => 'x', 'loc' => json_decode('{"type":"Point","coordinates":[1,1]}')], + ['c' => 'y', 'loc' => ['type' => 'Point', 'coordinates' => [5, 5]]], + ['c' => 'z', 'loc' => json_decode('{"type":"Point","coordinates":[3,3]}')], + ]); + $db->update($name, [['c' => 'y'], ['set' => ['loc' => json_decode('{"type":"Point","coordinates":[2,2]}')]]]); + $this->assertSame([0, 1, 2], $this->keysOf($db->withinBBox($name, 'loc', 0, 0, 4, 4)), json_encode($config)); + $this->assertSpatialIndexEqualsScan($name, $config); + } + } + + /** @test */ + public function highCardinalityFieldKeepsItsGlobalIndexWithoutRewrites(): void + { + // T2-7: a unique field rewrote the whole global index (all values) on every insert. (I4): the + // global index is a checkpoint of the shards' values, written when a shard's index is, never per + // write (the shards' newer lines are checked instead). + $db = $this->db(['shardSize' => 100]); + $db->createFieldIndex('hc', 'u'); + $db->createFieldIndex('hc', 'g'); + $db->insert('hc', array_map(function ($n) { return ['u' => 'u' . $n, 'g' => 'g' . ($n % 5)]; }, range(0, 149))); + $file = $this->dbPath('hc', 'gfx', 'u'); + $this->assertFileExists($file); + $this->assertFileExists($this->dbPath('hc', 'gfx', 'g')); + $before = file_get_contents($file); + $off = $this->db(['shardSize' => 100]); + foreach ([$db, $off, $db] as $i => $writer) { + $writer->insert('hc', ['u' => 'new' . $i, 'g' => 'g1']); + } + $this->assertSame($before, file_get_contents($file), 'not rewritten by an insert'); + $scan = $this->db(['shardSize' => 100, 'fieldIndexEnabled' => false]); + foreach ([['u' => 'u7'], ['u' => 'u120'], ['u' => 'new1'], ['u' => 'none'], ['g' => 'g1'], ['u' => 'u77', 'g' => 'g2']] as $filter) { + $this->assertSame($scan->find('hc', $filter), $db->find('hc', $filter), json_encode($filter)); + } + // rebuildIndex() writes it again from the data: the three new values (keys 150..152) are in shard 1. + // The filters (STORAGE_FORMAT.md §6.5.2) hold the 150 + 3 unique values: shard 0 its 100, shard 1 its 53; + // 4096 bits (the smallest power of two of at least 3200) + $off->rebuildIndex('hc'); + $this->assertNotSame($before, file_get_contents($file)); + [$header, $filters] = $this->readGlobalFieldFile($file); + $this->assertSame(4096, $header['bits']); + $u = function (int $from, int $to) { return array_map(function ($n) { return 'u' . $n; }, range($from, $to)); }; + $this->assertSame([0 => $this->shardFilter($u(0, 99), 4096), 1 => $this->shardFilter(array_merge($u(100, 149), ['new0', 'new1', 'new2']), 4096)], $filters); + $this->assertSame($scan->find('hc', ['u' => 'new2']), $off->find('hc', ['u' => 'new2'])); + } +} diff --git a/tests/Feature/InsertTest.php b/tests/Integration/InsertTest.php similarity index 66% rename from tests/Feature/InsertTest.php rename to tests/Integration/InsertTest.php index deae8d8..486a36a 100644 --- a/tests/Feature/InsertTest.php +++ b/tests/Integration/InsertTest.php @@ -1,6 +1,6 @@ noneDB->insert($this->testDbName, ['username' => 'john']); - $this->assertArrayHasKey('n', $result); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result, 'docs/API.md: ["n" => 1], no error'); } /** @@ -35,9 +34,8 @@ public function insertSingleRecordStoresData(): void $contents = $this->getDatabaseContents($this->testDbName); - $this->assertCount(1, $contents['data']); - $this->assertEquals('john', $contents['data'][0]['username']); - $this->assertEquals('john@test.com', $contents['data'][0]['email']); + $this->assertSame([['username' => 'john', 'email' => 'john@test.com']], $contents['data']); + $this->assertSame([['username' => 'john', 'email' => 'john@test.com', 'key' => 0]], $this->noneDB->find($this->testDbName, 0)); } /** @@ -53,7 +51,7 @@ public function insertMultipleRecordsReturnsCorrectCount(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertEquals(3, $result['n']); + $this->assertSame(['n' => 3], $result); } /** @@ -69,7 +67,8 @@ public function insertMultipleRecordsStoresAllData(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertCount(2, $contents['data']); + $this->assertSame($data, $contents['data']); + $this->assertSame([0, 1], array_column($this->noneDB->find($this->testDbName, 0), 'key')); } /** @@ -88,11 +87,9 @@ public function insertPreservesDataTypes(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals('text', $contents['data'][0]['string']); - $this->assertEquals(42, $contents['data'][0]['integer']); - $this->assertEquals(3.14, $contents['data'][0]['float']); - $this->assertTrue($contents['data'][0]['boolean']); - $this->assertNull($contents['data'][0]['null']); + // Types are kept exactly (assertSame): int stays int, float stays float, null is stored + $this->assertSame([$data], $contents['data']); + $this->assertSame([$data + ['key' => 0]], $this->noneDB->find($this->testDbName, 0)); } /** @@ -111,8 +108,7 @@ public function insertWithNestedArrays(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertArrayHasKey('address', $contents['data'][0]); - $this->assertEquals('123 Main St', $contents['data'][0]['address']['street']); + $this->assertSame([$data], $contents['data']); } /** @@ -127,7 +123,9 @@ public function insertWithNumericKeys(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertEquals(1, $result['n']); + // Not a list of records (the elements aren't arrays): one record with the fields "0" and "1" + $this->assertSame(['n' => 1], $result); + $this->assertSame([[0 => 'value1', 1 => 'value2', 'key' => 0]], $this->noneDB->find($this->testDbName, 0)); } /** @@ -140,9 +138,8 @@ public function insertAppendsToExistingData(): void $contents = $this->getDatabaseContents($this->testDbName); - $this->assertCount(2, $contents['data']); - $this->assertEquals('first', $contents['data'][0]['username']); - $this->assertEquals('second', $contents['data'][1]['username']); + $this->assertSame([['username' => 'first'], ['username' => 'second']], $contents['data']); + $this->assertSame([0, 1], array_column($this->noneDB->find($this->testDbName, 0), 'key'), 'keys in insertion order'); } /** @@ -150,10 +147,12 @@ public function insertAppendsToExistingData(): void */ public function insertCreatesDBIfNotExists(): void { + $this->assertDatabaseNotExists('newdb'); $result = $this->noneDB->insert('newdb', ['data' => 'test']); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); $this->assertDatabaseExists('newdb'); + $this->assertSame([['data' => 'test', 'key' => 0]], $this->noneDB->find('newdb', 0)); } /** @@ -165,7 +164,8 @@ public function insertToExistingDB(): void $result = $this->noneDB->insert('existingdb', ['data' => 'test']); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); + $this->assertSame([['data' => 'test', 'key' => 0]], $this->noneDB->find('existingdb', 0), 'the first key of a created database is 0'); } /** @@ -175,10 +175,10 @@ public function insertEmptyArray(): void { $result = $this->noneDB->insert($this->testDbName, []); - // v3.2.0: an empty insert is rejected instead of storing an empty record - $this->assertEquals(0, $result['n']); - $this->assertArrayHasKey('error', $result); + // An empty insert is rejected, no empty record is stored (docs/API.md) + $this->assertSame(['n' => 0, 'error' => 'insert data must not be empty'], $result); $this->assertSame([], $this->noneDB->find($this->testDbName, 0)); + $this->assertDatabaseNotExists($this->testDbName); } /** @@ -186,7 +186,7 @@ public function insertEmptyArray(): void */ public function insertWithSpecialCharsInDBName(): void { - // v3.2.0: invalid names are rejected, nothing is written + // Invalid names are rejected, nothing is written try { $this->noneDB->insert('test<>db', ['data' => 'test']); $this->fail('Expected InvalidArgumentException'); @@ -210,9 +210,8 @@ public function insertPreservesInsertionOrder(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals(1, $contents['data'][0]['order']); - $this->assertEquals(2, $contents['data'][1]['order']); - $this->assertEquals(3, $contents['data'][2]['order']); + $this->assertSame($data, $contents['data']); + $this->assertSame([[1, 0], [2, 1], [3, 2]], array_map(function ($r) { return [$r['order'], $r['key']]; }, $this->noneDB->find($this->testDbName, 0))); } /** @@ -227,10 +226,13 @@ public function insertLargeDataset(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertEquals(100, $result['n']); + $this->assertSame(['n' => 100], $result); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertCount(100, $contents['data']); + $this->assertSame($data, $contents['data']); + $found = $this->noneDB->find($this->testDbName, 0); + $this->assertSame(range(0, 99), array_column($found, 'key')); + $this->assertSame(range(0, 99), array_column($found, 'index'), 'record i got key i'); } // ========================================== @@ -244,8 +246,8 @@ public function insertNonArrayReturnsError(): void { $result = $this->noneDB->insert($this->testDbName, 'not an array'); - $this->assertArrayHasKey('error', $result); - $this->assertEquals(0, $result['n']); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); + $this->assertDatabaseNotExists($this->testDbName); } /** @@ -255,7 +257,7 @@ public function insertStringReturnsError(): void { $result = $this->noneDB->insert($this->testDbName, 'string data'); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); } /** @@ -265,7 +267,7 @@ public function insertNullReturnsError(): void { $result = $this->noneDB->insert($this->testDbName, null); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); } /** @@ -275,7 +277,7 @@ public function insertIntegerReturnsError(): void { $result = $this->noneDB->insert($this->testDbName, 123); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); } /** @@ -285,7 +287,7 @@ public function insertBooleanReturnsError(): void { $result = $this->noneDB->insert($this->testDbName, true); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); } /** @@ -297,8 +299,8 @@ public function insertWithReservedKeyFieldReturnsError(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertArrayHasKey('error', $result); - $this->assertStringContainsString('key', strtolower($result['error'])); + $this->assertSame(['n' => 0, 'error' => 'You cannot set key name to key'], $result, 'docs/API.md'); + $this->assertDatabaseNotExists($this->testDbName); } /** @@ -313,7 +315,8 @@ public function insertMultipleWithOneKeyFieldReturnsError(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'You cannot set key name to key'], $result); + $this->assertSame([], $this->noneDB->find($this->testDbName, 0), 'the valid first record is not stored either'); } // ========================================== @@ -333,7 +336,8 @@ public function insertWithUnicodeData(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals('Türkçe Karakter: şüğıöç', $contents['data'][0]['name']); + $this->assertSame([$data], $contents['data']); + $this->assertSame([$data + ['key' => 0]], $this->noneDB->find($this->testDbName, 0)); } /** @@ -346,10 +350,10 @@ public function insertWithVeryLongString(): void $result = $this->noneDB->insert($this->testDbName, $data); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals(10000, strlen($contents['data'][0]['content'])); + $this->assertSame($longString, $contents['data'][0]['content']); } /** @@ -366,7 +370,7 @@ public function insertWithSpecialCharsInValues(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals('', $contents['data'][0]['html']); + $this->assertSame([$data], $contents['data'], 'stored as given, not escaped or changed'); } /** @@ -388,10 +392,7 @@ public function insertDeepNestedArray(): void $this->noneDB->insert($dbName, $data); $contents = $this->getDatabaseContents($dbName); - $this->assertNotNull($contents); - $this->assertArrayHasKey('data', $contents); - $this->assertCount(1, $contents['data']); - $this->assertEquals('deep value', $contents['data'][0]['level1']['level2']['level3']['level4']); + $this->assertSame(['data' => [$data]], $contents); } /** @@ -407,7 +408,7 @@ public function insertWithEmptyStringValues(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertEquals('', $contents['data'][0]['empty']); + $this->assertSame([$data], $contents['data']); } /** @@ -422,11 +423,11 @@ public function insertMixedArrayTypes(): void ['username' => 'user2'], ]; - // When foreach encounters non-array, directInsert becomes true + // A list (key 0 is a record) with an element that isn't one: rejected, nothing stored (docs/API.md) $result = $this->noneDB->insert($this->testDbName, $data); - // Should handle gracefully - $this->assertArrayHasKey('n', $result); + $this->assertSame(['n' => 0, 'error' => 'insert data must be either one record or a list of records'], $result); + $this->assertSame([], $this->noneDB->find($this->testDbName, 0)); } /** @@ -443,10 +444,12 @@ public function insertWithZeroValues(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - // JSON decode may convert 0.0 to 0 (int) - $this->assertEquals(0, $contents['data'][0]['zero_int']); - $this->assertEquals(0, $contents['data'][0]['zero_float']); // JSON doesn't preserve float type for 0.0 + // docs/API.md: "Floats keep their type" - 0.0 is stored as 0.0 and read back as a float + $this->assertSame(0, $contents['data'][0]['zero_int']); + $this->assertSame(0.0, $contents['data'][0]['zero_float']); + $this->assertIsFloat($contents['data'][0]['zero_float']); $this->assertSame('0', $contents['data'][0]['zero_string']); + $this->assertSame([$data + ['key' => 0]], $this->noneDB->find($this->testDbName, 0)); } /** @@ -459,7 +462,7 @@ public function insertWithBooleanFalse(): void $this->noneDB->insert($this->testDbName, $data); $contents = $this->getDatabaseContents($this->testDbName); - $this->assertFalse($contents['data'][0]['active']); + $this->assertSame([['active' => false]], $contents['data']); } /** @@ -473,7 +476,8 @@ public function multipleInsertsInSequence(): void $contents = $this->getDatabaseContents($this->testDbName); - $this->assertCount(3, $contents['data']); + $this->assertSame([['id' => 1], ['id' => 2], ['id' => 3]], $contents['data']); + $this->assertSame([0, 1, 2], array_column($this->noneDB->find($this->testDbName, 0), 'key')); } /** @@ -488,7 +492,7 @@ public function insertObjectAsDataReturnsError(): void $result = $this->noneDB->insert($this->testDbName, $object); - $this->assertArrayHasKey('error', $result); - $this->assertEquals(0, $result['n']); + $this->assertSame(['n' => 0, 'error' => 'insert data must be array'], $result); + $this->assertDatabaseNotExists($this->testDbName); } } diff --git a/tests/Integration/IntegrityIndexTest.php b/tests/Integration/IntegrityIndexTest.php new file mode 100644 index 0000000..40a49e3 --- /dev/null +++ b/tests/Integration/IntegrityIndexTest.php @@ -0,0 +1,189 @@ + $i % 10 === 7 && $i < 70 ? 'many' : 'v' . $i]; + } + return $records; + } + + private function mismatches(array $report): array + { + return array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches']); + } + + public function sizes(): array + { + return ['a page' => [20], 'many pages' => [2000]]; + } + + public function editedBuckets(): array + { + return [ + 'a key under the value of another' => [function (array $map) { unset($map['sv3']); $map['sv4'] = [3, 4]; return $map; }], + 'a key missing' => [function (array $map) { unset($map['sv3']); return $map; }], + 'a value no record has' => [function (array $map) { $map['sghost'] = 3; return $map; }], + 'a list a key short' => [function (array $map) { $map['smany'] = array_slice($map['smany'], 1); return $map; }], + 'another key in a list' => [function (array $map) { $map['smany'][] = 1000000; return $map; }], + ]; + } + + /** + * @test + * @dataProvider editedBuckets + */ + public function aFieldIndexThatHoldsOtherKeysIsReported(callable $edit): void + { + foreach ([20, 2000] as $n) { + $name = 'fi' . $n; + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert($name, $this->records($n)); + $db->createFieldIndex($name, 'c'); + $this->assertSame([], $db->checkIntegrity($name)['indexMismatches'], 'the index as the engine wrote it'); + + $fx = $this->dbPath($name, 'fx', 'c'); + [$header, $map] = $this->readFieldIndexFile($fx); + $this->assertSame($n === 20 ? 0 : 1, min(1, $header['pg']['levels']), 'precondition: pages'); + $this->writeFieldIndexFile($fx, $header, $edit($map)); + \noneDB::clearStaticCache(); + + $report = $this->db($config)->checkIntegrity($name); + $this->assertFalse($report['ok'], "$n records"); + $this->assertSame([['main', 'field c']], $this->mismatches($report), "$n records"); + $this->assertSame($n, $report['records'], 'records come from the data'); + + $this->db($config)->rebuildIndex($name); + $this->assertSame([], $this->db($config)->checkIntegrity($name)['indexMismatches'], "$n records: repaired"); + } + } + + /** @test a shard's field index is compared too, and named by its shard */ + public function aShardsFieldIndexIsComparedWithItsRecords(): void + { + $config = ['shardSize' => 500]; + $db = $this->db($config); + $db->insert('fs', $this->records(1200)); + $db->createFieldIndex('fs', 'c'); + $this->assertSame([], $db->checkIntegrity('fs')['indexMismatches']); + + $fx = $this->dbPath('fs', 'fx', 'c', 1); + [$header, $map] = $this->readFieldIndexFile($fx); + unset($map['sv600']); + $this->writeFieldIndexFile($fx, $header, $map); + \noneDB::clearStaticCache(); + + $report = $this->db($config)->checkIntegrity('fs'); + $this->assertFalse($report['ok']); + $this->assertSame([[1, 'field c']], $this->mismatches($report)); + } + + /** + * @test the lines after the bytes a file covers are replayed: a clean database reports nothing, with every type of + * value, a key that came into a list below the keys in it, a value that moved and a record gone + * @dataProvider sizes + */ + public function aCurrentFieldIndexReportsNothing(int $n): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('cl', $this->records($n)); + $db->insert('cl', [['c' => 12], ['c' => 1.5], ['c' => true], ['c' => null], ['c' => '12'], ['c' => 12], ['n' => 'no field'], ['c' => [1]]]); + $db->createFieldIndex('cl', 'c'); + $this->assertSame(1, $db->update('cl', [['key' => [$n - 1]], ['set' => ['c' => 'many']]])['n'], 'a higher key joins the list'); + $this->assertSame(1, $db->update('cl', [['key' => [17]], ['set' => ['c' => 'elsewhere']]])['n']); + $this->assertSame(1, $db->update('cl', [['key' => [8]], ['set' => ['c' => 'many']]])['n'], 'a key below some of the list'); + $this->assertSame(1, $db->delete('cl', ['key' => [9]])['n']); + $db->insert('cl', ['c' => 'many']); + + $report = $db->checkIntegrity('cl'); + $this->assertSame([], $report['indexMismatches']); + $this->assertTrue($report['ok']); + + \noneDB::clearStaticCache(); + $report = $this->db($config)->checkIntegrity('cl'); + $this->assertSame([], $report['indexMismatches'], 'the same from the files alone'); + $this->assertTrue($report['ok']); + } + + public function damages(): array + { + return [ + 'a byte of the first page' => [function (string $body) { $body[20] = chr(ord($body[20]) ^ 1); return $body; }], + 'a byte of the last page' => [function (string $body) { $at = strlen($body) - 2; $body[$at] = chr(ord($body[$at]) ^ 1); return $body; }], + 'cut short' => [function (string $body) { return substr($body, 0, -100); }], + 'no body' => [function (string $body) { return ''; }], + ]; + } + + public function treeDamages(): array + { + return $this->damages() + ['more bytes' => [function (string $body) { return $body . 'x'; }]]; + } + + /** + * @test a file that fits its segment and can't be read holds none of the records; the answers stay, rebuildIndex() repairs it + * @dataProvider damages + */ + public function aFieldIndexOfDamagedPagesIsReported(callable $damage): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('dp', $this->records(2000)); + $db->createFieldIndex('dp', 'c'); + $fx = $this->dbPath('dp', 'fx', 'c'); + [$line, $body] = explode("\n", (string)file_get_contents($fx), 2); + file_put_contents($fx, $line . "\n" . $damage($body)); + \noneDB::clearStaticCache(); + + $report = $this->db($config)->checkIntegrity('dp'); + $this->assertFalse($report['ok']); + $this->assertSame([['main', 'field c']], $this->mismatches($report)); + $this->assertSame([1999], array_column($this->db($config)->find('dp', ['c' => 'v1999']), 'key')); + + $this->db($config)->rebuildIndex('dp'); + $this->assertSame([], $this->db($config)->checkIntegrity('dp')['indexMismatches'], 'repaired'); + } + + /** + * @test the same for the body of an R-tree, which `bc` covers whole: a byte more is damage there + * @dataProvider treeDamages + */ + public function anRTreeOfADamagedBodyIsReported(callable $damage): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $records = []; + for ($i = 0; $i < 300; $i++) { + $records[] = ['loc' => ['type' => 'Point', 'coordinates' => [$i % 50, $i % 40]]]; + } + $db->insert('dr', $records); + $db->createSpatialIndex('dr', 'loc'); + $rx = $this->dbPath('dr', 'rx', 'loc'); + [$line, $body] = explode("\n", (string)file_get_contents($rx), 2); + file_put_contents($rx, $line . "\n" . $damage($body)); + \noneDB::clearStaticCache(); + + $report = $this->db($config)->checkIntegrity('dr'); + $this->assertFalse($report['ok']); + $this->assertSame([['main', 'spatial loc']], $this->mismatches($report)); + + $this->db($config)->rebuildIndex('dr'); + $this->assertSame([], $this->db($config)->checkIntegrity('dr')['indexMismatches'], 'repaired'); + } +} diff --git a/tests/Integration/KeyIndexFormatTest.php b/tests/Integration/KeyIndexFormatTest.php new file mode 100644 index 0000000..e6843bd --- /dev/null +++ b/tests/Integration/KeyIndexFormatTest.php @@ -0,0 +1,370 @@ + $i, 'pad' => str_repeat('x', $i % 17)]; + } + return $rows; + } + + private function crcBefore(string $file, int $s): string + { + $start = max(0, $s - 256); + return hash('crc32b', substr((string)file_get_contents($file), $start, $s - $start)); + } + + /** Write sessions until the segment's checkpoint covers the whole data file (the documented threshold) */ + private function checkpointNow(\noneDB $db, string $name, ?int $shard = null): array + { + for ($i = 0; $i < 400; $i++) { + clearstatcache(); + $kix = $this->dbPath($name, 'kix', $shard); + if (is_file($kix)) { + $file = $this->readKeyIndexFile($kix); + if ($file['header']['s'] === filesize($this->dbPath($name, 'data', $shard))) { + return $file; + } + } + $db->insert($name, ['n' => -100 - $i]); + $db->delete($name, ['n' => -100 - $i]); + } + $this->fail('no checkpoint covering the data file'); + } + + /** @test §6.4: header, dense layout in 4096-byte blocks of 341 slots, entries = the data file's index */ + public function aDenseCheckpointDescribesTheDataFile(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('kd', $this->rows(0, 1000)); // the first write session writes a checkpoint (none existed) + $data = $this->dbPath('kd', 'data'); + $kix = $this->dbPath('kd', 'kix'); + $file = $this->readKeyIndexFile($kix); + $h = $file['header']; + $this->assertSame(['dense', 0, 1000, 1000, 1000], [$h['layout'], $h['base'], $h['slots'], $h['live'], $h['n']]); + $this->assertSame(filesize($data), $h['s']); + $this->assertSame($this->crcBefore($data, $h['s']), $h['h']); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', $h['e']); + $this->assertSame(0, $h['d']); + $this->assertSame(0, $h['t']); + // 1000 slots: blocks of 341, 341 and 318 slots, each with its crc + $this->assertSame(512 + 2 * 4096 + 318 * 12 + 4, filesize($kix)); + $this->assertSame($this->keyIndexOfData($data), $file['entries']); + $this->assertArrayNotHasKey('z', $h, 'single layout'); + } + + /** @test §6.4: updates and deletes; garbage and tombstone counters; the checkpoint follows the data */ + public function aCheckpointAfterUpdatesAndDeletesMatchesTheData(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); + $db->insert('ku', $this->rows(0, 600)); + $db->update('ku', [['n' => 5], ['set' => ['u' => 1]]]); + $db->delete('ku', ['n' => 7]); + $db->delete('ku', ['n' => 599]); + $file = $this->checkpointNow($db, 'ku'); + $data = $this->dbPath('ku', 'data'); + $model = $this->keyIndexOfData($data); + $this->assertSame($model, $file['entries']); + $this->assertSame(count($model), $file['header']['live']); + $this->assertArrayNotHasKey(7, $file['entries']); + $this->assertArrayNotHasKey(599, $file['entries']); + // Every line that is not the last line of a live key is garbage or a tombstone + $lines = count(array_filter(explode("\n", (string)file_get_contents($data)), 'strlen')); + $tombstones = substr_count((string)file_get_contents($data), '{"__nonedb_deleted":'); + $this->assertSame($tombstones, $file['header']['t']); + $this->assertSame($lines - $tombstones - count($model), $file['header']['d']); + $this->assertGreaterThan(599, $file['header']['n'] - 1, 'keys are never reused'); + } + + /** @test §6.4: more than max(1024, 4 × live) slots: sparse entries, ascending, 255 per block */ + public function aDeletionHeavySegmentIsSparse(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); + $db->insert('ks', $this->rows(0, 3000)); + $db->query('ks')->where(['n' => ['$lt' => 2700]])->delete(); // 300 live of 3000 slots: 3000 > max(1024, 1200) + $file = $this->checkpointNow($db, 'ks'); + $this->assertSame('sparse', $file['header']['layout']); + $this->assertSame($file['header']['live'], $file['header']['slots']); + $this->assertSame($this->keyIndexOfData($this->dbPath('ks', 'data')), $file['entries']); + $this->assertSame(512 + 2 * 4096, filesize($this->dbPath('ks', 'kix')), 'ceil(300 / 255) = 2 full-size blocks'); + $this->assertSame(range(2700, 2999), array_column($db->find('ks', 0), 'n')); + + // Few deletions stay dense + $db->insert('kn', $this->rows(0, 3000)); + $db->query('kn')->where(['n' => ['$lt' => 2000]])->delete(); // 1000 live: 3000 <= max(1024, 4000) + $this->assertSame('dense', $this->checkpointNow($db, 'kn')['header']['layout']); + } + + /** @test §6.4: a shard's checkpoint: base = sid × z, z and sid in the header */ + public function aShardCheckpointHasItsBase(): void + { + $db = $this->db(['shardSize' => 500]); + $db->insert('kh', $this->rows(0, 1200)); // shards 0, 1, 2 + foreach ([0, 1, 2] as $shard) { + $file = $this->readKeyIndexFile($this->dbPath('kh', 'kix', $shard)); + $h = $file['header']; + $this->assertSame([500 * $shard, 500, $shard], [$h['base'], $h['z'], $h['sid']], "shard $shard"); + $this->assertSame($shard < 2 ? 500 : 200, $h['live'], "shard $shard"); + $this->assertSame($this->keyIndexOfData($this->dbPath('kh', 'data', $shard)), $file['entries'], "shard $shard"); + $this->assertSame(array_keys($file['entries']), range(500 * $shard, 500 * $shard + $h['live'] - 1)); + } + } + + /** @test §6.4: a block crc, the body length, kind or v that don't fit: the file is not trusted */ + public function aDamagedCheckpointIsNeverTrusted(): void + { + $cases = [ + 'block crc' => function (string $kix) { $raw = file_get_contents($kix); $raw[512 + 4096 + 100] = chr(ord($raw[512 + 4096 + 100]) ^ 0x01); file_put_contents($kix, $raw); }, + 'body length' => function (string $kix) { $raw = file_get_contents($kix); file_put_contents($kix, substr($raw, 0, -12)); }, + 'unknown v' => function (string $kix) { $raw = file_get_contents($kix); file_put_contents($kix, str_replace('"v":1', '"v":9', $raw)); }, + 'unknown kind' => function (string $kix) { $raw = file_get_contents($kix); file_put_contents($kix, str_replace('"kind":"kix"', '"kind":"kiy"', $raw)); }, + 'unknown layout' => function (string $kix) { $raw = file_get_contents($kix); file_put_contents($kix, str_replace('"layout":"dense"', '"layout":"other"', $raw)); }, + ]; + foreach ($cases as $case => $damage) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('kc', $this->rows(0, 1000)); + $model = $this->plain($db->find('kc', 0)); + \noneDB::clearStaticCache(); + $damage($this->dbPath('kc', 'kix')); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame($model, $this->plain($db->find('kc', 0)), "$case: every record"); + $this->assertSame(['n' => 400, 'pad' => str_repeat('x', 400 % 17)], $this->plain($db->find('kc', ['key' => 400]))[0] ?? null, "$case: key in block 1"); + $this->assertSame(1000, $db->count('kc'), $case); + $db->insert('kc', ['n' => 1000]); + $file = $this->readKeyIndexFile($this->dbPath('kc', 'kix')); // a valid checkpoint again + $this->assertSame($this->keyIndexOfData($this->dbPath('kc', 'data')), $file['entries'], $case); + $this->assertTrue($db->checkIntegrity('kc')['ok'], $case); + } + } + + /** The JSON header of a field or spatial index file (line 1, STORAGE_FORMAT.md §6.5) */ + private function indexHeader(string $path): array + { + $raw = (string)file_get_contents($path); + return json_decode(substr($raw, 0, (int)strpos($raw, "\n")), true); + } + + /** + * @test §6.4, §6.5: a key index rebuilt from the data (its checkpoint no longer fits) has a new epoch, so every + * field and spatial index of the segment is invalid; the same exclusive session builds them again with that + * epoch, also when the rebuilt index has the entries of the old one (the data changed in place) + */ + public function aRebuiltKeyIndexBuildsTheSegmentsIndexesAgain(): void + { + $db = $this->db(['shardingEnabled' => false]); + $rows = []; + for ($i = 0; $i < 1000; $i++) { + $rows[] = ['n' => $i, 'city' => $i % 2 ? 'odd' : 'eve', 'loc' => ['type' => 'Point', 'coordinates' => [$i / 10, 1]]]; + } + $db->insert('t', $rows); + $this->assertTrue($db->createFieldIndex('t', 'city')['success']); + $this->assertTrue($db->createSpatialIndex('t', 'loc')['success']); + $data = $this->dbPath('t', 'data'); + $kix = $this->readKeyIndexFile($this->dbPath('t', 'kix')); + $this->assertSame(filesize($data), $kix['header']['s'], 'the checkpoint covers the data file'); + $epoch = $kix['header']['e']; + foreach ([$this->dbPath('t', 'fx', 'city'), $this->dbPath('t', 'rx', 'loc')] as $file) { + $this->assertSame($epoch, $this->indexHeader($file)['e'], basename($file) . ': of the segment epoch'); + } + // Key 999's city changed in place (same length): in the 256 bytes the checkpoint's h covers, the entries the same + $raw = (string)file_get_contents($data); + $at = strrpos($raw, '"city":"odd"'); + $this->assertGreaterThan(strlen($raw) - 256, $at); + file_put_contents($data, substr_replace($raw, '"city":"ddo"', $at, 12)); + \noneDB::clearStaticCache(); + + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(0, $db->find('t', ['key' => 0])[0]['n'], 'the read that rebuilds the key index'); + $rebuilt = $this->readKeyIndexFile($this->dbPath('t', 'kix')); + $this->assertNotSame($epoch, $rebuilt['header']['e'], 'a new segment epoch'); + $this->assertSame($kix['entries'], $rebuilt['entries'], 'the same entries'); + foreach ([$this->dbPath('t', 'fx', 'city'), $this->dbPath('t', 'rx', 'loc')] as $file) { + $this->assertSame($rebuilt['header']['e'], $this->indexHeader($file)['e'], basename($file) . ': built again in that session'); + } + $this->assertSame([999], array_column($db->find('t', ['city' => 'ddo']), 'key')); + $this->assertSame(499, $db->count('t', ['city' => 'odd'])); + $this->assertTrue($db->checkIntegrity('t')['ok']); + } + + /** @test §6.4: a checkpoint written by these rules for data the engine didn't write is read as it is */ + public function aCheckpointWrittenFromTheSpecificationIsRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('kw', $this->rows(0, 50)); + \noneDB::clearStaticCache(); + $data = $this->dbPath('kw', 'data'); + $entries = $this->keyIndexOfData($data); + // A checkpoint (written from the documented format) that leaves out key 10: it must be trusted as it is + // (fits: s, h, e), so key 10 is not found through it; the data still has the line, but the checkpoint + // covers it. That is what "loading reads the checkpoint" means; the engine's own files never do this. + unset($entries[10]); + $old = $this->readKeyIndexFile($this->dbPath('kw', 'kix'))['header']; + $this->writeKeyIndexFile($this->dbPath('kw', 'kix'), ['n' => 50, 'd' => 0, 't' => 0, 's' => filesize($data), + 'h' => $this->crcBefore($data, filesize($data)), 'e' => $old['e'], 'created' => 1, 'updated' => 1], $entries); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame(49, $db->count('kw'), 'the checkpoint is the index of the bytes it covers'); + $this->assertSame([], $db->find('kw', ['key' => 10])); + $this->assertSame([11], array_column($db->find('kw', ['key' => 11]), 'key')); + } + + /** + * @test §6.4: in a database of at most 16 segments a key index is decoded into arrays at its first use + * when its body is at most 64 KiB, except in the process's first operation, else at the start of the + * database's next operation; + * in a larger database it is used as its body until its use paid for decoding - its operations plus the + * keys looked up in bulk reach an eighth of its live keys. Never without room in memory; answers the + * same either way. + */ + public function aKeyIndexIsDecodedOnceItsUsePaysForIt(): void + { + $bodyOf = function ($db, string $suffix) { + foreach (Testing::getOf($db, 'indexCache') as $path => $index) { + if (substr($path, -strlen($suffix)) === $suffix) { + return $index['kb']; + } + } + $this->fail("the key index $suffix is not loaded"); + }; + + // One segment, in a process past its first operation: a body of at most 64 KiB (1000 keys, 12 KB) is + // decoded by the operation that loads it, a larger one (6000 keys, 72 KB) at its second (with room) + $db = $this->db(['shardingEnabled' => false]); + $db->insert('one', array_map(function ($i) { return ['n' => $i]; }, range(0, 999))); + $db->insert('big', array_map(function ($i) { return ['n' => $i]; }, range(0, 5999))); + foreach (['one' => true, 'big' => false] as $name => $small) { + $suffix = $this->locatorOf($name) . '.ndb4/' . substr($this->dbPath($name, 'kix'), strlen($this->dbPath($name, 'home'))); + $this->assertSame($small, filesize($this->dbPath($name, 'kix')) <= 65536 + 512, "precondition: $name"); + foreach ([true, false] as $room) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => $room ? 0 : memory_get_usage() + 4 * 1048576]); + $label = $name . ', ' . ($room ? 'with room' : 'no room'); + $this->assertSame([['n' => 5]], $this->plain($db->find($name, ['key' => 5]))); + $this->assertSame($room && $small, $bodyOf($db, $suffix) === '', "$label: decoded by the loading operation"); + $this->assertSame([['n' => 6]], $this->plain($db->find($name, ['key' => 6]))); + $this->assertSame($room, $bodyOf($db, $suffix) === '', "$label: decoded by the second operation"); + } + } + + // A larger body is decoded at the start of the database's next operation, even one that doesn't read + // it (a delete of a key that can't exist); never without room + $suffix = $this->locatorOf('big') . '.ndb4/' . substr($this->dbPath('big', 'kix'), strlen($this->dbPath('big', 'home'))); + foreach ([true, false] as $room) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => $room ? 0 : memory_get_usage() + 4 * 1048576]); + $label = 'big, ' . ($room ? 'with room' : 'no room'); + $this->assertSame([['n' => 5]], $this->plain($db->find('big', ['key' => 5]))); + $this->assertNotSame('', $bodyOf($db, $suffix), "$label: the loading operation keeps the body"); + $this->assertSame(['n' => 0], $db->delete('big', ['key' => -1])); + $this->assertSame($room, $bodyOf($db, $suffix) === '', "$label: decoded by the next operation, which doesn't read it"); + } + + // A process's first operation keeps even a small body (a single cold operation); its second decodes it + $out = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]) . '; + $decoded = function () use ($db) { + foreach (\noneDB\Tests\Support\Testing::getOf($db, "indexCache") as $path => $index) { + if (substr($path, -9) === "/main.kix" && strpos($path, ' . var_export($this->locatorOf('one'), true) . ') !== false) { + return $index["kb"] === ""; + } + } + return null; + }; + echo json_encode($db->find("one", ["key" => 5])[0]["n"]), " ", json_encode($decoded()), "\n"; + echo json_encode($db->find("one", ["key" => 6])[0]["n"]), " ", json_encode($decoded()), "\n";'); + $this->assertSame("5 false\n6 true", $out, 'first operation: the body; second: decoded'); + + // 20 shards of 1000 keys: shard 3 (keys 3000-3999) is used as a body until its use paid + $config = ['shardSize' => 1000]; + $db = $this->db($config + ['cacheMemoryLimit' => 0]); + for ($start = 0; $start < 20000; $start += 2000) { // in batches: the suite's memory + $db->insert('kd', array_map(function ($i) { return ['n' => $i]; }, range($start, $start + 1999))); + } + foreach ([true, false] as $room) { + // The setting is process-wide: each instance sets it + $config = ['shardSize' => 1000, 'cacheMemoryLimit' => $room ? 0 : memory_get_usage() + 4 * 1048576]; + $label = $room ? 'with room' : 'no room'; + + // Operations of one key: 1000 live keys, due at the 125th (125 × 8 >= 1000) + \noneDB::clearStaticCache(); + $db = $this->db($config); + for ($op = 1; $op <= 126; $op++) { + $key = 3000 + $op * 7 % 1000; + $this->assertSame([['n' => $key]], $this->plain($db->find('kd', ['key' => $key]))); + $this->assertSame($room && $op >= 125, $bodyOf($db, '/s3.kix') === '', "$label: decoded after operation $op"); + } + + // Keys looked up in bulk: 100 keys in the loading operation are not enough; 124 more in a 2nd + // operation make (2 + 224) × 8 >= 1000 + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(3000, 3099), array_column($db->find('kd', ['key' => range(3000, 3099)]), 'n')); + $this->assertNotSame('', $bodyOf($db, '/s3.kix'), "$label: 100 keys looked up (and the loading operation), the body"); + $this->assertSame(range(3100, 3223), array_column($db->find('kd', ['key' => range(3100, 3223)]), 'n')); + $this->assertSame($room, $bodyOf($db, '/s3.kix') === '', "$label: 224 keys looked up in 2 operations"); + } + } + + /** + * @test §6.4: the start of an operation that makes the key indexes again (rebuildIndex()) decodes none of the old + * ones the process loaded; another operation does + */ + public function anOperationThatMakesTheKeyIndexesAgainDecodesNoneOfTheOldOnes(): void + { + $config = ['shardingEnabled' => false, 'cacheMemoryLimit' => 0]; + // 6000 keys: a body above 64 KiB, decoded at the start of the database's next operation + $this->db($config)->insert('rk', array_map(function ($i) { return ['n' => $i]; }, range(0, 5999))); + foreach (['another operation' => 1, 'rebuildIndex()' => 0] as $label => $decodes) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([['n' => 5]], $this->plain($db->find('rk', ['key' => 5])), $label); + $before = Testing::get('indexStats')['keyDecodes']; + if ($decodes === 1) { + $this->assertSame(['n' => 0], $db->delete('rk', ['key' => -1]), $label); + } else { + $this->assertSame(6000, $db->rebuildIndex('rk')['totalRecords'], $label); + } + $this->assertSame($decodes, Testing::get('indexStats')['keyDecodes'] - $before, "$label: key indexes decoded"); + $this->assertSame([['n' => 5999]], $this->plain($db->find('rk', ['key' => 5999])), $label); + } + } + + /** + * @test §6.4: in memory a key index costs about 12 bytes per key (no per-key array). In a process of its + * own: a measurement of memory, not disturbed by what earlier tests left in the allocator + * @runInSeparateProcess + * @preserveGlobalState disabled + */ + public function aLoadedKeyIndexIsCompact(): void + { + $db = $this->db(['shardingEnabled' => false]); + $batch = []; + for ($i = 0; $i < 100000; $i++) { + $batch[] = ['n' => $i]; + } + $db->insert('km', $batch); + unset($batch); + \noneDB::clearStaticCache(); + // A single segment with room is decoded at once (§6.4); the body is what light use and a process + // without room keep: a limit a little above what the process uses + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => memory_get_usage() + 4 * 1048576]); + gc_collect_cycles(); + $before = memory_get_usage(); + $this->assertSame([['n' => 77777]], $this->plain($db->find('km', ['key' => 77777]))); + $grown = memory_get_usage() - $before; + $this->assertLessThan(100000 * 16 + 256 * 1024, $grown, 'about 12 bytes per key (1.2 MB), never arrays (4+ MB)'); + } +} diff --git a/tests/Integration/MetricsTest.php b/tests/Integration/MetricsTest.php new file mode 100644 index 0000000..334f4d5 --- /dev/null +++ b/tests/Integration/MetricsTest.php @@ -0,0 +1,369 @@ +getMetrics()['process']; + } + + /** $after - $before for the numeric members of a group */ + private function delta(array $before, array $after, string $group): array + { + $out = []; + foreach ($after[$group] as $name => $value) { + if (is_int($value) || is_float($value)) { + $out[$name] = $value - $before[$group][$name]; + } + } + return $out; + } + + private function snapshot(string $dir): array + { + $out = []; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $full = $path . '/' . $entry; + if (is_dir($full)) { + $out[$rel . $entry] = 'dir'; + $walk($full, $rel . $entry . '/'); + } else { + $out[$rel . $entry] = hash_file('sha256', $full); + } + } + }; + if (is_dir($dir)) { + $walk(rtrim($dir, '/'), ''); + } + ksort($out); + return $out; + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + /** @test the shape of the result (§1.1, §1.2) */ + public function theResultHasTheDocumentedMembers(): void + { + $db = $this->db(['shardSize' => 4]); + $db->insert('u', [['v' => 1]]); + $metrics = $db->getMetrics(); + $this->assertSame(['process'], array_keys($metrics)); + $process = $metrics['process']; + $this->assertSame(['indexCache', 'indexReads', 'checkpoints', 'compactions', 'locks', 'recovery', 'durability', 'integrity', 'memory'], array_keys($process)); + $this->assertSame(['hits', 'misses', 'evictions', 'limit'], array_keys($process['indexCache'])); + $this->assertSame(['pages', 'bytes'], array_keys($process['indexReads'])); + $this->assertSame(['written', 'seconds'], array_keys($process['checkpoints'])); + $this->assertSame(['count', 'seconds'], array_keys($process['compactions'])); + $this->assertSame(['acquired', 'waited', 'waitSeconds', 'timeouts'], array_keys($process['locks'])); + $this->assertSame(['events', 'actions'], array_keys($process['recovery'])); + $this->assertSame(['mode', 'lastOperation', 'lastSyncError', 'syncs'], array_keys($process['durability'])); + $this->assertSame(['usage', 'peak'], array_keys($process['memory'])); + foreach (['hits', 'misses', 'evictions', 'limit'] as $name) { + $this->assertIsInt($process['indexCache'][$name]); + } + $this->assertIsFloat($process['checkpoints']['seconds']); + $this->assertIsFloat($process['locks']['waitSeconds']); + $this->assertGreaterThan(0, $process['memory']['usage']); + $this->assertGreaterThanOrEqual($process['memory']['usage'], $process['memory']['peak']); + + $database = $db->getMetrics('u')['database']; + $this->assertSame(['name', 'exists', 'databaseId', 'generation', 'layout', 'shardSize', 'segments', 'records', 'nextKey', + 'bufferedRecords', 'fieldIndexes', 'spatialIndexes', 'bytes', 'state', 'integrity'], array_keys($database)); + $this->assertSame(['data', 'indexes', 'buffer', 'metadata', 'recovery', 'other', 'total'], array_keys($database['bytes'])); + $this->assertSame(['buffered', 'recoveryPending'], array_keys($database['state'])); + } + + /** @test §1.1 indexCache and indexReads */ + public function indexesReadFromDiskAreMissesAndThoseInMemoryHits(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('u', array_map(function ($n) { return ['n' => $n]; }, range(0, 499))); + $this->assertTrue($db->rebuildIndex('u')['success']); // a checkpoint on disk + $kix = filesize($this->dbPath('u', 'kix')); + \noneDB::clearStaticCache(); // nothing cached + $db = $this->db(['shardingEnabled' => false, 'cacheMemoryLimit' => 0]); + $before = $this->process($db); + $this->assertSame([3], array_column($db->find('u', ['key' => 3]), 'n')); + $after = $this->process($db); + $this->assertSame(1, $this->delta($before, $after, 'indexCache')['misses'], 'the key index read from disk'); + $this->assertSame(['pages' => intdiv($kix + 4095, 4096), 'bytes' => $kix], $this->delta($before, $after, 'indexReads'), 'the checkpoint file'); + + $before = $after; + $db->find('u', ['key' => 4]); + $after = $this->process($db); + $delta = $this->delta($before, $after, 'indexCache'); + $this->assertSame(0, $delta['misses'], 'in memory now'); + $this->assertGreaterThanOrEqual(1, $delta['hits']); + $this->assertSame(['pages' => 0, 'bytes' => 0], $this->delta($before, $after, 'indexReads')); + $this->assertSame(0, $delta['evictions'], 'no limit'); + $this->assertSame(0, $after['indexCache']['limit']); + + // A limit below what the process uses: what an operation loaded is dropped after it, and read again + Testing::set('cacheMemoryLimit', 1); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->process($db); + $db->find('u', ['key' => 5]); + $db->find('u', ['key' => 6]); + $after = $this->process($db); + $delta = $this->delta($before, $after, 'indexCache'); + $this->assertGreaterThanOrEqual(2, $delta['evictions']); + $this->assertGreaterThanOrEqual(1, $delta['misses'], 'evicted after the first find, read again by the second'); + $this->assertSame(1, $after['indexCache']['limit']); + } + + /** @test §1.1 checkpoints and compactions */ + public function checkpointsAndCompactionsAreCountedAndTimed(): void + { + $db = $this->db(['shardSize' => 4, 'autoCompactThreshold' => 1000]); + $db->insert('u', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2)]; }, range(0, 11))); // shards 0..2 + $db->delete('u', ['c' => 'c0']); + $before = $this->process($db); + $this->assertTrue($db->compact('u')['success']); + $after = $this->process($db); + $compactions = $this->delta($before, $after, 'compactions'); + $this->assertSame(3, $compactions['count'], 'one per segment'); + $this->assertGreaterThan(0.0, $compactions['seconds']); + $this->assertGreaterThanOrEqual(3, $this->delta($before, $after, 'checkpoints')['written'], 'a fresh checkpoint per compacted segment'); + + $before = $after; + $db->createFieldIndex('u', 'c'); + $after = $this->process($db); + $checkpoints = $this->delta($before, $after, 'checkpoints'); + $this->assertGreaterThanOrEqual(4, $checkpoints['written'], 'three shard field indexes and the global one'); + $this->assertGreaterThan(0.0, $checkpoints['seconds']); + $this->assertSame(0, $this->delta($before, $after, 'compactions')['count']); + } + + /** @test §1.1 locks: waits and timeouts against another process */ + public function lockWaitsAndTimeoutsAreCounted(): void + { + $db = $this->db(); + $db->insert('u', [['v' => 1]]); + $before = $this->process($db); + $db->find('u', 0); + $after = $this->process($db); + $this->assertGreaterThanOrEqual(1, $this->delta($before, $after, 'locks')['acquired']); + $this->assertSame(0, $this->delta($before, $after, 'locks')['waited']); + + foreach (['a wait' => [0.6, 5, null], 'a timeout' => [1.5, 0.2, \noneDB::LOCK_FAILED]] as $case => [$hold, $timeout, $code]) { + $marker = sys_get_temp_dir() . '/nonedb_held_' . bin2hex(random_bytes(6)); + $proc = proc_open([PHP_BINARY, $this->childScript(' + $db = ' . $this->childDb() . '; + noneDB\Tests\Support\Testing::call($db, "withLock", "u", true, function () { + touch(' . var_export($marker, true) . '); + usleep(' . (int)($hold * 1000000) . '); + }); + ')], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $deadline = microtime(true) + 20; + while (!is_file($marker) && microtime(true) < $deadline) { + usleep(5000); + } + $this->assertFileExists($marker, $case); + \noneDB::clearStaticCache(); + $writer = $this->db(['lockTimeout' => $timeout]); + $before = $this->process($writer); + $this->assertSame($code, $this->code(function () use ($writer) { $writer->insert('u', ['v' => 2]); }), $case); + $after = $this->process($writer); + stream_get_contents($pipes[1]); + stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($proc); + @unlink($marker); + $locks = $this->delta($before, $after, 'locks'); + $this->assertSame(1, $locks['waited'], $case); + $this->assertGreaterThanOrEqual($code === null ? 0.2 : $timeout, $locks['waitSeconds'], $case); + $this->assertSame($code === null ? 0 : 1, $locks['timeouts'], $case); + } + } + + /** @test §1.1 recovery events by action */ + public function recoveryEventsAreCountedByAction(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('u', [['v' => 1]]); + file_put_contents($this->dbPath('u', 'data'), '{"v":"torn","ke', FILE_APPEND); // a crashed append + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $before = $this->process($db); + $db->insert('u', [['v' => 2]]); // cuts the torn tail into quarantine first + $after = $this->process($db); + $this->assertSame(1, $after['recovery']['events'] - $before['recovery']['events']); + $this->assertSame(1, ($after['recovery']['actions']['cut-tail'] ?? 0) - ($before['recovery']['actions']['cut-tail'] ?? 0)); + $logged = array_column(array_map(function ($l) { return json_decode($l, true); }, $this->dataLines($this->dbPath('u', 'recovery'))), 'action'); + $this->assertSame(['cut-tail'], $logged, 'the event counted is the one logged'); + } + + /** @test §1.1 durability: getDurability()'s report */ + public function durabilityIsWhatGetDurabilityReports(): void + { + $db = $this->db(); + $db->insert('u', [['v' => 1]]); + $durability = $db->getDurability(); + $this->assertSame(['mode' => $durability['mode'], 'lastOperation' => $durability['lastOperation'], + 'lastSyncError' => $durability['lastSyncError'], 'syncs' => $durability['syncs']], $this->process($db)['durability']); + } + + /** @test §1.1, §1.2: the last checkIntegrity() of the process and of the database */ + public function theLastIntegrityCheckIsKept(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('u', [['v' => 1], ['v' => 2]]); + $db->insert('w', [['v' => 1]]); + $this->assertNull($db->getMetrics('w')['database']['integrity'], 'never checked'); + $t0 = time(); + $this->assertTrue($db->checkIntegrity('u')['ok']); + $summary = $this->process($db)['integrity']; + $this->assertSame(['database', 'ok', 'time', 'records', 'issues'], array_keys($summary)); + $this->assertSame(['u', true, 2, 0], [$summary['database'], $summary['ok'], $summary['records'], $summary['issues']]); + $this->assertGreaterThanOrEqual($t0, $summary['time']); + $this->assertSame($summary, $db->getMetrics('u')['database']['integrity']); + $this->assertNull($db->getMetrics('w')['database']['integrity']); + + file_put_contents($this->dbPath('u', 'data'), "{not json}\n", FILE_APPEND); // a damaged line + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $report = $db->checkIntegrity('u'); + $this->assertFalse($report['ok']); + $summary = $this->process($db)['integrity']; + $this->assertSame(['u', false, 2], [$summary['database'], $summary['ok'], $summary['records']]); + $this->assertSame(count($report['badLines']) + count($report['tornTail']) + count($report['indexMismatches']), $summary['issues']); + $this->assertGreaterThanOrEqual(1, $summary['issues']); + + // A restore verifies its generation with the same check: that is no checkIntegrity() of the application + $dir = sys_get_temp_dir() . '/nonedb_metrics_backup_' . bin2hex(random_bytes(6)); + try { + $db->backup('w', $dir); + $db->restore($dir, 'w'); + $this->assertNull($db->getMetrics('w')['database']['integrity'], 'w was never checked'); + $this->assertSame($summary, $this->process($db)['integrity'], 'the last check is still the one of u'); + } finally { + $this->removeDirectoryTree($dir); + } + } + + /** @test §1.2: from the metadata, without writing anything */ + public function aDatabaseIsDescribedFromItsMetadata(): void + { + $config = ['shardSize' => 4, 'bufferEnabled' => true]; + $db = $this->db($config); + $db->insert('u', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n]]]; }, range(0, 9))); + $db->flush('u'); + $db->createFieldIndex('u', 'c'); + $db->createSpatialIndex('u', 'loc'); + $db->insert('u', [['n' => 10], ['n' => 11]]); // buffered + $before = $this->snapshot($this->testDbDir); + $database = $db->getMetrics('u')['database']; + $this->assertSame($before, $this->snapshot($this->testDbDir), 'nothing written: the buffer is not flushed'); + + $format = $this->openSealedRecord(file_get_contents($this->dbPath('u', 'format')), 'database'); + $current = json_decode(file_get_contents($this->dbPath('u', 'current')), true); + $meta = json_decode(file_get_contents($this->dbPath('u', 'shards')), true); + $this->assertSame(['u', true, $format['databaseId'], $current['generation'], 'sharded', 4], + [$database['name'], $database['exists'], $database['databaseId'], $database['generation'], $database['layout'], $database['shardSize']]); + $this->assertSame([count($meta['shards']), $meta['totalRecords'], $meta['nextKey'], 2], + [$database['segments'], $database['records'], $database['nextKey'], $database['bufferedRecords']]); + $this->assertSame([['c'], ['loc']], [$database['fieldIndexes'], $database['spatialIndexes']]); + $this->assertSame(['buffered' => true, 'recoveryPending' => false], $database['state']); + + // The sizes, by the classes of §1.2, from an independent walk of the directory + $home = $this->dbPath('u', 'home'); + $gen = 'generations/' . $current['generation'] . '/'; + $bytes = ['data' => 0, 'indexes' => 0, 'buffer' => 0, 'metadata' => 0, 'recovery' => 0, 'other' => 0]; + foreach (array_keys(array_filter($this->snapshot($home), function ($v) { return $v !== 'dir'; })) as $file) { + if (strpos($file, $gen . 'segments/') === 0 && substr($file, -6) === '.jsonl') { + $class = 'data'; + } elseif (preg_match('#^' . preg_quote($gen, '#') . '(segments/.*\.kix|indexes/.*\.(fx|gfx)|spatial/.*\.(rx|grx))$#', $file)) { + $class = 'indexes'; + } elseif (strpos($file, $gen . 'buffer/') === 0) { + $class = 'buffer'; + } elseif (in_array($file, ['FORMAT', 'CURRENT', 'LOCK', $gen . 'metadata', $gen . 'shards.json', $gen . 'indexes.json'], true)) { + $class = 'metadata'; + } elseif (strpos($file, 'recovery/') === 0) { + $class = 'recovery'; + } else { + $class = 'other'; + } + $bytes[$class] += filesize($home . $file); + } + $bytes['total'] = array_sum($bytes); + $this->assertSame($bytes, $database['bytes']); + $this->assertGreaterThan(0, $bytes['data']); + $this->assertGreaterThan(0, $bytes['indexes']); + $this->assertGreaterThan(0, $bytes['buffer']); + + // A leftover counts as other; a crashed writer's journal as pending recovery + mkdir($home . 'generations/99'); + file_put_contents($home . 'generations/99/left.jsonl', 'xyz'); + $state = $this->lockState('u'); + $this->writeLockState('u', $state['gen'] . ':' . $state['epoch'] . ' b j=s0'); + $database = $this->db($config)->getMetrics('u')['database']; + $this->assertSame($bytes['other'] + 3, $database['bytes']['other']); + $this->assertSame(['buffered' => true, 'recoveryPending' => true], $database['state']); + + // The single layout + $single = $this->db(['shardingEnabled' => false]); + $single->insert('s', [['v' => 1], ['v' => 2], ['v' => 3]]); + $single->delete('s', ['v' => 2]); + unlink($this->dbPath('s', 'kix')); // a reader would write it again: getMetrics() does not + \noneDB::clearStaticCache(); + $single = $this->db(['shardingEnabled' => false]); + $before = $this->snapshot($this->testDbDir); + $database = $single->getMetrics('s')['database']; + $this->assertSame($before, $this->snapshot($this->testDbDir), 'nothing written: no checkpoint either'); + $this->assertSame(['single', 1, 2, 3, 0, [], []], [$database['layout'], $database['segments'], $database['records'], + $database['nextKey'], $database['bufferedRecords'], $database['fieldIndexes'], $database['spatialIndexes']]); + } + + /** @test §1.2: a missing database is reported, and nothing is created */ + public function aMissingDatabaseIsReportedAndNotCreated(): void + { + $db = $this->db(); + $db->insert('u', [['v' => 1]]); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(['name' => 'missing', 'exists' => false], $db->getMetrics('missing')['database']); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + + $location = sys_get_temp_dir() . '/nonedb_nostore_' . bin2hex(random_bytes(6)); + $other = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $location]); + $this->assertSame(['name' => 'x', 'exists' => false], $other->getMetrics('x')['database']); + $this->assertFileDoesNotExist($location); + } + + /** @test §1.2: what checkIntegrity() can't read, getMetrics() can't either */ + public function aDamagedDatabaseFailsAsCheckIntegrityDoes(): void + { + $db = $this->db(); + $db->insert('u', [['v' => 1]]); + file_put_contents($this->dbPath('u', 'current'), 'x'); + \noneDB::clearStaticCache(); + $db = $this->db(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () use ($db) { $db->checkIntegrity('u'); })); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () use ($db) { $db->getMetrics('u'); })); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } +} diff --git a/tests/Integration/NewChainingMethodsTest.php b/tests/Integration/NewChainingMethodsTest.php new file mode 100644 index 0000000..3c4c822 --- /dev/null +++ b/tests/Integration/NewChainingMethodsTest.php @@ -0,0 +1,2162 @@ + record */ + private const PEOPLE = [ + 0 => ['name' => 'Alice', 'age' => 25, 'city' => 'Istanbul', 'email' => 'alice@gmail.com', 'score' => 85, 'department' => 'IT'], + 1 => ['name' => 'Bob', 'age' => 30, 'city' => 'Ankara', 'email' => 'bob@yahoo.com', 'score' => 90, 'department' => 'HR'], + 2 => ['name' => 'Charlie', 'age' => 35, 'city' => 'Istanbul', 'email' => 'charlie@gmail.com', 'score' => 75, 'department' => 'IT'], + 3 => ['name' => 'David', 'age' => 28, 'city' => 'Izmir', 'email' => 'david@hotmail.com', 'score' => 95, 'department' => 'Sales'], + 4 => ['name' => 'Eve', 'age' => 22, 'city' => 'Istanbul', 'email' => 'eve@gmail.com', 'score' => 80, 'department' => 'IT'], + 5 => ['name' => 'Frank', 'age' => 40, 'city' => 'Ankara', 'email' => 'frank@yahoo.com', 'score' => 70, 'department' => 'HR'], + ]; + + private const ALL_NAMES = ['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank']; + + protected function setUp(): void + { + parent::setUp(); + + // Insert standard test data + $this->noneDB->insert($this->testDbName, array_values(self::PEOPLE)); + } + + /** + * Names of the results, in result order + */ + private function names(array $results): array + { + return array_map(function ($r) { return $r['name']; }, $results); + } + + /** + * Keys of the results, in result order + */ + private function keys(array $results): array + { + return array_map(function ($r) { return $r['key']; }, $results); + } + + /** + * A stored record as find()/get() return it (fields + key), field order normalized + */ + private function person(int $key, ?array $only = null): array + { + $record = self::PEOPLE[$key]; + if ($only !== null) { + $record = array_intersect_key($record, array_flip($only)); + } + $record['key'] = $key; + return $record; + } + + /** + * Recursively sort the keys of associative arrays (field order is not part of the contract; values and + * types are) + */ + private static function canon($value) + { + if (!is_array($value)) { + return $value; + } + $isList = $value === [] || array_keys($value) === range(0, count($value) - 1); + if (!$isList) { + ksort($value, SORT_STRING); + } + foreach ($value as $k => $item) { + $value[$k] = self::canon($item); + } + return $value; + } + + private function assertSameRecords(array $expected, array $actual, string $message = ''): void + { + $this->assertSame(self::canon($expected), self::canon($actual), $message); + } + + /** + * Group rows by their _group value (order-independent): json of _group => [_count, item names] + */ + private function groupsByValue(array $groups): array + { + $out = []; + foreach ($groups as $group) { + $this->assertSame(['_group', '_items', '_count'], array_keys($group), 'a group row has exactly _group, _items, _count'); + $id = json_encode($group['_group']); + $this->assertArrayNotHasKey($id, $out, 'every group value appears once'); + $out[$id] = [$group['_count'], $this->names($group['_items'])]; + } + ksort($out); + return $out; + } + + // ========================================== + // orWhere() TESTS + // ========================================== + + /** + * @test + */ + public function orWhereBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul']) + ->orWhere(['city' => 'Ankara']) + ->get(); + + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + $this->assertSameRecords([$this->person(0), $this->person(1), $this->person(2), $this->person(4), $this->person(5)], $results); + } + + /** + * @test + */ + public function orWhereWithMultipleConditions(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Izmir']) + ->orWhere(['name' => 'Alice']) + ->orWhere(['name' => 'Bob']) + ->get(); + + $this->assertSame(['Alice', 'Bob', 'David'], $this->names($results)); + } + + /** + * @test + */ + public function orWhereOnlyMatchesOrCondition(): void + { + // WHERE matches nothing, OR matches something + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'NonExistent']) + ->orWhere(['city' => 'Istanbul']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + } + + /** + * @test + */ + public function orWhereWithNoMatches(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'NonExistent1']) + ->orWhere(['city' => 'NonExistent2']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function orWhereWithMultipleFieldsInCondition(): void + { + // A record without a match on both fields of a group doesn't match that group: Bob is Ankara+HR + $this->noneDB->insert($this->testDbName, ['name' => 'Grace', 'city' => 'Ankara', 'department' => 'IT']); + + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul', 'department' => 'IT']) + ->orWhere(['city' => 'Ankara', 'department' => 'HR']) + ->get(); + + // 3 IT Istanbul + 2 HR Ankara; Grace (Ankara, IT) matches neither group as a whole + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function orWhereWithNullValue(): void + { + // Insert record with null, and one without the field (a plain null matches only an existing null) + $this->noneDB->insert($this->testDbName, ['name' => 'NullCity', 'city' => null]); + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul']) + ->orWhere(['city' => null]) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve', 'NullCity'], $this->names($results)); // 3 Istanbul + 1 null + $this->assertSame([0, 2, 4, 6], $this->keys($results)); + } + + /** + * @test + */ + public function orWhereWithoutWhere(): void + { + $results = $this->noneDB->query($this->testDbName) + ->orWhere(['city' => 'Istanbul']) + ->orWhere(['city' => 'Ankara']) + ->get(); + + // Without where(), only the orWhere() groups decide: Istanbul (3) + Ankara (2) + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + } + + /** + * QUERY.md orWhere(): "An empty group (orWhere([])) has no condition and matches every record" + * + * @test + */ + public function orWhereEmptyGroupMatchesEveryRecord(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'NonExistent']) + ->orWhere([]) + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); + $this->assertSame(6, $this->noneDB->query($this->testDbName)->where(['city' => 'NonExistent'])->orWhere([])->count()); + } + + /** + * QUERY.md: a mix of operators and plain keys in one value throws as soon as orWhere() is called + * + * @test + */ + public function orWhereMixingOperatorsAndPlainKeysThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('cannot mix operators'); + $query->orWhere(['age' => ['$gt' => 20, 'plain' => 1]]); + } + + /** + * QUERY.md: an unknown operator throws when the query runs, also on an empty database + * + * @test + */ + public function orWhereUnknownOperatorThrowsWhenTheQueryRunsEvenOnEmptyDb(): void + { + $this->noneDB->createDB('orwhere-empty'); + $query = $this->noneDB->query('orwhere-empty')->orWhere(['age' => ['$gtee' => 1]]); // no exception yet + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage("unknown operator '\$gtee'"); + $query->get(); + } + + // ========================================== + // whereIn() TESTS + // ========================================== + + /** + * @test + */ + public function whereInBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->get(); + + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function whereInWithSingleValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Izmir']) + ->get(); + + $this->assertSameRecords([$this->person(3)], $results); + } + + /** + * @test + */ + public function whereInWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', []) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereInWithNonExistentValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Paris', 'London']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereInWithMixedExistentNonExistent(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Paris', 'London']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + } + + /** + * @test + */ + public function whereInWithNumericValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('age', [25, 30, 35]) + ->get(); + + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function whereInCombinedWithWhere(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); // All IT are in Istanbul + } + + /** + * @test + */ + public function whereInWithMissingField(): void + { + // Insert record without the field + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); // Only records WITH city field + + // A null in the list doesn't match a missing field either + $withNull = $this->noneDB->query($this->testDbName)->whereIn('city', [null])->get(); + $this->assertSame([], $withNull); + } + + /** + * @test + */ + public function whereInStrictTypeComparison(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'TypeTest', 'code' => '100']); + $this->noneDB->insert($this->testDbName, ['name' => 'FloatTest', 'code' => 100.0]); + + $results = $this->noneDB->query($this->testDbName) + ->whereIn('code', [100]) // Integer 100 + ->get(); + + // Should NOT match string '100' nor float 100.0 due to strict comparison + $this->assertSame([], $results); + + // The same value with the same type matches + $this->assertSame(['TypeTest'], $this->names($this->noneDB->query($this->testDbName)->whereIn('code', ['100'])->get())); + $this->assertSame(['FloatTest'], $this->names($this->noneDB->query($this->testDbName)->whereIn('code', [100.0])->get())); + } + + /** + * More than 8 values take another code path (a hash set); the comparison stays strict (===) + * + * @test + */ + public function whereInWithLongValueListIsStillStrict(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'TypeTest', 'code' => '100']); + $this->noneDB->insert($this->testDbName, ['name' => 'IntTest', 'code' => 100]); + $this->noneDB->insert($this->testDbName, ['name' => 'FloatTest', 'code' => 100.0]); + $this->noneDB->insert($this->testDbName, ['name' => 'TrueTest', 'code' => true]); + + $ints = [1, 2, 3, 4, 5, 6, 7, 8, 9, 100]; // 10 values + $this->assertSame(['IntTest'], $this->names($this->noneDB->query($this->testDbName)->whereIn('code', $ints)->get())); + + $strings = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '100']; + $this->assertSame(['TypeTest'], $this->names($this->noneDB->query($this->testDbName)->whereIn('code', $strings)->get())); + + // 1 (int) in a long list is not true + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['name' => 'TrueTest'])->whereIn('code', $ints)->get()); + + // whereNotIn with a long list: strict as well (string '100' is kept) + $kept = $this->noneDB->query($this->testDbName)->where(['code' => ['$exists' => true]])->whereNotIn('code', $ints)->get(); + $this->assertSame(['TypeTest', 'FloatTest', 'TrueTest'], $this->names($kept)); + } + + /** + * @test + */ + public function multipleWhereInStacksFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->whereIn('department', ['IT']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); // IT in Istanbul + } + + // ========================================== + // whereNotIn() TESTS + // ========================================== + + /** + * @test + */ + public function whereNotInBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('city', ['Istanbul']) + ->get(); + + $this->assertSame(['Bob', 'David', 'Frank'], $this->names($results)); // Ankara(2) + Izmir(1) + } + + /** + * @test + */ + public function whereNotInWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('city', []) + ->get(); + + // Empty array = nothing excluded = all returned + $this->assertSame(self::ALL_NAMES, $this->names($results)); + } + + /** + * @test + */ + public function whereNotInExcludesAll(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('city', ['Istanbul', 'Ankara', 'Izmir']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereNotInWithMissingField(): void + { + // Insert record without the field + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('city', ['Istanbul']) + ->get(); + + // Records without field should be included (not in the list) + $this->assertSame(['Bob', 'David', 'Frank', 'NoCity'], $this->names($results)); // 3 non-Istanbul + 1 without city + } + + /** + * @test + */ + public function whereNotInIsStrict(): void + { + // '25' (string) is not 25: nothing is excluded + $results = $this->noneDB->query($this->testDbName)->whereNotIn('age', ['25', '30'])->get(); + $this->assertSame(self::ALL_NAMES, $this->names($results)); + + $results = $this->noneDB->query($this->testDbName)->whereNotIn('age', [25, 30])->get(); + $this->assertSame(['Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function whereNotInCombinedWithWhereIn(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->whereNotIn('department', ['IT']) + ->get(); + + $this->assertSame(['Bob', 'Frank'], $this->names($results)); // HR in Ankara + } + + // ========================================== + // whereNot() TESTS + // ========================================== + + /** + * @test + */ + public function whereNotBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => 'Istanbul']) + ->get(); + + $this->assertSame(['Bob', 'David', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function whereNotWithMultipleFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => 'Istanbul', 'department' => 'HR']) + ->get(); + + // Excludes records where city IS Istanbul OR department IS HR + $this->assertSame(['David'], $this->names($results)); // Only David (Izmir, Sales) + } + + /** + * @test + */ + public function whereNotWithMissingField(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => 'Istanbul']) + ->get(); + + // Records without field should be included + $this->assertSame(['Bob', 'David', 'Frank', 'NoCity'], $this->names($results)); + } + + /** + * @test + */ + public function whereNotCombinedWithWhere(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->whereNot(['city' => 'Istanbul']) + ->get(); + + // IT but not in Istanbul - none should match + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereNotWithNullValue(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NullCity', 'city' => null]); + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => null]) + ->get(); + + // All except the null city; a missing field doesn't match the plain value null (QUERY.md whereNot()) + $this->assertSame(array_merge(self::ALL_NAMES, ['NoCity']), $this->names($results)); + } + + /** + * @test + */ + public function whereNotPlainValueIsStrict(): void + { + // '25' is not 25 (===): nothing excluded + $results = $this->noneDB->query($this->testDbName)->whereNot(['age' => '25'])->get(); + $this->assertSame(self::ALL_NAMES, $this->names($results)); + } + + /** + * QUERY.md whereNot(): an operator array matches when all of that field's operators match + * + * @test + */ + public function whereNotWithOperatorsExcludesWhenAllOperatorsMatch(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['age' => ['$gte' => 25, '$lt' => 30]]) + ->get(); + + // Excluded: Alice (25), David (28); 30 is not < 30 + $this->assertSame(['Bob', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + + // Operators of different fields: any field that matches excludes the record + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['age' => ['$gt' => 35], 'score' => ['$gte' => 90]]) + ->get(); + // Frank (40) by age; Bob (90) and David (95) by score + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + } + + /** + * QUERY.md whereNot(): $ne, $nin and $exists => false match a missing field, so those records are dropped + * + * @test + */ + public function whereNotNegativeOperatorsAlsoDropRecordsWithoutTheField(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoDept']); + + $ne = $this->noneDB->query($this->testDbName)->whereNot(['department' => ['$ne' => 'IT']])->get(); + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($ne)); + + $nin = $this->noneDB->query($this->testDbName)->whereNot(['department' => ['$nin' => ['IT', 'HR']]])->get(); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($nin)); + + $notExists = $this->noneDB->query($this->testDbName)->whereNot(['department' => ['$exists' => false]])->get(); + $this->assertSame(self::ALL_NAMES, $this->names($notExists)); + + // Most operators don't match a missing field: the record is kept + $in = $this->noneDB->query($this->testDbName)->whereNot(['department' => ['$in' => ['IT']]])->get(); + $this->assertSame(['Bob', 'David', 'Frank', 'NoDept'], $this->names($in)); + } + + /** + * QUERY.md: whereNot() checks its operators when it is called + * + * @test + */ + public function whereNotUnknownOperatorThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage("unknown operator '\$gtee'"); + $query->whereNot(['age' => ['$gtee' => 18]]); + } + + /** + * @test + */ + public function whereNotInOperatorWithoutArrayThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('$in needs an array'); + $query->whereNot(['age' => ['$in' => 25]]); + } + + /** + * @test + */ + public function whereNotInvalidRegexThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('invalid $regex pattern'); + $query->whereNot(['name' => ['$regex' => '(unclosed']]); + } + + /** + * @test + */ + public function whereNotMixingOperatorsAndPlainKeysThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('cannot mix operators'); + $query->whereNot(['age' => ['$gt' => 1, 'x' => 2]]); + } + + /** + * @test + */ + public function multipleWhereNotCallsStackDifferentFields(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => 'Istanbul']) + ->whereNot(['department' => 'HR']) + ->get(); + + // Excludes Istanbul (3) AND HR (Bob, Frank in Ankara) + // Result: David (Izmir, Sales) only + $this->assertSameRecords([$this->person(3)], $results); + } + + /** + * Renamed from whereNotSameFieldOverrides: every whereNot() call applies, none overrides another + * + * @test + */ + public function whereNotSameFieldCallsAllApply(): void + { + // Every whereNot() call applies (calls for the same field never overwrite each other) + $results = $this->noneDB->query($this->testDbName) + ->whereNot(['city' => 'Istanbul']) + ->whereNot(['city' => 'Ankara']) + ->get(); + + $this->assertSameRecords([$this->person(3)], $results); // only David (Izmir) + } + + // ========================================== + // notLike() TESTS + // ========================================== + + /** + * @test + */ + public function notLikeBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', 'gmail') + ->get(); + + $this->assertSame(['Bob', 'David', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function notLikeStartsWith(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notLike('name', '^A') + ->get(); + + $this->assertSame(['Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); + + // Case-insensitive: '^a' excludes Alice too + $this->assertSame(['Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($this->noneDB->query($this->testDbName)->notLike('name', '^a')->get())); + } + + /** + * @test + */ + public function notLikeEndsWith(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', 'gmail.com$') + ->get(); + + $this->assertSame(['Bob', 'David', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function notLikeWithMissingField(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoEmail']); + + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', 'gmail') + ->get(); + + // Records without field should be included + $this->assertSame(['Bob', 'David', 'Frank', 'NoEmail'], $this->names($results)); + } + + /** + * @test + */ + public function notLikeWithArrayValue(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'ArrayEmail', 'email' => ['gmail', 'a@gmail.com']]); + + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', 'gmail') + ->get(); + + // Array values never match a pattern, so notLike keeps them (even with matching elements) + $this->assertSame(['Bob', 'David', 'Frank', 'ArrayEmail'], $this->names($results)); + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($this->noneDB->query($this->testDbName)->like('email', 'gmail')->get())); + } + + /** + * QUERY.md like()/notLike(): strings and numbers are matched; booleans, arrays and missing fields never are + * + * @test + */ + public function likeAndNotLikeMatchNumbersButNeverBooleans(): void + { + $this->assertSame(['Alice', 'David', 'Eve'], $this->names($this->noneDB->query($this->testDbName)->like('age', '^2')->get())); + $this->assertSame(['Bob', 'Charlie', 'Frank'], $this->names($this->noneDB->query($this->testDbName)->notLike('age', '^2')->get())); + + $this->noneDB->insert($this->testDbName, [ + ['name' => 'T', 'flag' => true], + ['name' => 'F', 'flag' => false], + ['name' => 'S', 'flag' => '1'], + ]); + $this->assertSame(['S'], $this->names($this->noneDB->query($this->testDbName)->like('flag', '1')->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('flag', 'true')->get()); + // notLike keeps the booleans and the records without the field + $this->assertSame(array_merge(self::ALL_NAMES, ['T', 'F']), $this->names($this->noneDB->query($this->testDbName)->notLike('flag', '1')->get())); + } + + /** + * @test + */ + public function notLikeCombinedWithLike(): void + { + $results = $this->noneDB->query($this->testDbName) + ->like('email', 'gmail') + ->notLike('name', '^E') + ->get(); + + $this->assertSame(['Alice', 'Charlie'], $this->names($results)); // gmail but not E* + } + + /** + * @test + */ + public function notLikeWithEmptyPattern(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', '') + ->get(); + + // Empty pattern matches all, so notLike excludes all + $this->assertSame([], $results); + } + + /** + * @test + */ + public function notLikeWithRegexSpecialChars(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'Test', 'email' => 'test+special@test.com']); + + $results = $this->noneDB->query($this->testDbName) + ->notLike('email', '+special') + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); // All original records + + // '+' is literal: like() finds exactly that record; '.' is literal too + $this->assertSame(['Test'], $this->names($this->noneDB->query($this->testDbName)->like('email', '+special')->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('email', 'gmail.co.')->get()); + } + + // ========================================== + // notBetween() TESTS + // ========================================== + + /** + * @test + */ + public function notBetweenBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notBetween('age', 25, 35) + ->get(); + + // Ages outside 25-35: Eve(22), Frank(40) + $this->assertSame(['Eve', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function notBetweenBoundaryValues(): void + { + // Values AT boundaries should be excluded (they ARE between) + $results = $this->noneDB->query($this->testDbName) + ->notBetween('age', 25, 25) // Exact match + ->get(); + + $this->assertSame(['Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); // All except Alice (25) + } + + /** + * @test + */ + public function notBetweenWithMissingField(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoAge']); + + $results = $this->noneDB->query($this->testDbName) + ->notBetween('age', 20, 30) + ->get(); + + // Records without field should be included + $this->assertSame(['Charlie', 'Frank', 'NoAge'], $this->names($results)); + } + + /** + * QUERY.md between()/notBetween(): notBetween() returns every other record, including values that can't be + * compared (null, booleans, text against numbers); numeric strings compare numerically + * + * @test + */ + public function notBetweenIsTheComplementOfBetweenIncludingIncomparableValues(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'NullAge', 'age' => null], + ['name' => 'BoolAge', 'age' => true], + ['name' => 'TextAge', 'age' => 'old'], + ['name' => 'NumStr', 'age' => '27'], + ['name' => 'ArrAge', 'age' => [27]], + ]); + + $between = $this->names($this->noneDB->query($this->testDbName)->between('age', 25, 30)->get()); + $notBetween = $this->names($this->noneDB->query($this->testDbName)->notBetween('age', 25, 30)->get()); + + $this->assertSame(['Alice', 'Bob', 'David', 'NumStr'], $between); + $this->assertSame(['Charlie', 'Eve', 'Frank', 'NullAge', 'BoolAge', 'TextAge', 'ArrAge'], $notBetween); + $this->assertSame([], array_intersect($between, $notBetween)); + $this->assertCount(11, array_merge($between, $notBetween)); + } + + /** + * @test + */ + public function notBetweenWithNegativeNumbers(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'Negative', 'balance' => -50]); + $this->noneDB->insert($this->testDbName, ['name' => 'Positive', 'balance' => 50]); + + $results = $this->noneDB->query($this->testDbName) + ->notBetween('balance', -100, 0) + ->get(); + + // notBetween includes records without the field (6 original + 1 Positive) + // Negative (-50) is between -100 and 0, so excluded + $this->assertSame(array_merge(self::ALL_NAMES, ['Positive']), $this->names($results)); + $this->assertSame(['Negative'], $this->names($this->noneDB->query($this->testDbName)->between('balance', -100, 0)->get())); + } + + /** + * @test + */ + public function notBetweenWithFloats(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'Float1', 'price' => 10.5]); + $this->noneDB->insert($this->testDbName, ['name' => 'Float2', 'price' => 20.5]); + + $results = $this->noneDB->query($this->testDbName) + ->notBetween('price', 10.0, 15.0) + ->get(); + + // notBetween includes records without the field (6 original + 1 Float2) + // Float1 (10.5) is between 10.0 and 15.0, so excluded + $this->assertSame(array_merge(self::ALL_NAMES, ['Float2']), $this->names($results)); + $this->assertSame(20.5, $results[6]['price']); + } + + /** + * @test + */ + public function notBetweenCombinedWithBetween(): void + { + // Age between 20-40 but not between 28-32 + $results = $this->noneDB->query($this->testDbName) + ->between('age', 20, 40) + ->notBetween('age', 28, 32) + ->get(); + + // Alice(25), Charlie(35), Eve(22), Frank(40) in key order + $this->assertSame(['Alice', 'Charlie', 'Eve', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function multipleNotBetweenCallsStack(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notBetween('age', 20, 25) + ->notBetween('age', 35, 40) + ->get(); + + // Bob(30), David(28) - not in 20-25 AND not in 35-40 + $this->assertSame(['Bob', 'David'], $this->names($results)); + } + + // ========================================== + // select() TESTS + // ========================================== + + /** + * @test + */ + public function selectBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['name', 'age']) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key, ['name', 'age']); // key is always included + } + $this->assertSameRecords($expected, $results); + } + + /** + * @test + */ + public function selectSingleField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['name']) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key, ['name']); // Only name + key + } + $this->assertSameRecords($expected, $results); + } + + /** + * @test + */ + public function selectNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['nonexistent']) + ->get(); + + // Only the key is left (a missing field isn't added, not even as null) + $this->assertSame([['key' => 0], ['key' => 1], ['key' => 2], ['key' => 3], ['key' => 4], ['key' => 5]], $results); + } + + /** + * @test + */ + public function selectIncludesKeyExplicitly(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['name', 'key']) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key, ['name']); + } + $this->assertSameRecords($expected, $results); + } + + /** + * select([]) selects nothing specific: the records are returned whole (QUERY.md doesn't define an empty + * list; this pins the behaviour of the existing test: no projection) + * + * @test + */ + public function selectWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select([]) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key); + } + $this->assertSameRecords($expected, $results); + $this->assertSame($this->noneDB->query($this->testDbName)->get(), $results); + } + + /** + * @test + */ + public function selectCombinedWithFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul']) + ->select(['name', 'department']) + ->get(); + + $this->assertSameRecords([ + $this->person(0, ['name', 'department']), + $this->person(2, ['name', 'department']), + $this->person(4, ['name', 'department']), + ], $results); + } + + /** + * QUERY.md select(): filters and sort() see all fields, the projection only shapes the result + * + * @test + */ + public function selectDoesNotHideFieldsFromFiltersAndSort(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['name']) + ->where(['age' => ['$gte' => 28]]) + ->sort('score', 'desc') + ->get(); + + // Ages >= 28: Bob 90, Charlie 75, David 95, Frank 70 -> by score desc + $this->assertSameRecords([ + ['name' => 'David', 'key' => 3], + ['name' => 'Bob', 'key' => 1], + ['name' => 'Charlie', 'key' => 2], + ['name' => 'Frank', 'key' => 5], + ], $results); + } + + // ========================================== + // except() TESTS + // ========================================== + + /** + * @test + */ + public function exceptBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['email', 'score']) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key, ['name', 'age', 'city', 'department']); + } + $this->assertSameRecords($expected, $results); + } + + /** + * @test + */ + public function exceptSingleField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['email']) + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key, ['name', 'age', 'city', 'score', 'department']); + } + $this->assertSameRecords($expected, $results); + } + + /** + * @test + */ + public function exceptNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['nonexistent']) + ->get(); + + // Should return all fields (nonexistent field doesn't affect) + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key); + } + $this->assertSameRecords($expected, $results); + } + + /** + * QUERY.md "select() / except()": "`key` is always kept." (the bullet applies to both methods) + * + * @test + */ + public function exceptKey(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['key']) + ->get(); + + // except() removes key when it is listed (QUERY.md select() / except()); + // every other field stays + $this->assertSameRecords(array_values(self::PEOPLE), $results); + } + + /** + * @test + */ + public function exceptWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except([]) + ->get(); + + // Empty except = all fields returned + $this->assertSame($this->noneDB->query($this->testDbName)->get(), $results); + $this->assertSame(self::ALL_NAMES, $this->names($results)); + } + + /** + * @test + */ + public function exceptAllFields(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['name', 'age', 'city', 'email', 'score', 'department', 'key']) + ->get(); + + // Every field listed, key too: empty records (QUERY.md select() / except()) + $this->assertSame([[], [], [], [], [], []], $results); + } + + // ========================================== + // groupBy() TESTS + // ========================================== + + /** + * @test + */ + public function groupByBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->get(); + + $this->assertSame([ + '"Ankara"' => [2, ['Bob', 'Frank']], + '"Istanbul"' => [3, ['Alice', 'Charlie', 'Eve']], + '"Izmir"' => [1, ['David']], + ], $this->groupsByValue($results)); + } + + /** + * @test + */ + public function groupByReturnsCorrectCounts(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->get(); + + $grouped = []; + foreach ($results as $group) { + $grouped[$group['_group']] = $group['_count']; + } + ksort($grouped); + + $this->assertSame(['Ankara' => 2, 'Istanbul' => 3, 'Izmir' => 1], $grouped); + } + + /** + * @test + */ + public function groupByContainsItems(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('department') + ->get(); + + $this->assertSame([ + '"HR"' => [2, ['Bob', 'Frank']], + '"IT"' => [3, ['Alice', 'Charlie', 'Eve']], + '"Sales"' => [1, ['David']], + ], $this->groupsByValue($results)); + // The items are the whole records + foreach ($results as $group) { + if ($group['_group'] === 'Sales') { + $this->assertSameRecords([$this->person(3)], $group['_items']); + } + } + } + + /** + * @test + */ + public function groupByWithNullValues(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity1', 'city' => null]); + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity2', 'city' => null]); + + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->get(); + + // Should have group with null + $nullGroup = array_values(array_filter($results, function ($g) { return $g['_group'] === null; })); + $this->assertCount(1, $nullGroup); + $this->assertSame(2, $nullGroup[0]['_count']); + $this->assertSame(['NoCity1', 'NoCity2'], $this->names($nullGroup[0]['_items'])); + $this->assertCount(4, $results); + } + + /** + * @test + */ + public function groupByWithMissingField(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); + $this->noneDB->insert($this->testDbName, ['name' => 'NullCity', 'city' => null]); + + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->get(); + + // Missing field is grouped under null, together with an explicit null (QUERY.md groupBy()) + $this->assertSame([ + '"Ankara"' => [2, ['Bob', 'Frank']], + '"Istanbul"' => [3, ['Alice', 'Charlie', 'Eve']], + '"Izmir"' => [1, ['David']], + 'null' => [2, ['NoCity', 'NullCity']], + ], $this->groupsByValue($results)); + } + + /** + * QUERY.md groupBy(): group values keep their type (1, "1", 1.5 and 1.0 are separate groups); + * -0.0 and 0.0 are one group + * + * @test + */ + public function groupByKeepsValueTypes(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'i1', 'code' => 1], + ['name' => 's1', 'code' => '1'], + ['name' => 'f15', 'code' => 1.5], + ['name' => 'i1b', 'code' => 1], + ['name' => 'f1', 'code' => 1.0], + ['name' => 'zneg', 'zero' => -0.0], + ['name' => 'zpos', 'zero' => 0.0], + ]); + + $groups = $this->noneDB->query($this->testDbName)->where(['code' => ['$exists' => true]])->groupBy('code')->get(); + $this->assertCount(4, $groups); + $byValue = []; + foreach ($groups as $group) { + $type = gettype($group['_group']); + $byValue[$type . ':' . var_export($group['_group'], true)] = [$group['_count'], $this->names($group['_items'])]; + } + ksort($byValue); + $this->assertSame([ + 'double:1.0' => [1, ['f1']], + 'double:1.5' => [1, ['f15']], + 'integer:1' => [2, ['i1', 'i1b']], + "string:'1'" => [1, ['s1']], + ], $byValue); + + $zero = $this->noneDB->query($this->testDbName)->where(['zero' => ['$exists' => true]])->groupBy('zero')->get(); + $this->assertCount(1, $zero); + $this->assertSame(2, $zero[0]['_count']); + $this->assertIsFloat($zero[0]['_group']); + $this->assertSame(0.0, abs($zero[0]['_group'])); + $this->assertSame(['zneg', 'zpos'], $this->names($zero[0]['_items'])); + } + + /** + * @test + */ + public function groupByCombinedWithFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->groupBy('city') + ->get(); + + // All IT are in Istanbul + $this->assertSame(['"Istanbul"' => [3, ['Alice', 'Charlie', 'Eve']]], $this->groupsByValue($results)); + } + + /** + * @test + */ + public function groupByWithSort(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->sort('_count', 'desc') + ->get(); + + // Istanbul(3), Ankara(2), Izmir(1) + $this->assertSame(['Istanbul', 'Ankara', 'Izmir'], array_column($results, '_group')); + $this->assertSame([3, 2, 1], array_column($results, '_count')); + + $asc = $this->noneDB->query($this->testDbName)->groupBy('city')->sort('_group')->get(); + $this->assertSame(['Ankara', 'Istanbul', 'Izmir'], array_column($asc, '_group')); + } + + // ========================================== + // having() TESTS + // ========================================== + + /** + * Group values of a grouped result, sorted + */ + private function groupValues(array $groups): array + { + $values = array_column($groups, '_group'); + sort($values); + return $values; + } + + /** + * @test + */ + public function havingWithCount(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', '>', 1) + ->get(); + + // Istanbul(3) and Ankara(2) have count > 1 + $this->assertSame(['Ankara', 'Istanbul'], $this->groupValues($results)); + } + + /** + * @test + */ + public function havingWithCountEquals(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', '=', 1) + ->get(); + + // Only Izmir has exactly 1 + $this->assertCount(1, $results); + $this->assertSame('Izmir', $results[0]['_group']); + $this->assertSame(1, $results[0]['_count']); + $this->assertSameRecords([$this->person(3)], $results[0]['_items']); + } + + /** + * @test + */ + public function havingWithSumAggregate(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('sum:score', '>', 150) + ->get(); + + // Istanbul: 85+75+80=240, Ankara: 90+70=160, Izmir: 95 + $this->assertSame(['Ankara', 'Istanbul'], $this->groupValues($results)); + + // Boundary: Ankara's sum is exactly 160 + $this->assertSame(['Istanbul'], $this->groupValues($this->noneDB->query($this->testDbName)->groupBy('city')->having('sum:score', '>', 160)->get())); + $this->assertSame(['Ankara', 'Istanbul'], $this->groupValues($this->noneDB->query($this->testDbName)->groupBy('city')->having('sum:score', '>=', 160)->get())); + } + + /** + * @test + */ + public function havingWithAvgAggregate(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('avg:age', '>=', 30) + ->get(); + + // Istanbul: avg(25,35,22)=27.33, Ankara: avg(30,40)=35, Izmir: 28 + $this->assertSame(['Ankara'], $this->groupValues($results)); + } + + /** + * @test + */ + public function havingWithMinAggregate(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('min:age', '<', 25) + ->get(); + + // Istanbul min: 22 + $this->assertSame(['Istanbul'], $this->groupValues($results)); + } + + /** + * @test + */ + public function havingWithMaxAggregate(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('max:score', '>=', 90) + ->get(); + + // Ankara max: 90, Izmir max: 95, Istanbul max: 85 + $this->assertSame(['Ankara', 'Izmir'], $this->groupValues($results)); + } + + /** + * @test + */ + public function havingWithNotEquals(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', '!=', 3) + ->get(); + + // Ankara(2) and Izmir(1) + $this->assertSame(['Ankara', 'Izmir'], $this->groupValues($results)); + } + + /** + * @test + */ + public function havingWithLessThanOrEqual(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', '<=', 2) + ->get(); + + $this->assertSame(['Ankara', 'Izmir'], $this->groupValues($results)); + + $lt = $this->noneDB->query($this->testDbName)->groupBy('city')->having('count', '<', 2)->get(); + $this->assertSame(['Izmir'], $this->groupValues($lt)); + } + + /** + * @test + */ + public function multipleHavingConditions(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', '>=', 2) + ->having('avg:score', '>', 75) + ->get(); + + // Istanbul: count=3, avg=80. Ankara: count=2, avg=80. Izmir: count 1 + $this->assertSame(['Ankara', 'Istanbul'], $this->groupValues($results)); + + // Both conditions must hold: avg:score > 80 drops both + $none = $this->noneDB->query($this->testDbName)->groupBy('city')->having('count', '>=', 2)->having('avg:score', '>', 80)->get(); + $this->assertSame([], $none); + } + + /** + * @test + */ + public function havingWithInvalidAggregatePassesThrough(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('invalid:field', '>', 0) + ->get(); + + // Unknown aggregate: the condition is ignored, all groups pass + $this->assertSame(['Ankara', 'Istanbul', 'Izmir'], $this->groupValues($results)); + } + + /** + * QUERY.md having(): an unknown operator ignores the condition + * + * @test + */ + public function havingWithUnknownOperatorIsIgnored(): void + { + foreach (['<>', '==', 'gt', ''] as $operator) { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('city') + ->having('count', $operator, 100) + ->get(); + $this->assertSame(['Ankara', 'Istanbul', 'Izmir'], $this->groupValues($results), "operator '{$operator}'"); + } + } + + /** + * QUERY.md having(): without groupBy() the condition is ignored (plain records are returned) + * + * @test + */ + public function havingWithoutGroupByIsIgnored(): void + { + $results = $this->noneDB->query($this->testDbName)->having('count', '>', 100)->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key); + } + $this->assertSameRecords($expected, $results); + } + + // ========================================== + // search() TESTS + // ========================================== + + /** + * @test + */ + public function searchBasicUsage(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('alice') + ->get(); + + $this->assertSameRecords([$this->person(0)], $results); + } + + /** + * @test + */ + public function searchCaseInsensitive(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('ALICE') + ->get(); + + $this->assertSameRecords([$this->person(0)], $results); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->search('aLiCe@GMAIL')->get())); + } + + /** + * @test + */ + public function searchAcrossMultipleFields(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('istanbul') + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + + // 'hr' is found in the department of Bob and Frank, 'hotmail' in David's email + $this->assertSame(['Bob', 'Frank'], $this->names($this->noneDB->query($this->testDbName)->search('hr')->get())); + $this->assertSame(['David'], $this->names($this->noneDB->query($this->testDbName)->search('hotmail')->get())); + } + + /** + * @test + */ + public function searchInSpecificFields(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('gmail', ['email']) + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + + // Only the listed fields are searched + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('gmail', ['name', 'city'])->get()); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->search('alice', ['name'])->get())); + } + + /** + * @test + */ + public function searchWithNoMatches(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('nonexistent') + ->get(); + + $this->assertSame([], $results); + } + + /** + * QUERY.md search(): the whole term must appear in one field value (no word splitting) + * + * @test + */ + public function searchDoesNotSplitWords(): void + { + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('alice istanbul')->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('istanbul alice')->get()); + } + + /** + * @test + */ + public function searchPartialMatch(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('li') + ->get(); + + // Alice, Charlie contain 'li' + $this->assertSame(['Alice', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function searchIgnoresArrayFields(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'ArrayTest', 'tags' => ['alice', 'test']]); + + $results = $this->noneDB->query($this->testDbName) + ->search('alice') + ->get(); + + // Should only find Alice, not ArrayTest (array values not searched) + $this->assertSame(['Alice'], $this->names($results)); + } + + /** + * @test + */ + public function searchIgnoresNullFields(): void + { + $this->noneDB->insert($this->testDbName, ['name' => null, 'email' => 'test@test.com']); + + $results = $this->noneDB->query($this->testDbName) + ->search('test') + ->get(); + + // Should find the record via email (and only it: no original record contains "test") + $this->assertSame([['email' => 'test@test.com', 'key' => 6, 'name' => null]], self::canon($results)); + } + + /** + * @test + */ + public function searchWithNumericValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('25') + ->get(); + + // Matches Alice's age (25) and nothing else (no other field contains "25") + $this->assertSame(['Alice'], $this->names($results)); + + // '5' appears in Alice (25, 85), Charlie (35, 75) and David (95) + $this->assertSame(['Alice', 'Charlie', 'David'], $this->names($this->noneDB->query($this->testDbName)->search('5')->get())); + } + + /** + * QUERY.md search(): without a field list every field except key is searched + * + * @test + */ + public function searchWithoutFieldListIgnoresKey(): void + { + // Frank has key 5 but no "5" in any field; Bob has key 1 and no "1" anywhere + $this->assertSame(['Alice', 'Charlie', 'David'], $this->names($this->noneDB->query($this->testDbName)->search('5')->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('1')->get()); + // With an explicit list that names key, key is searched + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->search('1', ['key'])->get())); + } + + /** + * @test + */ + public function multipleSearchCallsStack(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('gmail') + ->search('alice') + ->get(); + + // Must match both 'gmail' AND 'alice' + $this->assertSame(['Alice'], $this->names($results)); + } + + /** + * @test + */ + public function searchCombinedWithFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul']) + ->search('gmail') + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + } + + // ========================================== + // join() TESTS + // ========================================== + + private const DEPARTMENTS = [ + ['id' => 'IT', 'name' => 'Information Technology', 'budget' => 100000], + ['id' => 'HR', 'name' => 'Human Resources', 'budget' => 50000], + ['id' => 'Sales', 'name' => 'Sales Department', 'budget' => 75000], + ]; + + /** + * @test + */ + public function joinBasicUsage(): void + { + // Create foreign database + $this->noneDB->insert('departments', self::DEPARTMENTS); + $deptByName = ['IT' => 0, 'HR' => 1, 'Sales' => 2]; + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id') + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $row = $this->person($key); + $deptKey = $deptByName[$row['department']]; + $row['departments'] = self::DEPARTMENTS[$deptKey] + ['key' => $deptKey]; + $expected[] = $row; + } + $this->assertSameRecords($expected, $results); + } + + /** + * @test + */ + public function joinWithCustomAlias(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'IT', 'name' => 'Information Technology'], + ]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id', 'dept') + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); + foreach ($results as $record) { + $this->assertArrayHasKey('dept', $record); + $this->assertArrayNotHasKey('departments', $record); + $expected = $record['department'] === 'IT' ? ['id' => 'IT', 'key' => 0, 'name' => 'Information Technology'] : null; + $this->assertSame($expected, self::canon($record['dept']), $record['name']); + } + } + + /** + * @test + */ + public function joinWithNoMatch(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'Finance', 'name' => 'Finance Department'], + ]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id') + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); + foreach ($results as $record) { + $this->assertArrayHasKey('departments', $record); + $this->assertNull($record['departments']); // No match + } + } + + /** + * @test + */ + public function joinPartialMatch(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'IT', 'name' => 'Information Technology'], + // HR and Sales not present + ]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id') + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); + foreach ($results as $record) { + if ($record['department'] === 'IT') { + $this->assertSame(['id' => 'IT', 'key' => 0, 'name' => 'Information Technology'], self::canon($record['departments'])); + } else { + $this->assertNull($record['departments'], $record['name']); // HR and Sales + } + } + } + + /** + * @test + */ + public function joinWithMissingLocalKey(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoDept']); // No department field + $this->noneDB->insert('departments', [['id' => 'IT', 'name' => 'IT']]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id') + ->get(); + + $noDeptRecord = array_values(array_filter($results, function ($r) { return $r['name'] === 'NoDept'; })); + $this->assertCount(1, $noDeptRecord); + + $this->assertSame(['departments' => null, 'key' => 6, 'name' => 'NoDept'], self::canon($noDeptRecord[0])); + } + + /** + * QUERY.md join(): "Joining a database that doesn't exist attaches null and creates nothing." + * + * @test + */ + public function joinWithNonExistentForeignDb(): void + { + $results = $this->noneDB->query($this->testDbName) + ->join('nonexistent-db', 'department', 'id') + ->get(); + + $expected = []; + foreach (array_keys(self::PEOPLE) as $key) { + $expected[] = $this->person($key) + ['nonexistent-db' => null]; + } + $this->assertSameRecords($expected, $results); + + // Nothing was created: no database, no file of any kind (not even a lock file) + $this->assertFalse($this->noneDB->getDBs('nonexistent-db')); + $this->assertNotContains('nonexistent-db', $this->noneDB->getDBs(false)); + $this->assertDirectoryDoesNotExist($this->dbPath('nonexistent-db', 'home')); + + // The same with a key join + $byKey = $this->noneDB->query($this->testDbName)->join('nonexistent-db', 'age', 'key', 'x')->get(); + $this->assertSame(array_fill(0, 6, null), array_column($byKey, 'x')); + $this->assertDirectoryDoesNotExist($this->dbPath('nonexistent-db', 'home')); + } + + /** + * QUERY.md join(): if several foreign records have the same value, the last one wins + * + * @test + */ + public function joinDuplicateForeignKeyLastRecordWins(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'IT', 'name' => 'First IT'], + ['id' => 'HR', 'name' => 'Human Resources'], + ['id' => 'IT', 'name' => 'Second IT'], + ]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id') + ->get(); + + foreach ($results as $record) { + if ($record['department'] === 'IT') { + $this->assertSame(['id' => 'IT', 'key' => 2, 'name' => 'Second IT'], self::canon($record['departments']), $record['name']); + } elseif ($record['department'] === 'HR') { + $this->assertSame(['id' => 'HR', 'key' => 1, 'name' => 'Human Resources'], self::canon($record['departments']), $record['name']); + } else { + $this->assertNull($record['departments'], $record['name']); + } + } + } + + /** + * QUERY.md join(): the attached record replaces a field of the same name in the result only + * + * @test + */ + public function joinReplacesSameNamedFieldInResultOnly(): void + { + $this->noneDB->insert('departments', [['id' => 'IT', 'name' => 'IT Dept']]); + $this->noneDB->insert($this->testDbName, ['name' => 'Grace', 'department' => 'IT', 'departments' => 'stored value']); + + $grace = $this->noneDB->query($this->testDbName)->where(['name' => 'Grace'])->join('departments', 'department', 'id')->get(); + $this->assertCount(1, $grace); + $this->assertSame(['id' => 'IT', 'key' => 0, 'name' => 'IT Dept'], self::canon($grace[0]['departments'])); + + // Storage unchanged + $stored = $this->noneDB->find($this->testDbName, ['name' => 'Grace']); + $this->assertSame([['department' => 'IT', 'departments' => 'stored value', 'key' => 6, 'name' => 'Grace']], self::canon($stored)); + $this->assertSame([['id' => 'IT', 'key' => 0, 'name' => 'IT Dept']], self::canon($this->noneDB->find('departments', 0))); + } + + /** + * @test + */ + public function multipleJoins(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'IT', 'name' => 'IT Dept'], + ]); + $this->noneDB->insert('cities', [ + ['name' => 'Istanbul', 'country' => 'Turkey'], + ]); + + $results = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'id', 'dept') + ->join('cities', 'city', 'name', 'cityInfo') + ->get(); + + $this->assertSame(self::ALL_NAMES, $this->names($results)); + $it = ['id' => 'IT', 'key' => 0, 'name' => 'IT Dept']; + $istanbul = ['country' => 'Turkey', 'key' => 0, 'name' => 'Istanbul']; + foreach ($results as $record) { + $this->assertSame($record['department'] === 'IT' ? $it : null, self::canon($record['dept']), $record['name']); + $this->assertSame($record['city'] === 'Istanbul' ? $istanbul : null, self::canon($record['cityInfo']), $record['name']); + } + } + + /** + * @test + */ + public function joinCombinedWithFilters(): void + { + $this->noneDB->insert('departments', [ + ['id' => 'IT', 'name' => 'Information Technology'], + ]); + + $results = $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->join('departments', 'department', 'id') + ->get(); + + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); + foreach ($results as $record) { + $this->assertSame(['id' => 'IT', 'key' => 0, 'name' => 'Information Technology'], self::canon($record['departments'])); + } + } + + // ========================================== + // skip() AND orderBy() ALIASES TESTS + // ========================================== + + /** + * @test + */ + public function skipIsAliasForOffset(): void + { + $skipResults = $this->noneDB->query($this->testDbName) + ->skip(2) + ->get(); + + $offsetResults = $this->noneDB->query($this->testDbName) + ->offset(2) + ->get(); + + $this->assertSame($offsetResults, $skipResults); + $this->assertSame(['Charlie', 'David', 'Eve', 'Frank'], $this->names($skipResults)); + } + + /** + * @test + */ + public function orderByIsAliasForSort(): void + { + $orderByResults = $this->noneDB->query($this->testDbName) + ->orderBy('age', 'desc') + ->get(); + + $sortResults = $this->noneDB->query($this->testDbName) + ->sort('age', 'desc') + ->get(); + + $this->assertSame($sortResults, $orderByResults); + $this->assertSame(['Frank', 'Charlie', 'Bob', 'David', 'Alice', 'Eve'], $this->names($orderByResults)); + } + + /** + * @test + */ + public function skipWithLimit(): void + { + $results = $this->noneDB->query($this->testDbName) + ->skip(1) + ->limit(3) + ->get(); + + $this->assertSame(['Bob', 'Charlie', 'David'], $this->names($results)); + } + + /** + * @test + */ + public function orderByAscending(): void + { + $results = $this->noneDB->query($this->testDbName) + ->orderBy('age', 'asc') + ->get(); + + $this->assertSame([22, 25, 28, 30, 35, 40], array_column($results, 'age')); + $this->assertSame(['Eve', 'Alice', 'David', 'Bob', 'Charlie', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function orderByDescending(): void + { + $results = $this->noneDB->query($this->testDbName) + ->orderBy('score', 'desc') + ->get(); + + $this->assertSame([95, 90, 85, 80, 75, 70], array_column($results, 'score')); + $this->assertSame(['David', 'Bob', 'Alice', 'Eve', 'Charlie', 'Frank'], $this->names($results)); + } + + /** + * @test + */ + public function chainableMethodsReturnSelf(): void + { + $query = $this->noneDB->query($this->testDbName); + + $this->assertSame($query, $query->where(['city' => 'Istanbul'])); + $this->assertSame($query, $query->orWhere(['city' => 'Istanbul'])); + $this->assertSame($query, $query->whereIn('city', ['Istanbul'])); + $this->assertSame($query, $query->whereNotIn('city', ['Ankara'])); + $this->assertSame($query, $query->whereNot(['active' => false])); + $this->assertSame($query, $query->like('email', 'mail')); + $this->assertSame($query, $query->notLike('email', 'yahoo')); + $this->assertSame($query, $query->between('age', 1, 99)); + $this->assertSame($query, $query->notBetween('age', 50, 60)); + $this->assertSame($query, $query->select(['name'])); + $this->assertSame($query, $query->except(['email'])); + $this->assertSame($query, $query->groupBy('city')); + $this->assertSame($query, $query->having('count', '>', 0)); + $this->assertSame($query, $query->search('test')); + $this->assertSame($query, $query->join('other', 'id', 'id')); + $this->assertSame($query, $query->sort('name')); + $this->assertSame($query, $query->limit(5)); + $this->assertSame($query, $query->offset(0)); + $this->assertSame($query, $query->skip(1)); + $this->assertSame($query, $query->orderBy('name')); + } + + // ========================================== + // COMPLEX COMBINATION TESTS + // ========================================== + + /** + * @test + */ + public function complexQueryWithMultipleFilters(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->whereNotIn('department', ['Sales']) + ->between('age', 20, 35) + ->notBetween('score', 70, 75) + ->like('email', 'gmail') + ->notLike('name', '^F') + ->sort('score', 'desc') + ->limit(2) + ->select(['name', 'score', 'city']) + ->get(); + + // whereIn: Alice Bob Charlie Eve Frank; between age 20..35 drops Frank; notBetween score 70..75 drops + // Charlie (75); like gmail drops Bob; left: Alice (85), Eve (80) -> by score desc + $this->assertSame([ + ['city' => 'Istanbul', 'key' => 0, 'name' => 'Alice', 'score' => 85], + ['city' => 'Istanbul', 'key' => 4, 'name' => 'Eve', 'score' => 80], + ], self::canon($results)); + } + + /** + * @test + */ + public function paginationWithSearch(): void + { + $page1 = $this->noneDB->query($this->testDbName) + ->search('istanbul') + ->orderBy('name', 'asc') + ->limit(2) + ->skip(0) + ->get(); + + $page2 = $this->noneDB->query($this->testDbName) + ->search('istanbul') + ->orderBy('name', 'asc') + ->limit(2) + ->skip(2) + ->get(); + + $this->assertSame(['Alice', 'Charlie'], $this->names($page1)); + $this->assertSame(['Eve'], $this->names($page2)); + } +} diff --git a/tests/Feature/NewMethodsEdgeCasesTest.php b/tests/Integration/NewMethodsEdgeCasesTest.php similarity index 62% rename from tests/Feature/NewMethodsEdgeCasesTest.php rename to tests/Integration/NewMethodsEdgeCasesTest.php index d0940d0..082afbb 100644 --- a/tests/Feature/NewMethodsEdgeCasesTest.php +++ b/tests/Integration/NewMethodsEdgeCasesTest.php @@ -1,6 +1,6 @@ assertCount(3, $results); + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); } /** @@ -60,7 +73,7 @@ public function orWhereWithBetween(): void // IT OR HR, then age 25-35 // IT: Alice(25), Charlie(35), Eve(22-out) // HR: Bob(30), Frank(40-out) - $this->assertCount(3, $results); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); } /** @@ -76,7 +89,7 @@ public function orWhereWithWhereIn(): void // Izmir OR Ankara = Bob, David, Frank // Then whereIn HR or Sales = Bob(HR), David(Sales), Frank(HR) - $this->assertCount(3, $results); + $this->assertSame(['Bob', 'David', 'Frank'], $this->names($results)); } /** @@ -92,7 +105,7 @@ public function orWhereWithWhereNot(): void // Istanbul OR Ankara = Alice, Bob, Charlie, Eve, Frank // Then exclude active=false: Charlie, Frank excluded - $this->assertCount(3, $results); + $this->assertSame(['Alice', 'Bob', 'Eve'], $this->names($results)); } /** @@ -108,7 +121,7 @@ public function orWhereWithSearch(): void // IT OR Sales = Alice, Charlie, Eve, David // Search gmail = Alice, Charlie, Eve (David is hotmail) - $this->assertCount(3, $results); + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); } // ========================================== @@ -121,13 +134,14 @@ public function orWhereWithSearch(): void public function whereInWithNullInArray(): void { $this->noneDB->insert($this->testDbName, ['name' => 'NullCity', 'city' => null]); + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); $results = $this->noneDB->query($this->testDbName) ->whereIn('city', [null, 'Istanbul']) ->get(); - // Should match Istanbul (3) + null (1) - $this->assertCount(4, $results); + // Istanbul (3) + the stored null (1); a missing field is not a null value + $this->assertSame(['Alice', 'Charlie', 'Eve', 'NullCity'], $this->names($results)); } /** @@ -136,13 +150,14 @@ public function whereInWithNullInArray(): void public function whereNotInWithNullInArray(): void { $this->noneDB->insert($this->testDbName, ['name' => 'NullCity', 'city' => null]); + $this->noneDB->insert($this->testDbName, ['name' => 'NoCity']); $results = $this->noneDB->query($this->testDbName) ->whereNotIn('city', [null]) ->get(); - // Should exclude null, return 6 original records - $this->assertCount(6, $results); + // Excludes the stored null only: the 6 original records and the record without city + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank', 'NoCity'], $this->names($results)); } /** @@ -155,7 +170,9 @@ public function whereInWithBooleanValues(): void ->get(); // Alice, Bob, David, Eve are active - $this->assertCount(4, $results); + $this->assertSame(['Alice', 'Bob', 'David', 'Eve'], $this->names($results)); + // strict: 1 is not true + $this->assertSame([], $this->noneDB->query($this->testDbName)->whereIn('active', [1, 'true'])->get()); } /** @@ -172,8 +189,14 @@ public function whereInWithMixedTypes(): void ->get(); // Strict comparison: only integer 0 matches - $this->assertCount(1, $results); - $this->assertEquals('Mixed', $results[0]['name']); + $this->assertSame(['Mixed'], $this->names($results)); + $this->assertSame(['Mixed2'], $this->names($this->noneDB->query($this->testDbName)->whereIn('value', ['0'])->get())); + $this->assertSame(['Mixed3'], $this->names($this->noneDB->query($this->testDbName)->whereIn('value', [false])->get())); + $this->assertSame(['Mixed', 'Mixed3'], $this->names($this->noneDB->query($this->testDbName)->whereIn('value', [0, false])->get())); + // also strict with a long list (more than 8 values) + $long = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + $this->assertSame(['Mixed'], $this->names($this->noneDB->query($this->testDbName)->whereIn('value', $long)->get())); + $this->assertSame(['Mixed2', 'Mixed3'], $this->names($this->noneDB->query($this->testDbName)->whereNotIn('value', $long)->where(['name' => ['$like' => '^Mixed']])->get())); } // ========================================== @@ -190,7 +213,7 @@ public function searchWithEmptyTerm(): void ->get(); // Empty string matches everything - $this->assertCount(6, $results); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); } /** @@ -208,8 +231,9 @@ public function searchWithSpecialCharacters(): void ->search('(parentheses)') ->get(); - $this->assertCount(1, $results); - $this->assertEquals('Special', $results[0]['name']); + $this->assertSame(['Special'], $this->names($results)); + $this->assertSame(['Special'], $this->names($this->noneDB->query($this->testDbName)->search('+special@')->get())); + $this->assertSame(['Special'], $this->names($this->noneDB->query($this->testDbName)->search('[BRACKETS]')->get())); } /** @@ -226,7 +250,8 @@ public function searchWithUnicodeCharacters(): void ->search('dünya') ->get(); - $this->assertCount(1, $results); + $this->assertSame(['Turkish'], $this->names($results)); + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('dunya')->get(), 'ü is not u'); } /** @@ -239,7 +264,8 @@ public function searchOnlyInNonExistentField(): void ->get(); // No records have nonexistent_field, so nothing matches - $this->assertEmpty($results); + $this->assertSame([], $results); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->search('alice', ['nonexistent_field', 'name'])->get())); } // ========================================== @@ -257,8 +283,8 @@ public function groupByWithEmptyDatabase(): void ->groupBy('city') ->get(); - $this->assertIsArray($results); - $this->assertEmpty($results); + $this->assertSame([], $results); + $this->assertSame(0, $this->noneDB->query('emptydb')->groupBy('city')->count()); } /** @@ -273,9 +299,7 @@ public function groupByWithLimit(): void ->get(); // Should get top 2 cities by count - $this->assertCount(2, $results); - $this->assertEquals('Istanbul', $results[0]['_group']); // 3 records - $this->assertEquals('Ankara', $results[1]['_group']); // 2 records + $this->assertSame([['Istanbul', 3], ['Ankara', 2]], array_map(function ($g) { return [$g['_group'], $g['_count']]; }, $results)); } /** @@ -290,7 +314,7 @@ public function groupByWithOffset(): void ->get(); // Skip Istanbul (first), get Ankara and Izmir - $this->assertCount(2, $results); + $this->assertSame([['Ankara', 2], ['Izmir', 1]], array_map(function ($g) { return [$g['_group'], $g['_count']]; }, $results)); } /** @@ -305,14 +329,12 @@ public function groupByPreservesOriginalRecordsInItems(): void // All Istanbul are IT, so 1 group $this->assertCount(1, $results); - $this->assertEquals('IT', $results[0]['_group']); + $this->assertSame('IT', $results[0]['_group']); + $this->assertSame(3, $results[0]['_count']); - // Each item should have all original fields - foreach ($results[0]['_items'] as $item) { - $this->assertArrayHasKey('name', $item); - $this->assertArrayHasKey('email', $item); - $this->assertArrayHasKey('age', $item); - } + // The items are the complete stored records + $stored = $this->noneDB->find($this->testDbName, ['key' => [0, 2, 4]]); + $this->assertSame(array_map([$this, 'normalized'], $stored), array_map([$this, 'normalized'], $results[0]['_items'])); } // ========================================== @@ -330,7 +352,9 @@ public function havingWithoutGroupByIsIgnored(): void ->get(); // Should return all records (having ignored) - $this->assertCount(6, $results); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); + $this->assertSame(6, $this->noneDB->query($this->testDbName)->having('count', '>', 1)->count()); + $this->assertTrue($this->noneDB->query($this->testDbName)->having('count', '>', 100)->exists()); } /** @@ -345,7 +369,7 @@ public function havingWithEmptyGroups(): void ->having('count', '>', 0) ->get(); - $this->assertEmpty($results); + $this->assertSame([], $results); } // ========================================== @@ -363,13 +387,18 @@ public function selectAndExceptTogether(): void ->except(['age']) // Should be ignored since select is set ->get(); - // select takes priority, age should be included - foreach ($results as $record) { - $this->assertArrayHasKey('name', $record); - $this->assertArrayHasKey('age', $record); - $this->assertArrayHasKey('city', $record); - $this->assertArrayNotHasKey('email', $record); - } + // select takes priority, age should be included (not documented; see QUERY.md select()/except()) + $this->assertSame([ + ['name' => 'Alice', 'age' => 25, 'city' => 'Istanbul', 'key' => 0], + ['name' => 'Bob', 'age' => 30, 'city' => 'Ankara', 'key' => 1], + ['name' => 'Charlie', 'age' => 35, 'city' => 'Istanbul', 'key' => 2], + ['name' => 'David', 'age' => 28, 'city' => 'Izmir', 'key' => 3], + ['name' => 'Eve', 'age' => 22, 'city' => 'Istanbul', 'key' => 4], + ['name' => 'Frank', 'age' => 40, 'city' => 'Ankara', 'key' => 5], + ], array_map(function ($r) { + // field order of the expectation (the position of "key" is not documented) + return ['name' => $r['name'] ?? null, 'age' => $r['age'] ?? null, 'city' => $r['city'] ?? null, 'key' => $r['key'] ?? null] + $r; + }, $results)); } /** @@ -382,12 +411,14 @@ public function selectAfterGroupBy(): void ->select(['_group', '_count']) ->get(); - // select on grouped results - foreach ($results as $record) { - $this->assertArrayHasKey('_group', $record); - $this->assertArrayHasKey('_count', $record); - $this->assertArrayNotHasKey('_items', $record); - } + // QUERY.md: _group, _count and _items select parts of each group row: only those two + $rows = array_map([$this, 'normalized'], $results); + usort($rows, function ($a, $b) { return strcmp($a['_group'], $b['_group']); }); + $this->assertSame([ + ['_count' => 2, '_group' => 'Ankara'], + ['_count' => 3, '_group' => 'Istanbul'], + ['_count' => 1, '_group' => 'Izmir'], + ], $rows); } // ========================================== @@ -405,8 +436,7 @@ public function firstWithOrWhere(): void ->sort('name') ->first(); - $this->assertNotNull($result); - $this->assertEquals('Alice', $result['name']); + $this->assertSame($this->normalized($this->noneDB->find($this->testDbName, ['key' => 0])[0]), $this->normalized($result)); } /** @@ -419,8 +449,8 @@ public function lastWithWhereIn(): void ->sort('name') ->last(); - $this->assertNotNull($result); - $this->assertEquals('Frank', $result['name']); + $this->assertSame('Frank', $result['name']); + $this->assertSame(5, $result['key']); } /** @@ -432,7 +462,7 @@ public function countWithWhereNot(): void ->whereNot(['active' => false]) ->count(); - $this->assertEquals(4, $count); // Alice, Bob, David, Eve + $this->assertSame(4, $count); // Alice, Bob, David, Eve } /** @@ -446,6 +476,8 @@ public function existsWithNotLike(): void ->exists(); $this->assertTrue($exists); // David (hotmail) + $this->assertFalse($this->noneDB->query($this->testDbName) + ->notLike('email', 'gmail')->notLike('email', 'yahoo')->notLike('email', 'HOTMAIL')->exists()); } /** @@ -457,8 +489,8 @@ public function sumWithNotBetween(): void ->notBetween('age', 25, 35) ->sum('score'); - // Eve(22)=80, Frank(40)=70 - $this->assertEquals(150, $sum); + // Eve(22)=80, Frank(40)=70; the builder's sum() is a float + $this->assertSame(150.0, $sum); } /** @@ -470,8 +502,8 @@ public function avgWithSearch(): void ->search('gmail') ->avg('age'); - // Alice(25), Charlie(35), Eve(22) = avg(82/3) ≈ 27.33 - $this->assertEqualsWithDelta(27.33, $avg, 0.01); + // Alice(25), Charlie(35), Eve(22) = 82/3 + $this->assertSame(82 / 3, $avg); } /** @@ -484,7 +516,7 @@ public function minWithWhereIn(): void ->min('age'); // IT: 25, 35, 22. Sales: 28. Min = 22 - $this->assertEquals(22, $min); + $this->assertSame(22, $min); } /** @@ -497,7 +529,7 @@ public function maxWithWhereNotIn(): void ->max('score'); // Ankara: 90, 70. Izmir: 95. Max = 95 - $this->assertEquals(95, $max); + $this->assertSame(95, $max); } /** @@ -510,10 +542,8 @@ public function distinctWithOrWhere(): void ->orWhere(['department' => 'HR']) ->distinct('city'); - // IT: Istanbul(3). HR: Ankara(2). Distinct cities = Istanbul, Ankara - $this->assertCount(2, $cities); - $this->assertContains('Istanbul', $cities); - $this->assertContains('Ankara', $cities); + // IT: Istanbul(3). HR: Ankara(2). Distinct cities in order of first appearance (Alice, Bob) + $this->assertSame(['Istanbul', 'Ankara'], $cities); } // ========================================== @@ -530,11 +560,12 @@ public function updateWithOrWhere(): void ->orWhere(['city' => 'Ankara']) ->update(['region' => 'Marmara']); - $this->assertEquals(5, $result['n']); + $this->assertSame(['n' => 5], $result); - // Verify update + // Verify update: every Istanbul and Ankara record, nothing else $updated = $this->noneDB->find($this->testDbName, ['region' => 'Marmara']); - $this->assertCount(5, $updated); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($updated)); + $this->assertArrayNotHasKey('region', $this->noneDB->find($this->testDbName, ['key' => 3])[0]); } /** @@ -546,7 +577,9 @@ public function updateWithWhereIn(): void ->whereIn('department', ['IT', 'HR']) ->update(['type' => 'office']); - $this->assertEquals(5, $result['n']); // 3 IT + 2 HR + $this->assertSame(['n' => 5], $result); // 3 IT + 2 HR + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Eve', 'Frank'], $this->names($this->noneDB->find($this->testDbName, ['type' => 'office']))); + $this->assertSame(6, $this->noneDB->count($this->testDbName)); } /** @@ -558,11 +591,11 @@ public function deleteWithWhereNot(): void ->whereNot(['active' => true]) ->delete(); - $this->assertEquals(2, $result['n']); // Charlie, Frank + $this->assertSame(['n' => 2], $result); // Charlie, Frank // Verify remaining - $remaining = $this->noneDB->count($this->testDbName); - $this->assertEquals(4, $remaining); + $this->assertSame(['Alice', 'Bob', 'David', 'Eve'], $this->names($this->noneDB->find($this->testDbName, 0))); + $this->assertSame(4, $this->noneDB->count($this->testDbName)); } /** @@ -574,14 +607,10 @@ public function deleteWithNotBetween(): void ->notBetween('age', 25, 35) ->delete(); - $this->assertEquals(2, $result['n']); // Eve(22), Frank(40) + $this->assertSame(['n' => 2], $result); // Eve(22), Frank(40) - // Verify remaining ages are 25-35 - $remaining = $this->noneDB->find($this->testDbName, 0); - foreach ($remaining as $record) { - $this->assertGreaterThanOrEqual(25, $record['age']); - $this->assertLessThanOrEqual(35, $record['age']); - } + // Remaining: the records with ages 25-35 + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David'], $this->names($this->noneDB->find($this->testDbName, 0))); } // ========================================== @@ -599,7 +628,9 @@ public function notBetweenWithMinGreaterThanMax(): void ->get(); // All records should be returned - $this->assertCount(6, $results); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); + // and nothing lies in a reversed range + $this->assertSame([], $this->noneDB->query($this->testDbName)->between('age', 50, 20)->get()); } /** @@ -611,9 +642,10 @@ public function notBetweenWithStringValues(): void ->notBetween('name', 'B', 'D') ->get(); - // Bob, Charlie are between B-D alphabetically + // Bob, Charlie are between B-D alphabetically ("David" > "D" lexically) // Alice, David, Eve, Frank are outside - $this->assertCount(4, $results); + $this->assertSame(['Alice', 'David', 'Eve', 'Frank'], $this->names($results)); + $this->assertSame(['Bob', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->between('name', 'B', 'D')->get())); } // ========================================== @@ -638,13 +670,11 @@ public function realWorldUserFilteringScenario(): void ->select(['name', 'score', 'city', 'email']) ->get(); - $this->assertLessThanOrEqual(3, count($results)); - - // All results should match criteria - foreach ($results as $record) { - $this->assertContains($record['city'], ['Istanbul', 'Ankara']); - $this->assertStringContainsString('gmail', $record['email']); - } + // active + Istanbul/Ankara + gmail + 20..35: Alice (85), Eve (80); Charlie is inactive + $this->assertSame([ + ['name' => 'Alice', 'score' => 85, 'city' => 'Istanbul', 'email' => 'alice@gmail.com', 'key' => 0], + ['name' => 'Eve', 'score' => 80, 'city' => 'Istanbul', 'email' => 'eve@gmail.com', 'key' => 4], + ], $results); } /** @@ -662,7 +692,9 @@ public function realWorldReportingScenario(): void // IT: count=3, avg=80. HR: count=2, avg=80. Both pass. // Sales: count=1, fails count condition - $this->assertCount(2, $results); + $this->assertSame([['IT', 3], ['HR', 2]], array_map(function ($g) { return [$g['_group'], $g['_count']]; }, $results)); + // a stricter average drops both + $this->assertSame([], $this->noneDB->query($this->testDbName)->groupBy('department')->having('count', '>=', 2)->having('avg:score', '>', 80)->get()); } /** @@ -691,9 +723,9 @@ public function realWorldSearchWithPagination(): void ->skip($pageSize) ->get(); - $this->assertEquals(3, $totalCount); // Alice, Charlie, Eve - $this->assertCount(2, $page1); - $this->assertCount(1, $page2); + $this->assertSame(3, $totalCount); // Alice, Charlie, Eve + $this->assertSame(['Alice', 'Charlie'], $this->names($page1)); + $this->assertSame(['Eve'], $this->names($page2)); } // ========================================== @@ -712,7 +744,9 @@ public function joinWithEmptyForeignDb(): void ->get(); // All joins should be null + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'], $this->names($results)); foreach ($results as $record) { + $this->assertArrayHasKey('empty-departments', $record); $this->assertNull($record['empty-departments']); } } @@ -734,8 +768,10 @@ public function joinWithDuplicateForeignKeys(): void ->get(); // Last matching record should be used (v2) + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); foreach ($results as $record) { - $this->assertEquals('IT Department v2', $record['departments']['name']); + $this->assertSame('IT Department v2', $record['departments']['name']); + $this->assertSame(1, $record['departments']['key']); } } @@ -758,12 +794,15 @@ public function joinThenFilter(): void ->get(); // Should have IT employees with joined department info - $this->assertCount(3, $results); + $this->assertSame(['Alice', 'Charlie', 'Eve'], $this->names($results)); foreach ($results as $record) { - $this->assertArrayHasKey('departments', $record); - $this->assertNotNull($record['departments']); - $this->assertEquals('Information Technology', $record['departments']['name']); + $this->assertSame( + ['budget' => 100000, 'id' => 'IT', 'key' => 0, 'name' => 'Information Technology'], + $this->normalized($record['departments']) + ); } + // the joined data is not searched: search() runs on the stored fields only + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['department' => 'IT'])->join('departments', 'department', 'id')->search('Information')->get()); } /** @@ -779,9 +818,12 @@ public function joinWithNullLocalKeyValue(): void ->join('departments', 'department', 'id') ->get(); - $nullRecord = array_values(array_filter($results, fn($r) => $r['name'] === 'NullDept'))[0]; + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank', 'NullDept'], $this->names($results)); + $nullRecord = $results[6]; $this->assertArrayHasKey('departments', $nullRecord); $this->assertNull($nullRecord['departments']); // null key should result in null join + $this->assertSame(['IT Dept', null, 'IT Dept', null, 'IT Dept', null, null], + array_map(function ($r) { return $r['departments']['name'] ?? null; }, $results)); } /** @@ -800,10 +842,13 @@ public function joinWithArrayLocalKeyValue(): void ->join('departments', 'department', 'id') ->get(); - // Should not crash, array key won't match string foreign key - $arrayRecord = array_values(array_filter($results, fn($r) => $r['name'] === 'ArrayDept'))[0]; + // Arrays match nothing (QUERY.md join matching) + $this->assertCount(7, $results); + $arrayRecord = $results[6]; + $this->assertSame('ArrayDept', $arrayRecord['name']); $this->assertArrayHasKey('departments', $arrayRecord); $this->assertNull($arrayRecord['departments']); + $this->assertSame('IT Dept', $results[0]['departments']['name']); } /** @@ -822,10 +867,11 @@ public function joinWithNumericKeyValue(): void ->join('departments', 'department', 'id') ->get(); - $numericRecord = array_values(array_filter($results, fn($r) => $r['name'] === 'NumericDept'))[0]; - $this->assertArrayHasKey('departments', $numericRecord); - // Numeric 123 should match either 123 or '123' depending on indexing - $this->assertNotNull($numericRecord['departments']); + $numericRecord = $results[6]; + $this->assertSame('NumericDept', $numericRecord['name']); + // QUERY.md: integers and canonical integer strings match each other, and the last duplicate wins + $this->assertSame('Dept String 123', $numericRecord['departments']['name']); + $this->assertSame(1, $numericRecord['departments']['key']); } /** @@ -844,8 +890,12 @@ public function joinWithBooleanKeyValue(): void ->join('statuses', 'active', 'id') ->get(); - // Should not crash - $this->assertGreaterThan(0, count($results)); + // QUERY.md: true matches only true and false only false + $this->assertSame( + ['Active', 'Active', 'Inactive', 'Active', 'Active', 'Inactive', 'Active'], + array_map(function ($r) { return $r['statuses']['label'] ?? null; }, $results) + ); + $this->assertSame('BoolDept', $results[6]['name']); } /** @@ -863,10 +913,15 @@ public function joinWithNullForeignKeyValue(): void ->join('departments', 'department', 'id') ->get(); - // Records with IT department should match IT Dept - $itRecord = array_values(array_filter($results, fn($r) => $r['department'] === 'IT'))[0]; - $this->assertNotNull($itRecord['departments']); - $this->assertEquals('IT Dept', $itRecord['departments']['name']); + // IT records match IT Dept; HR/Sales match nothing + $this->assertSame(['IT Dept', null, 'IT Dept', null, 'IT Dept', null], + array_map(function ($r) { return $r['departments']['name'] ?? null; }, $results)); + + // a local null doesn't join the foreign null (null matches nothing) + $this->noneDB->insert($this->testDbName, ['name' => 'NullDept', 'department' => null]); + $withNull = $this->noneDB->query($this->testDbName)->where(['name' => 'NullDept'])->join('departments', 'department', 'id')->get(); + $this->assertCount(1, $withNull); + $this->assertNull($withNull[0]['departments']); } // ========================================== @@ -890,13 +945,15 @@ public function filterOrderDoesNotMatter(): void ->where(['city' => 'Istanbul']) ->get(); - $this->assertEquals(count($results1), count($results2)); + // Istanbul + IT + age 20..30: Alice (25), Eve (22) + $this->assertSame(['Alice', 'Eve'], $this->names($results1)); + $this->assertSame($results1, $results2); } /** * @test */ - public function sortAndLimitOrderMatters(): void + public function limitAppliesAfterSortWhateverTheCallOrder(): void { // Sort then limit - should get sorted top 2 $sorted = $this->noneDB->query($this->testDbName) @@ -904,26 +961,13 @@ public function sortAndLimitOrderMatters(): void ->limit(2) ->get(); - // Verify sorted correctly - $this->assertEquals(95, $sorted[0]['score']); // David - $this->assertEquals(90, $sorted[1]['score']); // Bob - } + $this->assertSame(['David', 'Bob'], $this->names($sorted)); // 95, 90 - protected function tearDown(): void - { - // Clean up additional test databases - $dbDir = $this->getPrivateProperty('dbDir'); - $hashMethod = $this->getPrivateMethod('hashDBName'); - foreach (['departments', 'empty-departments', 'emptydb'] as $db) { - $hash = $hashMethod->invoke($this->noneDB, $db); - $files = glob($dbDir . $hash . '-' . $db . '*'); - foreach ($files as $file) { - if (file_exists($file)) { - unlink($file); - } - } - } - - parent::tearDown(); + // limit() called before sort(): the same records (the order of the calls doesn't matter) + $limitFirst = $this->noneDB->query($this->testDbName) + ->limit(2) + ->sort('score', 'desc') + ->get(); + $this->assertSame($sorted, $limitFirst); } } diff --git a/tests/Integration/NewMethodsShardedTest.php b/tests/Integration/NewMethodsShardedTest.php new file mode 100644 index 0000000..d567c29 --- /dev/null +++ b/tests/Integration/NewMethodsShardedTest.php @@ -0,0 +1,616 @@ +setPrivateProperty('shardSize', 5); + $this->setPrivateProperty('shardingEnabled', true); + + // Insert enough data to trigger sharding (more than shardSize) + $this->noneDB->insert($this->shardedDbName, array_values($this->records())); + } + + /** + * The inserted records, key => record (without key) + */ + private function records(): array + { + $data = []; + for ($i = 0; $i < 12; $i++) { + $data[$i] = [ + 'name' => 'User' . $i, + 'age' => 20 + $i, + 'city' => self::CITIES[$i % 4], + 'department' => self::DEPARTMENTS[$i % 4], + 'score' => 70 + ($i * 2), + 'active' => ($i % 3 !== 0) + ]; + } + return $data; + } + + /** + * Stored record as get() returns it, field order normalized + */ + private function record(int $key, ?array $only = null): array + { + $record = $this->records()[$key]; + if ($only !== null) { + $record = array_intersect_key($record, array_flip($only)); + } + $record['key'] = $key; + ksort($record); + return $record; + } + + private function keys(array $results): array + { + return array_map(function ($r) { return $r['key']; }, $results); + } + + private function names(array $results): array + { + return array_map(function ($r) { return $r['name']; }, $results); + } + + private static function canon($value) + { + if (!is_array($value)) { + return $value; + } + $isList = $value === [] || array_keys($value) === range(0, count($value) - 1); + if (!$isList) { + ksort($value, SORT_STRING); + } + foreach ($value as $k => $item) { + $value[$k] = self::canon($item); + } + return $value; + } + + /** + * @test + */ + public function databaseIsSharded(): void + { + // Verify data was inserted correctly + $this->assertSame(12, $this->noneDB->count($this->shardedDbName)); + + // 12 keys / 5 per shard = shards 0 (0-4), 1 (5-9), 2 (10-11) + $this->assertSame([ + 'sharded' => true, + 'shards' => 3, + 'totalRecords' => 12, + 'deletedCount' => 0, + 'shardSize' => 5, + 'nextKey' => 12, + ], $this->noneDB->getShardInfo($this->shardedDbName)); + + // All records are readable, in key order across the shards + $all = $this->noneDB->query($this->shardedDbName)->get(); + $expected = []; + foreach (array_keys($this->records()) as $key) { + $expected[] = $this->record($key); + } + $this->assertSame($expected, self::canon($all)); + } + + /** + * @test + */ + public function whereInOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->get(); + + // Istanbul (i % 4 == 0) and Ankara (i % 4 == 1), in key order across the shards + $this->assertSame([0, 1, 4, 5, 8, 9], $this->keys($results)); + $this->assertSame([$this->record(0), $this->record(1), $this->record(4), $this->record(5), $this->record(8), $this->record(9)], self::canon($results)); + } + + /** + * @test + */ + public function whereNotInOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->whereNotIn('city', ['Istanbul', 'Ankara']) + ->get(); + + // Izmir and Bursa each appear 3 times + $this->assertSame([2, 3, 6, 7, 10, 11], $this->keys($results)); + } + + /** + * @test + */ + public function orWhereOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->where(['city' => 'Istanbul']) + ->orWhere(['city' => 'Izmir']) + ->get(); + + $this->assertSame([0, 2, 4, 6, 8, 10], $this->keys($results)); + } + + /** + * @test + */ + public function whereNotOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->whereNot(['active' => false]) + ->get(); + + // 8 active users (indices 1,2,4,5,7,8,10,11) + $this->assertSame([1, 2, 4, 5, 7, 8, 10, 11], $this->keys($results)); + + // whereNot with operators: excluded when all operators of the field match + $ops = $this->noneDB->query($this->shardedDbName)->whereNot(['age' => ['$gte' => 23, '$lte' => 27]])->get(); + $this->assertSame([0, 1, 2, 8, 9, 10, 11], $this->keys($ops)); + } + + /** + * @test + */ + public function notLikeOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->notLike('name', 'User1') + ->get(); + + // Excludes User1, User10, User11 = 9 remaining + $this->assertSame([0, 2, 3, 4, 5, 6, 7, 8, 9], $this->keys($results)); + $this->assertSame([1, 10, 11], $this->keys($this->noneDB->query($this->shardedDbName)->like('name', 'user1')->get())); + } + + /** + * @test + */ + public function notBetweenOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->notBetween('age', 22, 28) + ->get(); + + // Ages: 20,21 (below) + 29,30,31 (above) = 5 + $this->assertSame([0, 1, 9, 10, 11], $this->keys($results)); + $this->assertSame([2, 3, 4, 5, 6, 7, 8], $this->keys($this->noneDB->query($this->shardedDbName)->between('age', 22, 28)->get())); + } + + /** + * @test + */ + public function selectOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->select(['name', 'city']) + ->get(); + + $expected = []; + foreach (array_keys($this->records()) as $key) { + $expected[] = $this->record($key, ['name', 'city']); // key always included + } + $this->assertSame($expected, self::canon($results)); + } + + /** + * @test + */ + public function exceptOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->except(['score', 'active']) + ->get(); + + $expected = []; + foreach (array_keys($this->records()) as $key) { + $expected[] = $this->record($key, ['name', 'age', 'city', 'department']); + } + $this->assertSame($expected, self::canon($results)); + } + + /** + * @test + */ + public function groupByOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->groupBy('city') + ->get(); + + $this->assertCount(4, $results); + + $byCity = []; + foreach ($results as $group) { + $this->assertSame(['_group', '_items', '_count'], array_keys($group)); + $byCity[$group['_group']] = [$group['_count'], $this->keys($group['_items'])]; + } + ksort($byCity); + // Each city has 3 users, items in key order across the shards + $this->assertSame([ + 'Ankara' => [3, [1, 5, 9]], + 'Bursa' => [3, [3, 7, 11]], + 'Istanbul' => [3, [0, 4, 8]], + 'Izmir' => [3, [2, 6, 10]], + ], $byCity); + } + + /** + * @test + */ + public function havingOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->groupBy('department') + ->having('avg:score', '>', 80) + ->get(); + + // avg score: IT (70,78,86) 78; HR (72,80,88) 80; Sales (74,82,90) 82; Marketing (76,84,92) 84 + $groups = array_column($results, '_group'); + sort($groups); + $this->assertSame(['Marketing', 'Sales'], $groups); + + // >= 80 adds HR (exactly 80) + $ge = array_column($this->noneDB->query($this->shardedDbName)->groupBy('department')->having('avg:score', '>=', 80)->get(), '_group'); + sort($ge); + $this->assertSame(['HR', 'Marketing', 'Sales'], $ge); + } + + /** + * @test + */ + public function searchOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->search('istanbul') + ->get(); + + $this->assertSame([0, 4, 8], $this->keys($results)); + } + + /** + * @test + */ + public function joinOnShardedDb(): void + { + // Create a small non-sharded lookup table + $this->noneDB->insert('city-info', [ + ['name' => 'Istanbul', 'country' => 'Turkey', 'population' => 15000000], + ['name' => 'Ankara', 'country' => 'Turkey', 'population' => 5500000], + ]); + + $results = $this->noneDB->query($this->shardedDbName) + ->join('city-info', 'city', 'name') + ->get(); + + $this->assertSame(range(0, 11), $this->keys($results)); + + $info = [ + 'Istanbul' => ['country' => 'Turkey', 'key' => 0, 'name' => 'Istanbul', 'population' => 15000000], + 'Ankara' => ['country' => 'Turkey', 'key' => 1, 'name' => 'Ankara', 'population' => 5500000], + ]; + foreach ($results as $record) { + // Izmir and Bursa have no city-info record: null + $this->assertSame($info[$record['city']] ?? null, self::canon($record['city-info']), $record['name']); + } + } + + /** + * A sharded database joined on its record keys from another database (keys in every shard) + * + * @test + */ + public function joinOnKeyOfShardedDb(): void + { + $this->noneDB->insert('city-info', [ + ['name' => 'a', 'user' => 11], + ['name' => 'b', 'user' => '3'], + ['name' => 'c', 'user' => 7.0], + ['name' => 'd', 'user' => 99], + ]); + + $results = $this->noneDB->query('city-info')->join($this->shardedDbName, 'user', 'key', 'u')->get(); + + // 11, "3" and 7.0 join keys 11, 3 and 7 (QUERY.md join: integers, integral floats and canonical + // integer strings match each other); 99 doesn't exist + $this->assertSame([11, 3, 7, null], array_map(function ($r) { return $r['u']['key'] ?? null; }, $results)); + $this->assertSame($this->record(3), self::canon($results[1]['u'])); + } + + /** + * @test + */ + public function complexQueryOnShardedDb(): void + { + $results = $this->noneDB->query($this->shardedDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->whereNot(['active' => false]) + ->between('age', 21, 28) + ->sort('score', 'desc') + ->limit(3) + ->select(['name', 'city', 'score']) + ->get(); + + // whereIn: 0,1,4,5,8,9; active drops 0 and 9; age 21..28: 1(21),4(24),5(25),8(28); + // score desc: 8 (86), 5 (80), 4 (78), 1 (72); limit 3 + $this->assertSame([ + $this->record(8, ['name', 'city', 'score']), + $this->record(5, ['name', 'city', 'score']), + $this->record(4, ['name', 'city', 'score']), + ], self::canon($results)); + } + + /** + * @test + */ + public function updateOnShardedDbWithNewFilters(): void + { + $result = $this->noneDB->query($this->shardedDbName) + ->whereIn('city', ['Istanbul']) + ->update(['region' => 'Marmara']); + + $this->assertSame(['n' => 3], $result); + + // Verify update: exactly the Istanbul records (one per shard) got the field, nothing else changed + $updated = $this->noneDB->find($this->shardedDbName, ['region' => 'Marmara']); + $expected = []; + foreach ([0, 4, 8] as $key) { + $row = $this->record($key) + ['region' => 'Marmara']; + ksort($row); + $expected[] = $row; + } + $this->assertSame($expected, self::canon($updated)); + foreach ($this->noneDB->find($this->shardedDbName, 0) as $record) { + if (!in_array($record['key'], [0, 4, 8], true)) { + $this->assertSame($this->record($record['key']), self::canon($record)); + } + } + } + + /** + * @test + */ + public function deleteOnShardedDbWithNewFilters(): void + { + $result = $this->noneDB->query($this->shardedDbName) + ->whereNot(['active' => true]) + ->delete(); + + $this->assertSame(['n' => 4], $result); // Users 0,3,6,9 are inactive + + // Verify remaining records + $this->assertSame(8, $this->noneDB->count($this->shardedDbName)); + $this->assertSame([1, 2, 4, 5, 7, 8, 10, 11], $this->keys($this->noneDB->find($this->shardedDbName, 0))); + } + + /** + * QUERY.md update(): sort(), offset() and limit() choose which records are written + * + * @test + */ + public function updateWithSortAndLimitChoosesRecordsAcrossShards(): void + { + // The 3 highest scores are 11 (92), 10 (90), 9 (88); skip the first one + $result = $this->noneDB->query($this->shardedDbName)->sort('score', 'desc')->offset(1)->limit(2)->update(['top' => true]); + $this->assertSame(['n' => 2], $result); + $this->assertSame([9, 10], $this->keys($this->noneDB->find($this->shardedDbName, ['top' => true]))); + + $deleted = $this->noneDB->query($this->shardedDbName)->sort('age')->limit(3)->delete(); + $this->assertSame(['n' => 3], $deleted); + $this->assertSame(range(3, 11), $this->keys($this->noneDB->find($this->shardedDbName, 0))); + } + + /** + * @test + */ + public function terminalMethodsOnShardedDb(): void + { + // first + $first = $this->noneDB->query($this->shardedDbName) + ->whereIn('city', ['Izmir']) + ->sort('age') + ->first(); + $this->assertSame($this->record(2), self::canon($first)); // Izmir: 2 (22), 6 (26), 10 (30) + + // last + $last = $this->noneDB->query($this->shardedDbName)->whereIn('city', ['Izmir'])->sort('age')->last(); + $this->assertSame($this->record(10), self::canon($last)); + + // count + $count = $this->noneDB->query($this->shardedDbName) + ->whereNotIn('department', ['IT']) + ->count(); + $this->assertSame(9, $count); + + // exists + $this->assertTrue($this->noneDB->query($this->shardedDbName)->search('User5')->exists()); + $this->assertFalse($this->noneDB->query($this->shardedDbName)->search('User99')->exists()); + + // sum (float, QUERY.md: the builder's sum() returns a float): 70 + 78 + 86 + $sum = $this->noneDB->query($this->shardedDbName) + ->whereIn('city', ['Istanbul']) + ->sum('score'); + $this->assertSame(234.0, $sum); + $this->assertSame(78.0, $this->noneDB->query($this->shardedDbName)->whereIn('city', ['Istanbul'])->avg('score')); + $this->assertSame(70, $this->noneDB->query($this->shardedDbName)->min('score')); + $this->assertSame(92, $this->noneDB->query($this->shardedDbName)->max('score')); + + // distinct, in order of first appearance + $cities = $this->noneDB->query($this->shardedDbName) + ->distinct('city'); + $this->assertSame(['Istanbul', 'Ankara', 'Izmir', 'Bursa'], $cities); + } + + /** + * QUERY.md: limit() and offset() apply to count(); limit(0) returns nothing + * + * @test + */ + public function countWithLimitOffsetAndLimitZeroOnShardedDb(): void + { + $q = function () { return $this->noneDB->query($this->shardedDbName)->whereIn('city', ['Istanbul', 'Ankara']); }; // 6 matches + $this->assertSame(6, $q()->count()); + $this->assertSame(4, $q()->offset(2)->count()); + $this->assertSame(3, $q()->limit(3)->count()); + $this->assertSame(1, $q()->offset(5)->limit(3)->count()); + $this->assertSame(0, $q()->offset(6)->count()); + $this->assertSame(8, $this->noneDB->query($this->shardedDbName)->offset(4)->count()); + + $this->assertSame([], $q()->limit(0)->get()); + $this->assertSame(0, $q()->limit(0)->count()); + $this->assertNull($q()->limit(0)->first()); + $this->assertFalse($q()->limit(0)->exists()); + $this->assertSame([4, 5], $this->keys($q()->offset(2)->limit(2)->get())); + } + + /** + * QUERY.md sort(): stable, reversed for desc; ties keep key order (also across shards) + * + * @test + */ + public function sortTiesKeepKeyOrderAcrossShards(): void + { + $asc = $this->noneDB->query($this->shardedDbName)->sort('city')->get(); + $this->assertSame([1, 5, 9, 3, 7, 11, 0, 4, 8, 2, 6, 10], $this->keys($asc)); + + $desc = $this->noneDB->query($this->shardedDbName)->sort('city', 'desc')->get(); + $this->assertSame([2, 6, 10, 0, 4, 8, 3, 7, 11, 1, 5, 9], $this->keys($desc)); + + // Top-k (offset + limit <= 1000) gives the same rows as the full sort + $this->assertSame([0, 4, 8, 3], $this->keys($this->noneDB->query($this->shardedDbName)->sort('city', 'desc')->offset(3)->limit(4)->get())); + + // A second sort key breaks ties: city asc, then age desc + $two = $this->noneDB->query($this->shardedDbName)->sort('city')->sort('age', 'desc')->get(); + $this->assertSame([9, 5, 1, 11, 7, 3, 8, 4, 0, 10, 6, 2], $this->keys($two)); + } + + /** + * @test + */ + public function paginationOnShardedDb(): void + { + $pageSize = 4; + + $page1 = $this->noneDB->query($this->shardedDbName) + ->sort('name') + ->limit($pageSize) + ->skip(0) + ->get(); + + $page2 = $this->noneDB->query($this->shardedDbName) + ->sort('name') + ->limit($pageSize) + ->skip($pageSize) + ->get(); + + $page3 = $this->noneDB->query($this->shardedDbName) + ->sort('name') + ->limit($pageSize) + ->skip($pageSize * 2) + ->get(); + + // Names are non-numeric strings: byte order ("User10" < "User2") + $this->assertSame(['User0', 'User1', 'User10', 'User11'], $this->names($page1)); + $this->assertSame(['User2', 'User3', 'User4', 'User5'], $this->names($page2)); + $this->assertSame(['User6', 'User7', 'User8', 'User9'], $this->names($page3)); + } + + /** + * Results on the sharded database equal an independent reference model, with and without field indexes + * (QUERY.md: an index only decides what is read, never what matches) + * + * @test + */ + public function queriesMatchReferenceModelWithAndWithoutFieldIndexes(): void + { + $records = $this->records(); + $all = function (callable $keep) use ($records): array { + $keys = []; + foreach ($records as $key => $r) { + if ($keep($r)) { + $keys[] = $key; + } + } + return $keys; + }; + $sorted = function (array $keys, string $field, bool $desc) use ($records): array { + usort($keys, function ($a, $b) use ($records, $field, $desc) { + $c = $records[$a][$field] <=> $records[$b][$field]; // all ints or non-numeric strings here + if ($desc) { + $c = -$c; + } + return $c !== 0 ? $c : $a <=> $b; // ties: key order + }); + return $keys; + }; + + $cases = [ + 'eq' => [function ($q) { return $q->where(['city' => 'Izmir']); }, $all(function ($r) { return $r['city'] === 'Izmir'; })], + 'eq-type' => [function ($q) { return $q->where(['age' => '25']); }, []], + 'range' => [function ($q) { return $q->where(['age' => ['$gte' => 23, '$lt' => 29]]); }, $all(function ($r) { return $r['age'] >= 23 && $r['age'] < 29; })], + 'range-numeric-string' => [function ($q) { return $q->where(['age' => ['$gt' => '29']]); }, $all(function ($r) { return $r['age'] > 29; })], + 'in' => [function ($q) { return $q->where(['score' => ['$in' => [72, 90, 999, '80']]]); }, $all(function ($r) { return in_array($r['score'], [72, 90, 999], true); })], + 'whereIn' => [function ($q) { return $q->whereIn('department', ['HR', 'Sales']); }, $all(function ($r) { return in_array($r['department'], ['HR', 'Sales'], true); })], + 'or' => [function ($q) { return $q->where(['city' => 'Bursa'])->orWhere(['age' => ['$lte' => 21]]); }, $all(function ($r) { return $r['city'] === 'Bursa' || $r['age'] <= 21; })], + 'and-groups' => [function ($q) { return $q->where(['age' => ['$gte' => 22]])->where(['age' => ['$lte' => 26]])->where(['active' => true]); }, $all(function ($r) { return $r['age'] >= 22 && $r['age'] <= 26 && $r['active'] === true; })], + 'between' => [function ($q) { return $q->between('score', 76, 84); }, $all(function ($r) { return $r['score'] >= 76 && $r['score'] <= 84; })], + 'ne' => [function ($q) { return $q->where(['city' => ['$ne' => 'Istanbul'], 'department' => ['$nin' => ['HR']]]); }, $all(function ($r) { return $r['city'] !== 'Istanbul' && $r['department'] !== 'HR'; })], + 'whereNot' => [function ($q) { return $q->whereNot(['department' => 'IT'])->where(['active' => true]); }, $all(function ($r) { return $r['department'] !== 'IT' && $r['active'] === true; })], + // Ascending list: the order of a key list vs. key order for sort ties isn't specified for the + // builder; "05" and 99 are no keys (QUERY.md key filters: canonical digit strings only) + 'key' => [function ($q) { return $q->where(['key' => [3, '7', 11, 99, '05']]); }, [3, 7, 11]], + 'key-and-field' => [function ($q) { return $q->where(['key' => [0, 1, 2, 3, 4, 5, 6]])->where(['city' => 'Istanbul']); }, [0, 4]], + ]; + + foreach ([false, true] as $indexed) { + if ($indexed) { + foreach (['city', 'age', 'department', 'score', 'active'] as $field) { + $created = $this->noneDB->createFieldIndex($this->shardedDbName, $field); + $this->assertTrue($created['success'], $field); + $this->assertSame(12, $created['indexed'], $field); + } + } + foreach ($cases as $name => [$build, $expected]) { + $label = $name . ($indexed ? ' (indexed)' : ''); + $this->assertSame($expected, $this->keys($build($this->noneDB->query($this->shardedDbName))->get()), $label); + $this->assertSame(count($expected), $build($this->noneDB->query($this->shardedDbName))->count(), $label . ' count'); + $this->assertSame($expected !== [], $build($this->noneDB->query($this->shardedDbName))->exists(), $label . ' exists'); + $first = $build($this->noneDB->query($this->shardedDbName))->first(); + $this->assertSame($expected[0] ?? null, $first['key'] ?? null, $label . ' first'); + + // Sorted by score desc, top-k (limit 2) and full + $byScore = $sorted($expected, 'score', true); + $this->assertSame($byScore, $this->keys($build($this->noneDB->query($this->shardedDbName))->sort('score', 'desc')->get()), $label . ' sort'); + $this->assertSame(array_slice($byScore, 1, 2), $this->keys($build($this->noneDB->query($this->shardedDbName))->sort('score', 'desc')->offset(1)->limit(2)->get()), $label . ' top-k'); + // Ties: city asc keeps key order + $byCity = $sorted($expected, 'city', false); + $this->assertSame($byCity, $this->keys($build($this->noneDB->query($this->shardedDbName))->sort('city')->get()), $label . ' sort ties'); + } + } + } +} diff --git a/tests/Integration/OperationsToolsTest.php b/tests/Integration/OperationsToolsTest.php new file mode 100644 index 0000000..cd18ce0 --- /dev/null +++ b/tests/Integration/OperationsToolsTest.php @@ -0,0 +1,411 @@ +setValue(null, $value); + } + + protected function tearDown(): void + { + $this->setStatic('syncFn', null); + $this->setStatic('renameFn', null); + $this->setStatic('flushFn', null); + $this->setStatic('osFamily', null); + parent::tearDown(); + } + + /** @test DURABILITY.md §6: the report says what the last write needed and did, in each mode */ + public function durabilityReportDescribesWhatWasDone(): void + { + $db = $this->db(); + $report = $db->getDurability(); + $this->assertSame(['mode', 'platform', 'lastOperation', 'lastSyncError', 'syncs'], array_keys($report)); + $this->assertSame('performance', $report['mode'], 'the default'); + $db->insert('dur', ['v' => 1]); + $op = $db->getDurability()['lastOperation']; + $this->assertSame(['committed', [], []], [$op['outcome'], $op['required'], $op['completed']], 'performance mode requires no sync'); + + $this->setStatic('osFamily', 'Linux'); // durable mode's rules, on every platform + $db = $this->db(['durability' => 'durable']); + $this->assertSame('durable', $db->getDurability()['mode']); + + $db->insert('dur', ['v' => 2]); // append to an existing file + $op = $db->getDurability()['lastOperation']; + $this->assertSame(['file'], $op['required']); + $this->assertSame(['file'], $op['completed']); + $this->assertSame('committed', $op['outcome']); + + $db->insert('dur2', ['v' => 1]); // creates files + $op = $db->getDurability()['lastOperation']; + $this->assertSame(['file', 'dir'], $op['required']); + $this->assertSame(['file', 'dir'], $op['completed']); + $this->assertSame('committed', $op['outcome']); + } + + /** @test DURABILITY.md §3: a dbDir the first write creates is synced into its parent like any new directory */ + public function aDatabaseDirectoryCreatedByTheFirstWriteIsSynced(): void + { + // dbDir is used by its canonical path (STORAGE_FORMAT.md §3): the temp directory may be a link (macOS) + $parent = realpath(sys_get_temp_dir()) . '/nonedb-dirsync-' . bin2hex(random_bytes(6)); + $dir = $parent . '/db/'; + mkdir($parent); + $this->setStatic('osFamily', 'Linux'); + $dirs = []; + $this->setStatic('syncFn', function ($handle, string $kind, $path) use (&$dirs) { + if ($kind === 'dir') { + $dirs[] = (string)$path; + } + return true; + }); + try { + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dir, 'durability' => 'durable']); + $this->assertDirectoryDoesNotExist($dir); + $db->insert('later', ['v' => 1]); + $this->assertContains($parent, $dirs, 'the new dbDir\'s entry in its parent'); + $this->assertContains(rtrim($dir, '/'), $dirs, 'the store files and the database directory in dbDir'); + $op = $db->getDurability()['lastOperation']; + $this->assertSame(['committed', ['file', 'dir'], ['file', 'dir']], [$op['outcome'], $op['required'], $op['completed']]); + } finally { + $this->setStatic('syncFn', null); + \noneDB::clearStaticCache(); + $this->removeDirectoryTree($parent); + } + } + + /** @test DURABILITY.md §2: a failed sync fails the write (never success), and is never carried over */ + public function aFailedSyncIsReportedAndNeverCarriedOver(): void + { + $this->setStatic('osFamily', 'Linux'); + $db = $this->db(['durability' => 'durable']); + $db->insert('sf', ['v' => 1]); + $this->assertSame('committed', $db->getDurability()['lastOperation']['outcome']); + + // Directory syncs fail (some file systems answer EINVAL) on an operation that creates files + $this->setStatic('syncFn', function ($handle, string $kind) { + return $kind !== 'dir'; + }); + try { + $db->insert('sf2', ['v' => 1]); + $this->fail('a failed directory sync must fail the write'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::DURABILITY_FAILED, $e->getCode(), $e->getMessage()); + } + $report = $db->getDurability(); + $this->assertSame('rolled_back', $report['lastOperation']['outcome']); + $this->assertSame('dir', $report['lastSyncError']['op']); + $this->assertGreaterThan(0, $report['syncs']['failed']); + + // A failed file sync of an append: the write is refused and its bytes are undone + $failed = 0; + $this->setStatic('syncFn', function ($handle, string $kind) use (&$failed) { + return !($kind === 'file' && ++$failed === 1); + }); + try { + $db->insert('sf', ['v' => 2]); + $this->fail('a failed fsync of the data must fail the write'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('fsync', $e->getMessage()); + $this->assertSame(\noneDB::DURABILITY_FAILED, $e->getCode()); + } + $this->setStatic('syncFn', null); + $this->assertSame([1], array_column($this->db()->find('sf', 0), 'v')); + $this->assertSame('rolled_back', $db->getDurability()['lastOperation']['outcome']); + + $db->insert('sf', ['v' => 3]); + $op = $db->getDurability()['lastOperation']; + $this->assertSame('committed', $op['outcome'], 'success again once syncs work'); + $this->assertSame($op['required'], $op['completed']); + } + + private function nonEmptyFileSizes(string $name): array + { + clearstatcache(); + $sizes = []; + $home = $this->dbPath($name, 'home'); + foreach ($this->dbFiles($name) as $file) { + if (filesize($home . $file) > 0 && $file !== 'LOCK') { + $sizes[$file] = filesize($home . $file); + } + } + return $sizes; + } + + /** @test */ + public function aFailedFlushFailsTheWriteAndIsUndone(): void + { + // fflush() of an append fails (disk full, I/O error): the write fails and its bytes are cut back, + // with fsync off too + foreach ([false, true] as $buffered) { + $this->cleanTestDirectory(); + $config = ['shardingEnabled' => false, 'bufferEnabled' => $buffered]; + $db = $this->db($config); + $db->insert('fl', ['v' => 'a']); + if ($buffered) { + $db->flush('fl'); + } + $sizes = $this->nonEmptyFileSizes('fl'); + $this->assertArrayHasKey('generations/1/segments/main.jsonl', $sizes, 'set up'); + $this->setStatic('flushFn', function ($handle, string $path) { return false; }); + try { + $db->insert('fl', ['v' => 'b']); + $this->fail('a failed fflush() must fail the write (an I/O failure: RuntimeException)'); + } catch (\RuntimeException $e) { + $this->assertNotSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode(), $e->getMessage()); + } + $this->assertSame('rolled_back', $db->getDurability()['lastOperation']['outcome']); + $this->setStatic('flushFn', null); + $this->assertSame($sizes, $this->nonEmptyFileSizes('fl'), 'nothing left behind (an append may leave an empty buffer file)'); + $this->assertSame(['a'], array_column($this->db($config)->find('fl', 0), 'v')); + + // The bytes can't be cut back (by the append, nor by the rollback): the outcome is reported as + // unknown, never as a plain failure that leaves them to be read as a record + $this->setStatic('flushFn', function ($handle, string $path) { return false; }); + $this->setFaultHook(function ($point) { + if ($point === 'append:undo' || $point === 'rollback:truncate') { + throw new \RuntimeException('disk gone'); + } + }); + try { + $db->insert('fl', ['v' => 'c']); + $this->fail('must throw'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode(), ($buffered ? 'buffered: ' : '') . $e->getMessage()); + } + $this->setFaultHook(null); + $this->setStatic('flushFn', null); + \noneDB::clearStaticCache(); + $db = $this->db($config); + // Nothing was cut: the complete line is recovered (roll forward), the same for every reader + $this->assertSame(['a', 'c'], array_column($db->find('fl', 0), 'v')); + $db->insert('fl', ['v' => 'd']); + $this->assertSame(['a', 'c', 'd'], array_column($db->find('fl', 0), 'v')); + } + } + + /** @test */ + public function quarantineRecordsSayWhetherTheyWouldSurviveAPowerLoss(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('pl', ['v' => 'a']); + clearstatcache(); + $offset = filesize($this->dbPath('pl', 'data')); + file_put_contents($this->dbPath('pl', 'data'), '{"torn', FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('pl', ['v' => 'b']); + $log = $db->getRecoveryLog('pl'); + $this->assertCount(1, $log); + $entry = $log[0]; + $this->assertSame('cut-tail', $entry['action']); + $this->assertSame('{"torn', file_get_contents($this->dbPath('pl', 'quarantine') . $entry['quarantine']), 'the cut bytes are kept'); + // recovery/q.{seg}.{off}.{len}.{sha256}.bin (STORAGE_FORMAT.md §7.2) + $this->assertSame('q.main.' . $offset . '.6.' . hash('sha256', '{"torn') . '.bin', $entry['quarantine']); + $this->assertSame(['a', 'b'], array_column($db->find('pl', 0), 'v')); + $this->assertTrue($entry['bytesVerified']); + $this->assertSame(PHP_OS_FAMILY === 'Windows' ? 'unknown' : true, $entry['powerLossDurable']); + } + + /** @test */ + public function integrityCheckReportsAndWritesNothing(): void + { + $db = $this->db(['shardingEnabled' => false, 'bufferEnabled' => true]); + $db->createFieldIndex('ic', 'c'); + $db->insert('ic', [['c' => 'x'], ['c' => 'y']]); + $db->flush('ic'); + $clean = $db->checkIntegrity('ic'); + $this->assertTrue($clean['ok']); + $this->assertSame(2, $clean['lines']); + + // A damaged line, an unterminated tail, a stale field index and records still buffered + $data = $this->dbPath('ic', 'data'); + clearstatcache(); + $size = filesize($data); + file_put_contents($data, "not json\n", FILE_APPEND); + file_put_contents($data, '{"torn', FILE_APPEND); + // The field index's header still fits the data, its buckets don't: "w" (no record has it), no "x" + $fx = $this->dbPath('ic', 'fx', 'c'); + [$header, $buckets] = $this->readFieldIndexFile($fx); + $buckets['sw'] = 1; + unset($buckets['sx']); + $this->writeFieldIndexFile($fx, $header, $buckets); + $db->insert('ic', ['c' => 'z']); // buffered + \noneDB::clearStaticCache(); + + $snapshot = function () { + clearstatcache(); + $files = []; + $walk = function (string $dir) use (&$walk, &$files): void { + foreach (scandir($dir) as $f) { + if ($f === '.' || $f === '..') { + continue; + } + if (is_dir($dir . $f)) { + $files[$dir . $f . '/'] = 'dir'; + $walk($dir . $f . '/'); + } elseif ($f[0] !== '.') { + // Docker Desktop's file sharing gives files new inode numbers by itself (seen with nothing + // run between two snapshots): there the content stands in for the inode + $files[$dir . $f] = [filesize($dir . $f), filemtime($dir . $f), + file_exists('/.dockerenv') ? sha1_file($dir . $f) : fileinode($dir . $f)]; + } + } + }; + $walk(rtrim($this->testDbDir, '/') . '/'); + unset($files[$this->dbPath('ic', 'lock')]); + return $files; + }; + $before = $snapshot(); + $report = $this->db(['shardingEnabled' => false])->checkIntegrity('ic'); + $this->assertSame($before, $snapshot(), 'read-only: nothing created, changed, cut or flushed'); + $this->assertFalse($report['ok']); + // Derived from the bytes written above: "not json\n" (9 bytes) at the old end, '{"torn' (6 bytes) after it + $this->assertSame([['main', $size, 9]], $report['badLines']); + $this->assertSame([['segment' => 'main', 'offset' => $size + 9, 'length' => 6]], $report['tornTail']); + $this->assertSame([3, 2], [$report['lines'], $report['records']], 'complete lines: 2 records and the damaged one'); + $this->assertSame([['main', 'field c']], array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches']), + 'only the field index disagrees: the key index replays the data'); + $this->assertSame(1, $report['bufferedRecords']); + $this->assertFalse($report['pendingRecovery']); + $this->assertSame([], $report['quarantineFiles']); + $this->assertSame([], $report['recoveryLog']); + + // rebuildIndex() repairs the index; the damaged line and the torn tail stay until a write / compaction + $db = $this->db(['shardingEnabled' => false]); + $db->rebuildIndex('ic'); + $after = $db->checkIntegrity('ic'); + $this->assertSame([], $after['indexMismatches']); + $this->assertSame(0, $after['bufferedRecords'], 'rebuildIndex() flushed the buffer first'); + $this->assertSame(['x', 'y', 'z'], array_column($db->find('ic', 0), 'c')); + } + + /** @test */ + public function anInterruptedSwitchToShardsRemovesTheOldFile(): void + { + // A-6: a switch to shards that died after its commit point left the old data forever (double size). The + // commit point is PUBLISH: CURRENT names the new generation (STORAGE_FORMAT.md §5.2) + $db = $this->db(['shardSize' => 4]); + $db->insert('a6', array_map(function ($n) { return ['n' => $n]; }, range(0, 2))); + $old = $this->dbPath('a6', 'data'); + $this->assertStringContainsString('/generations/1/', $old, 'set up'); + \noneDB::clearStaticCache(); + $this->runChild(' + $db = ' . $this->childDb(['shardSize' => 4]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === "generation:published") { exit(0); } }); + $db->insert("a6", ["n" => 3]); + '); + $this->assertFileExists($old, 'old file left by the crash'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + $this->assertSame(range(0, 3), array_column($db->find('a6', 0), 'n')); + $this->assertFileDoesNotExist($old, 'recovered: old file removed'); + $this->assertDirectoryDoesNotExist(dirname($old, 2), 'with its whole generation'); + } + + /** @test */ + public function flushAllBuffersFindsInterruptedFlushes(): void + { + // A-8: the file name pattern didn't match {buffer}.flushing.{key} + $db = $this->db(['bufferEnabled' => true]); + $db->insert('a8', ['v' => 1]); + $db->flush('a8'); + $db->insert('a8', ['v' => 2]); + $buffer = $this->dbPath('a8', 'buffer'); + rename($buffer, $this->dbPath('a8', 'flushing', 1)); + \noneDB::clearStaticCache(); + $result = $this->db()->flushAllBuffers(); + $this->assertSame(1, $result['databases']); + $this->assertSame(1, $result['flushed']); + $this->assertSame([], glob(dirname($buffer) . '/*')); + $this->assertSame([[1, 0], [2, 1]], array_map(function ($r) { return [$r['v'], $r['key']]; }, $this->db()->find('a8', 0))); + } + + /** @test */ + public function readsCreateNoDirectoryAndNoFile(): void + { + // A-11: find() on a database that doesn't exist must create neither dbDir nor any file in it (a store + // is created by the first write, STORAGE_FORMAT.md §3.2) + $dir = $this->testDbDir . 'not-yet/'; + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dir]); + $this->assertSame([], $db->find('x', 0)); + $this->assertSame(0, $db->count('x')); + $this->assertSame([], $db->query('x')->get()); + $this->assertFalse($db->exists('x', ['a' => 1])); + $this->assertSame([], $db->getDBs()); + $this->assertDirectoryDoesNotExist($dir); + $db->insert('x', ['a' => 1]); + $this->assertDirectoryExists($dir, 'a write creates it'); + $this->assertCount(1, (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dir]))->find('x', 0)); + \noneDB::clearStaticCache(); // closes the kept lock handles + $this->removeDirectoryTree($dir); + } + + /** @test */ + public function distancesDoNotDependOnThePrecisionSetting(): void + { + // T2-8: the distance memo was keyed by (string) floats: other points at low precision shared a key + $before = ini_get('precision'); + try { + $db = $this->db(); + $db->insert('dist', [['loc' => ['type' => 'Point', 'coordinates' => [29.0, 41.0]]]]); + ini_set('precision', '17'); + $a = $db->nearest('dist', 'loc', 29.0001, 41.0, 1)[0]['_distance']; + ini_set('precision', '3'); + $b = $db->nearest('dist', 'loc', 29.0002, 41.0, 1)[0]['_distance']; + ini_set('precision', '17'); + $this->assertEqualsWithDelta(8.42, $a, 0.1); + $this->assertEqualsWithDelta(16.84, $b, 0.1, 'not the cached distance of the other point'); + } finally { + ini_set('precision', (string)$before); + } + } + + /** @test */ + public function renameRetriesTransientFailures(): void + { + // S-6: on Windows a rename fails while another process has the target open; it is retried + $calls = 0; + $this->setStatic('renameFn', function ($from, $to) use (&$calls) { + return ++$calls > 3 ? rename($from, $to) : false; + }); + $db = $this->db(); + $db->insert('rr', ['v' => 1]); + $this->assertGreaterThan(3, $calls); + $this->assertSame([1], array_column($db->find('rr', 0), 'v')); + + // Gives up: an index must be written (inserts only append, the key index is a checkpoint) + $this->setStatic('renameFn', function () { return false; }); + try { + $db->createFieldIndex('rr', 'v'); + $this->fail('gives up'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('index', $e->getMessage()); + } + $this->setStatic('renameFn', null); + $temp = []; + $walk = function (string $dir) use (&$walk, &$temp): void { + foreach (scandir($dir) as $f) { + if ($f === '.' || $f === '..') { + continue; + } + if (is_dir($dir . $f)) { + $walk($dir . $f . '/'); + } elseif (strpos($f, '.tmp') !== false) { + $temp[] = $dir . $f; + } + } + }; + $walk(rtrim($this->testDbDir, '/') . '/'); + $this->assertSame([], $temp, 'temp files removed'); + } +} diff --git a/tests/Regression/PublicApiCoverageTest.php b/tests/Integration/PublicApiCoverageTest.php similarity index 55% rename from tests/Regression/PublicApiCoverageTest.php rename to tests/Integration/PublicApiCoverageTest.php index cca8b81..fc5c00d 100644 --- a/tests/Regression/PublicApiCoverageTest.php +++ b/tests/Integration/PublicApiCoverageTest.php @@ -1,9 +1,11 @@ db(); - $this->assertFalse($db->isBufferingEnabled(), 'off by default since 3.2'); + $this->assertFalse($db->isBufferingEnabled(), 'off by default'); $db->enableBuffering(true); $this->assertTrue($db->isBufferingEnabled()); $db->setBufferSizeLimit(10); // clamped to 1KB @@ -31,86 +33,49 @@ public function bufferingApi(): void for ($i = 5; $i < 12; $i++) { $db->insert('bf', ['i' => $i]); // the 10th record triggers a flush } - $this->assertLessThan(10, $db->getBufferInfo('bf')['buffers']['main']['records']); + $waiting = $db->getBufferInfo('bf')['buffers']['main']['records']; + $this->assertLessThan(10, $waiting); + $this->assertGreaterThan(0, $waiting, '12 records, a flush at 10: some wait again'); + $this->assertSame(12 - $waiting, count($this->dataLines($this->dbPath('bf', 'data'))), 'the rest is in the data file'); $db->insert('other', ['x' => 1]); $r = $db->flushAllBuffers(); $this->assertSame(2, $r['databases']); + $this->assertSame($waiting + 1, $r['flushed'], 'every buffered record of both databases'); $this->assertFalse($db->getBufferInfo('bf')['buffers']['main']['exists']); $this->assertSame(range(0, 11), array_column($db->find('bf', 0), 'i')); $this->assertSame(['success' => true, 'flushed' => 0, 'error' => null], $db->flush('bf')); } - /** @test K-08 */ - public function interruptedFlushIsRecovered(): void - { - $db = $this->db(); - $db->insert('fl', ['i' => 0]); - $buffer = $this->dbBasePath('fl') . '.nonedb.buffer'; - // Left by a pre-3.2 version (".flushing" without start key; those versions had no lock file) - file_put_contents($buffer . '.flushing', json_encode(['i' => 1]) . "\n" . json_encode(['i' => 2]) . "\n"); - file_put_contents($buffer, json_encode(['i' => 3]) . "\n"); - $this->removeLockFile('fl'); - $this->assertSame([0, 1, 2, 3], array_column($this->db()->find('fl', 0), 'i')); - $this->assertNotContains('b', $this->lockState('fl')['flags'], 'buffer flag cleared after the flush'); - $this->assertFileDoesNotExist($buffer); - $this->assertFileDoesNotExist($buffer . '.flushing'); - } - - /** @test K-08 pre-3.2 shard buffers (already counted in the meta) */ - public function legacyShardBuffersAreFlushed(): void - { - $db = $this->db(['shardSize' => 5]); - $rows = []; - for ($i = 0; $i < 7; $i++) { - $rows[] = ['i' => $i]; - } - $db->insert('ls', $rows); - // An old version had buffered two records for shard 1 and counted them in the meta - $metaPath = $this->dbBasePath('ls') . '.nonedb.meta'; - $meta = json_decode(file_get_contents($metaPath), true); - $meta['nextKey'] += 2; - $meta['totalRecords'] += 2; - $meta['shards'][1]['count'] += 2; - file_put_contents($metaPath, json_encode($meta)); - file_put_contents($this->dbBasePath('ls') . '_s1.nonedb.buffer', json_encode(['i' => 7]) . "\n" . json_encode(['i' => 8]) . "\n"); - - $this->removeLockFile('ls'); // pre-3.2 versions had no lock file - $db = $this->db(); - $this->assertSame(range(0, 8), array_column($db->find('ls', 0), 'i')); - $this->assertSame(range(0, 8), array_column($db->find('ls', 0), 'key')); - $this->assertSame(9, $db->count('ls')); - $db->insert('ls', ['i' => 9]); - $this->assertSame(9, $db->find('ls', ['i' => 9])[0]['key']); - } - /** @test */ public function indexApi(): void { $db = $this->db(); - $db->enableIndexing(false); - $this->assertFalse($db->isIndexingEnabled()); - $db->enableIndexing(true); - $this->assertTrue($db->isIndexingEnabled()); - $this->assertNull($db->getIndexInfo('ix')); - $this->assertFalse($db->rebuildIndex('ix')['success']); + $this->assertSame(['success' => false, 'error' => 'Database does not exist'], $db->rebuildIndex('ix')); + // No database created: no FORMAT, at most an empty location (only LOCK, STORAGE_FORMAT.md §8.3) + $this->assertSame([], array_values(array_diff($this->dbFiles('ix'), ['LOCK'])), 'no database created'); + $this->assertSame([], $db->getDBs()); $db->insert('ix', [['c' => 'a'], ['c' => 'b'], ['c' => 'a']]); $db->delete('ix', ['key' => 1]); $info = $db->getIndexInfo('ix'); $this->assertTrue($info['exists']); + $this->assertSame(1, $info['version'], 'the key-index checkpoint version (STORAGE_FORMAT.md §6.4: {"kind":"kix","v":1})'); $this->assertSame(2, $info['totalRecords']); $this->assertFalse($info['sharded']); + $this->assertSame($this->dbPath('ix', 'kix'), $info['path']); - unlink($this->dbBasePath('ix') . '.nonedb.jidx'); + unlink($this->dbPath('ix', 'kix')); $r = $db->rebuildIndex('ix'); $this->assertTrue($r['success']); $this->assertSame(2, $r['totalRecords']); + $this->assertMatchesRegularExpression('/^\d+(\.\d+)?ms$/', $r['time']); + $this->assertFileExists($this->dbPath('ix', 'kix'), 'rebuilt as a checkpoint'); $this->assertSame([0, 2], array_column($db->find('ix', 0), 'key')); $this->assertNull($db->getKeysFromFieldIndex('ix', 'c', 'a'), 'no index yet'); $db->createFieldIndex('ix', 'c'); $this->assertSame([0, 2], $db->getKeysFromFieldIndex('ix', 'c', 'a')); - $this->assertSame(['success' => true, 'indexed' => 2, 'values' => 1, 'error' => null], $db->rebuildFieldIndex('ix', 'c')); + $this->assertSame(['success' => true, 'indexed' => 2, 'error' => null], $db->rebuildFieldIndex('ix', 'c')); } /** @test */ @@ -119,12 +84,13 @@ public function spatialHelpers(): void $db = $this->db(); $this->assertEqualsWithDelta(0, $db->haversineDistance(29, 41, 29, 41), 0.0001); $this->assertEqualsWithDelta(111194.9, $db->haversineDistance(0, 0, 1, 0), 1); - $db->clearDistanceCache(); - $db->clearCentroidCache(); + // Removed in noneDB 4 (they had no effect): the API no longer has them + foreach (['clearDistanceCache', 'clearCentroidCache', 'flushSpatialIndexes', 'upgradeIndexes', 'enableIndexing', 'isIndexingEnabled'] as $removed) { + $this->assertFalse(method_exists(\noneDB::class, $removed), $removed); + } $db->insert('sp', ['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]]); $db->createSpatialIndex('sp', 'loc'); - $this->assertSame(0, $db->flushSpatialIndexes(), 'nothing pending outside a write'); $r = $db->rebuildSpatialIndex('sp', 'loc'); $this->assertTrue($r['success']); $this->assertSame(1, $r['indexed']); diff --git a/tests/Integration/QueryDocumentationTest.php b/tests/Integration/QueryDocumentationTest.php new file mode 100644 index 0000000..e4cc613 --- /dev/null +++ b/tests/Integration/QueryDocumentationTest.php @@ -0,0 +1,1772 @@ +now = time(); + $day = 86400; + $this->people = [ + ['name' => 'John', 'email' => 'john@gmail.com', 'age' => 25, 'role' => 'admin', 'status' => 'active', 'salary' => 75000, 'department' => 'Engineering', 'created_at' => $now - 10 * $day, 'tags' => ['developer', 'senior']], + ['name' => 'Jane', 'email' => 'jane@yahoo.com', 'age' => 30, 'role' => 'moderator', 'status' => 'active', 'salary' => 65000, 'department' => 'Design', 'created_at' => $now - 5 * $day, 'tags' => ['designer']], + ['name' => 'Bob', 'email' => 'bob@gmail.com', 'age' => 35, 'role' => 'user', 'status' => 'inactive', 'salary' => 55000, 'department' => 'Engineering', 'created_at' => $now - 2 * $day], + ['name' => 'Alice', 'email' => 'alice@company.com', 'age' => 28, 'role' => 'editor', 'status' => 'active', 'salary' => 60000, 'department' => 'Marketing', 'created_at' => $now - $day, 'tags' => ['content', 'marketing']], + ['name' => 'Charlie', 'email' => 'charlie@gmail.com', 'age' => 45, 'role' => 'guest', 'status' => 'pending', 'salary' => 0, 'department' => 'Guest', 'created_at' => $now], + ['name' => 'Diana', 'email' => 'diana@test.com', 'age' => 22, 'role' => 'user', 'status' => 'active', 'salary' => 45000, 'department' => 'Support', 'created_at' => $now - 15 * $day, 'tags' => ['support']], + ['name' => 'Eve', 'email' => 'eve@gmail.com', 'age' => 33, 'role' => 'admin', 'status' => 'active', 'salary' => 80000, 'department' => 'Engineering', 'created_at' => $now - 20 * $day, 'tags' => ['developer', 'lead']], + ]; + $this->assertSame(['n' => 7], $this->noneDB->insert($this->testDbName, $this->people)); + } + + // ========== HELPERS ========== + + private function q(?string $db = null): \noneDBQuery + { + return $this->noneDB->query($db ?? $this->testDbName); + } + + /** Names of the result rows, in result order */ + private function names(array $rows): array + { + return array_column($rows, 'name'); + } + + /** A fixture record as find() returns it (with its key) */ + private function person(int $key): array + { + return $this->people[$key] + ['key' => $key]; + } + + /** Records compared field by field (the position of "key" inside a record is not documented) */ + private function assertSameRecords(array $expected, array $actual, string $message = ''): void + { + $normalize = function (array $rows) { + return array_map(function ($row) { ksort($row); return $row; }, array_values($rows)); + }; + $this->assertSame($normalize($expected), $normalize($actual), $message); + } + + // ========== QUICK START ========== + + /** + * Quick Start: where(active) + sort(created_at desc) + limit(10): the active users, newest first + */ + public function testQuickStartSimpleQuery(): void + { + $users = $this->q() + ->where(['status' => 'active']) + ->sort('created_at', 'desc') + ->limit(10) + ->get(); + + // active: John(-10d), Jane(-5d), Alice(-1d), Diana(-15d), Eve(-20d) + $this->assertSame(['Alice', 'Jane', 'John', 'Diana', 'Eve'], $this->names($users)); + $this->assertSameRecords([$this->person(3), $this->person(1), $this->person(0), $this->person(5), $this->person(6)], $users); + } + + /** + * Quick Start: price range + $in + stock > 0, sorted by rating desc + */ + public function testQuickStartAdvancedOperatorQuery(): void + { + $this->noneDB->insert('products', [ + ['name' => 'Laptop', 'price' => 999, 'category' => 'electronics', 'stock' => 5, 'rating' => 4.5], // price > 500 + ['name' => 'Mouse', 'price' => 100, 'category' => 'gadgets', 'stock' => 3, 'rating' => 4.1], // boundary 100 + ['name' => 'Cable', 'price' => 500, 'category' => 'electronics', 'stock' => 0, 'rating' => 4.9], // no stock + ['name' => 'Watch', 'price' => 500, 'category' => 'gadgets', 'stock' => 2, 'rating' => 4.7], // boundary 500 + ['name' => 'Lamp', 'price' => 200, 'category' => 'home', 'stock' => 9, 'rating' => 5.0], // category + ['name' => 'Phone', 'price' => '300', 'category' => 'electronics', 'stock' => 1, 'rating' => 3.9], // numeric string price + ]); + + $products = $this->q('products') + ->where([ + 'price' => ['$gte' => 100, '$lte' => 500], + 'category' => ['$in' => ['electronics', 'gadgets']], + 'stock' => ['$gt' => 0] + ]) + ->sort('rating', 'desc') + ->get(); + + $this->assertSame(['Watch', 'Mouse', 'Phone'], $this->names($products)); + } + + // ========== COMPARISON OPERATORS ========== + + /** + * $gte 18 ("Find adults") includes 18, $gt 18 excludes it + */ + public function testGreaterThanOperator(): void + { + $this->noneDB->insert($this->testDbName, [['name' => 'Kid', 'age' => 17], ['name' => 'Teen', 'age' => 18]]); + + $adults = $this->q()->where(['age' => ['$gte' => 18]])->get(); + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'Eve', 'Teen'], $this->names($adults)); + + $over18 = $this->q()->where(['age' => ['$gt' => 18]])->get(); + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'Eve'], $this->names($over18)); + } + + /** + * $lt ("Find products under $100"): strictly less + */ + public function testLessThanOperator(): void + { + $this->noneDB->insert('products', [ + ['name' => 'A', 'price' => 99.99], + ['name' => 'B', 'price' => 100], + ['name' => 'C', 'price' => 150], + ['name' => 'D', 'price' => 5], + ]); + + $affordable = $this->q('products')->where(['price' => ['$lt' => 100]])->get(); + $this->assertSame(['A', 'D'], $this->names($affordable)); + } + + /** + * Range query: both operators apply, boundaries inclusive, numeric strings compare numerically + */ + public function testRangeQuery(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Kid', 'age' => 17], + ['name' => 'Teen', 'age' => 18], + ['name' => 'Senior', 'age' => 65], + ['name' => 'Retired', 'age' => 66], + ['name' => 'StrAge', 'age' => '40'], + ]); + + $workingAge = $this->q()->where(['age' => ['$gte' => 18, '$lte' => 65]])->get(); + + $this->assertSame( + ['John', 'Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'Eve', 'Teen', 'Senior', 'StrAge'], + $this->names($workingAge) + ); + } + + /** + * where() called twice on the same field: both conditions apply (QUERY.md where()) + */ + public function testSeveralWhereCallsOnTheSameFieldAreAnded(): void + { + $this->noneDB->insert($this->testDbName, [['name' => 'Kid', 'age' => 17], ['name' => 'Retired', 'age' => 65]]); + + $results = $this->q()->where(['age' => ['$gte' => 18]])->where(['age' => ['$lt' => 65]])->get(); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'Eve'], $this->names($results)); + } + + /** + * $ne: everything but the value; a record without the field also matches + */ + public function testNotEqualOperator(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoRole']); + + $nonAdmins = $this->q()->where(['role' => ['$ne' => 'admin']])->get(); + + $this->assertSame(['Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'NoRole'], $this->names($nonAdmins)); + } + + /** + * $eq: "Explicit equality (same as simple value)" + */ + public function testExplicitEqualityMatchesSimpleValue(): void + { + $explicit = $this->q()->where(['status' => ['$eq' => 'active']])->get(); + $simple = $this->q()->where(['status' => 'active'])->get(); + + $this->assertSame(['John', 'Jane', 'Alice', 'Diana', 'Eve'], $this->names($explicit)); + $this->assertSame($simple, $explicit); + } + + /** + * $in: the roles in the list, in key order + */ + public function testInOperator(): void + { + $staff = $this->q()->where(['role' => ['$in' => ['admin', 'moderator', 'editor']]])->get(); + + $this->assertSame(['John', 'Jane', 'Alice', 'Eve'], $this->names($staff)); + } + + /** + * $nin: the statuses not in the list + */ + public function testNotInOperator(): void + { + $results = $this->q()->where(['status' => ['$nin' => ['inactive', 'pending']]])->get(); + + $this->assertSame(['John', 'Jane', 'Alice', 'Diana', 'Eve'], $this->names($results)); + } + + /** + * $exists true / false split the records by whether the field is present + */ + public function testExistsOperator(): void + { + $withTags = $this->q()->where(['tags' => ['$exists' => true]])->get(); + $this->assertSame(['John', 'Jane', 'Alice', 'Diana', 'Eve'], $this->names($withTags)); + + $noTags = $this->q()->where(['tags' => ['$exists' => false]])->get(); + $this->assertSame(['Bob', 'Charlie'], $this->names($noTags)); + + // QUERY.md: "Find records without phone field" - no fixture record has one + $this->assertCount(7, $this->q()->where(['phone' => ['$exists' => false]])->get()); + $this->assertSame([], $this->q()->where(['phone' => ['$exists' => true]])->get()); + } + + /** + * $like 'john': case-insensitive contains + */ + public function testLikeContains(): void + { + $this->noneDB->insert($this->testDbName, [['name' => 'JOHNNY'], ['name' => 'Jon']]); + + $johns = $this->q()->where(['name' => ['$like' => 'john']])->get(); + + $this->assertSame(['John', 'JOHNNY'], $this->names($johns)); + } + + /** + * $like '^M' / '^J': starts with (case-insensitive), not "contains" + */ + public function testLikeStartsWith(): void + { + $this->noneDB->insert($this->testDbName, [['name' => 'Mary'], ['name' => 'Maj'], ['name' => 'Tom']]); + + $this->assertSame(['Mary', 'Maj'], $this->names($this->q()->where(['name' => ['$like' => '^M']])->get())); + $this->assertSame(['John', 'Jane'], $this->names($this->q()->where(['name' => ['$like' => '^J']])->get())); + $this->assertSame(['John', 'Jane'], $this->names($this->q()->where(['name' => ['$like' => '^j']])->get())); + } + + /** + * $like 'gmail.com$': ends with; "." is literal + */ + public function testLikeEndsWith(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Dotless', 'email' => 'x@gmailxcom'], + ['name' => 'Turkey', 'email' => 'x@gmail.com.tr'], + ['name' => 'Upper', 'email' => 'X@GMAIL.COM'], + ]); + + $gmails = $this->q()->where(['email' => ['$like' => 'gmail.com$']])->get(); + + $this->assertSame(['John', 'Bob', 'Charlie', 'Eve', 'Upper'], $this->names($gmails)); + } + + /** + * Unicode case folding of $like / like() / notLike(): straße = STRASSE, İstanbul = istanbul, + * Turkish dotless ı and I don't match + */ + public function testLikeUnicodeCaseFolding(): void + { + if (!function_exists('mb_convert_case')) { + $this->markTestSkipped('needs the mbstring extension (without it the PCRE /iu rules apply)'); + } + $this->noneDB->insert('cities', [ + ['name' => 'straße'], + ['name' => 'İstanbul'], + ['name' => 'ılık'], + ['name' => 'Ilık'], + ]); + + $this->assertSame(['straße'], $this->names($this->q('cities')->where(['name' => ['$like' => 'STRASSE']])->get())); + $this->assertSame(['straße'], $this->names($this->q('cities')->like('name', 'strasse')->get())); + $this->assertSame(['İstanbul'], $this->names($this->q('cities')->like('name', 'istanbul')->get())); + // ı (dotless) and I don't fold to each other: "ılık" and "Ilık" are different words here + $this->assertSame(['ılık'], $this->names($this->q('cities')->like('name', '^ı')->get())); + // İ folds to i (+ a dropped combining dot), I folds to i: both start with "i" + $this->assertSame(['İstanbul', 'Ilık'], $this->names($this->q('cities')->like('name', '^i')->get())); + $this->assertSame(['straße', 'İstanbul', 'Ilık'], $this->names($this->q('cities')->notLike('name', '^ı')->get())); + } + + /** + * $regex '^[A-D]': case-insensitive regular expression without delimiters + */ + public function testRegexOperator(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'bart']); + + $results = $this->q()->where(['name' => ['$regex' => '^[A-D]']])->get(); + + $this->assertSame(['Bob', 'Alice', 'Charlie', 'Diana', 'bart'], $this->names($results)); + } + + /** + * The e-mail validation example of QUERY.md + */ + public function testRegexEmailValidationExample(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Bad1', 'email' => 'not-an-email'], + ['name' => 'Bad2', 'email' => 'a@b'], + ['name' => 'Good', 'email' => 'first.last+tag@sub.example.org'], + ]); + + $valid = $this->q()->where(['email' => ['$regex' => '^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$']])->get(); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Charlie', 'Diana', 'Eve', 'Good'], $this->names($valid)); + } + + /** + * $contains on an array field: element match + */ + public function testContainsArray(): void + { + $developers = $this->q()->where(['tags' => ['$contains' => 'developer']])->get(); + + $this->assertSame(['John', 'Eve'], $this->names($developers)); + } + + /** + * $contains on a string field: substring, case-sensitive ("use $like for case-insensitive matching") + */ + public function testContainsStringIsCaseSensitive(): void + { + $this->noneDB->insert('articles', [ + ['title' => 'New Technology Trends'], + ['title' => 'technology today'], + ['title' => 'TECHNOLOGY'], + ['title' => 'Cooking'], + ]); + + $tech = $this->q('articles')->where(['title' => ['$contains' => 'Technology']])->get(); + $this->assertSame(['New Technology Trends'], array_column($tech, 'title')); + + $insensitive = $this->q('articles')->where(['title' => ['$like' => 'Technology']])->get(); + $this->assertSame(['New Technology Trends', 'technology today', 'TECHNOLOGY'], array_column($insensitive, 'title')); + } + + /** + * Operators and plain values in one where(): all conditions apply + */ + public function testMixedOperatorsWithEquality(): void + { + $activeAdmins = $this->q() + ->where(['role' => 'admin', 'status' => 'active', 'salary' => ['$gt' => 50000]]) + ->get(); + $this->assertSame(['John', 'Eve'], $this->names($activeAdmins)); + + // The QUERY.md example with login_count / last_login + $this->noneDB->insert('accounts', [ + ['name' => 'a', 'role' => 'admin', 'status' => 'active', 'login_count' => 150, 'last_login' => 1], + ['name' => 'b', 'role' => 'admin', 'status' => 'active', 'login_count' => 100, 'last_login' => 1], // not > 100 + ['name' => 'c', 'role' => 'admin', 'status' => 'active', 'login_count' => 500], // no last_login + ['name' => 'd', 'role' => 'admin', 'status' => 'banned', 'login_count' => 500, 'last_login' => 1], + ['name' => 'e', 'role' => 'user', 'status' => 'active', 'login_count' => 500, 'last_login' => null], // role + ['name' => 'f', 'role' => 'admin', 'status' => 'active', 'login_count' => 101, 'last_login' => null], // null exists + ]); + $result = $this->q('accounts')->where([ + 'role' => 'admin', + 'status' => 'active', + 'login_count' => ['$gt' => 100], + 'last_login' => ['$exists' => true] + ])->get(); + $this->assertSame(['a', 'f'], $this->names($result)); + } + + /** + * "How values compare" in QUERY.md + */ + public function testHowValuesCompare(): void + { + $this->noneDB->insert('values', [ + ['name' => 'int', 'v' => 30], + ['name' => 'numstr', 'v' => '30'], + ['name' => 'float', 'v' => 18.5], + ['name' => 'text', 'v' => 'abc'], + ['name' => 'null', 'v' => null], + ['name' => 'true', 'v' => true], + ['name' => 'false', 'v' => false], + ['name' => 'array', 'v' => [100]], + ['name' => 'missing'], + ]); + + // numbers and numeric strings numerically ("30" > 18); text vs number, null, booleans, arrays, missing: never + $this->assertSame(['int', 'numstr', 'float'], $this->names($this->q('values')->where(['v' => ['$gt' => 18]])->get())); + $this->assertSame(['float'], $this->names($this->q('values')->where(['v' => ['$lt' => 20]])->get())); + // other strings lexically ('2024-02-01' > '2024-01-15'); numbers never compare with a non-numeric string + $this->noneDB->insert('dates', [ + ['name' => 'feb', 'v' => '2024-02-01'], ['name' => 'jan10', 'v' => '2024-01-10'], ['name' => 'num', 'v' => 20240201], + ['name' => 'jan15', 'v' => '2024-01-15'], ['name' => 'null', 'v' => null], + ]); + $this->assertSame(['feb'], $this->names($this->q('dates')->where(['v' => ['$gt' => '2024-01-15']])->get())); + $this->assertSame(['feb', 'jan15'], $this->names($this->q('dates')->where(['v' => ['$gte' => '2024-01-15']])->get())); + $this->assertSame(['jan10'], $this->names($this->q('dates')->where(['v' => ['$lt' => '2024-01-15']])->get())); + $this->assertSame(['jan10', 'jan15'], $this->names($this->q('dates')->between('v', '2024-01-01', '2024-01-31')->get())); + // A NAN operand never matches + foreach (['$gt', '$gte', '$lt', '$lte'] as $op) { + $this->assertSame([], $this->q('values')->where(['v' => [$op => NAN]])->get(), $op . ' NAN'); + } + $this->assertSame([], $this->q('values')->between('v', NAN, 100)->get()); + // notBetween() is the complement: every record + $this->assertCount(9, $this->q('values')->notBetween('v', NAN, 100)->get()); + + // $eq, $ne, $in, $nin, whereIn(), whereNot() and plain values are strict: '30' is not 30 + $this->assertSame(['int'], $this->names($this->q('values')->where(['v' => 30])->get())); + $this->assertSame(['numstr'], $this->names($this->q('values')->where(['v' => '30'])->get())); + $this->assertSame(['int'], $this->names($this->q('values')->where(['v' => ['$eq' => 30]])->get())); + $this->assertSame([], $this->q('values')->where(['v' => ['$eq' => 30.0]])->get()); + $this->assertSame(['numstr'], $this->names($this->q('values')->where(['v' => ['$in' => ['30']]])->get())); + $this->assertSame(['int'], $this->names($this->q('values')->whereIn('v', [30])->get())); + $this->assertCount(8, $this->q('values')->where(['v' => ['$ne' => 30]])->get()); + $this->assertCount(8, $this->q('values')->where(['v' => ['$nin' => ['30']]])->get()); + $this->assertCount(8, $this->q('values')->whereNot(['v' => 30])->get()); + $this->assertSame(['null'], $this->names($this->q('values')->where(['v' => null])->get())); + $this->assertSame(['true'], $this->names($this->q('values')->where(['v' => true])->get())); + } + + /** + * An array that is not made of operators is compared as a whole + */ + public function testNonOperatorArrayIsComparedAsAWhole(): void + { + $this->noneDB->insert('addresses', [ + ['name' => 'exact', 'addr' => ['city' => 'Ist']], + ['name' => 'more', 'addr' => ['city' => 'Ist', 'zip' => 34000]], + ['name' => 'other', 'addr' => ['city' => 'Ank']], + ['name' => 'string', 'addr' => 'Ist'], + ]); + + $this->assertSame(['exact'], $this->names($this->q('addresses')->where(['addr' => ['city' => 'Ist']])->get())); + $this->assertSame(['John'], $this->names($this->q()->where(['tags' => ['developer', 'senior']])->get())); + $this->assertSame([], $this->q()->where(['tags' => ['senior', 'developer']])->get()); + } + + /** + * QUERY.md exceptions: a mix throws at where()/orWhere(); unknown operator, $in/$nin without an array and an + * invalid $regex throw when the query runs, also on an empty database + */ + public function testInvalidFiltersThrowInvalidArgumentException(): void + { + foreach (['where', 'orWhere'] as $method) { + try { + $this->q()->$method(['age' => ['$gt' => 1, 'plain' => 2]]); + $this->fail("$method() with a mix of operators and plain keys must throw"); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString('cannot mix operators', $e->getMessage()); + } + } + + $this->noneDB->createDB('emptydb'); + $cases = [ + [['age' => ['$gtee' => 18]], "unknown operator '\$gtee'"], + [['age' => ['$in' => 18]], '$in needs an array'], + [['age' => ['$nin' => 'x']], '$nin needs an array'], + [['name' => ['$regex' => '[unclosed']], 'invalid $regex pattern'], + ]; + foreach ([$this->testDbName, 'emptydb'] as $db) { + foreach ($cases as [$filter, $message]) { + $query = $this->q($db)->where($filter); // checked when the query runs, not here + try { + $query->get(); + $this->fail("$message: get() on $db must throw"); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString($message, $e->getMessage()); + } + } + } + } + + /** + * A key filter means the same in every clause as in find(): integers and canonical digit strings only + */ + public function testKeyFilterMeansTheSameInEveryClause(): void + { + $this->assertSame(['Diana'], $this->names($this->q()->where(['key' => 5])->get())); + $this->assertSame(['Diana'], $this->names($this->q()->where(['key' => '5'])->get())); + $this->assertSame(['Jane', 'Bob'], $this->names($this->q()->where(['key' => [1, 2]])->get())); + foreach (["5\n", ' 5', '05', 5.0, '5.0'] as $notKey) { + $this->assertSame([], $this->q()->where(['key' => $notKey])->get(), var_export($notKey, true)); + $this->assertSame([], $this->q()->where(['role' => 'nobody'])->orWhere(['key' => $notKey])->get(), var_export($notKey, true)); + } + // orWhere and several where() calls + $this->assertSame(['John', 'Diana', 'Eve'], $this->names($this->q()->where(['role' => 'admin'])->orWhere(['key' => '5'])->get())); + $this->assertSame(['Jane', 'Diana'], $this->names($this->q()->where(['key' => [1, 2, 5]])->where(['status' => 'active'])->get())); + $this->assertSame(['Jane'], $this->names($this->q()->where(['status' => 'active'])->where(['key' => '1'])->get())); + // as find() + $this->assertSame(['Diana'], $this->names($this->noneDB->find($this->testDbName, ['key' => '5']))); + } + + // ========== FILTER METHODS ========== + + /** + * orWhere(): all where() groups OR any orWhere() group + */ + public function testOrWhere(): void + { + // Admins (John, Eve) or salary >= 70000 (John, Eve) + $this->assertSame(['John', 'Eve'], $this->names( + $this->q()->where(['role' => 'admin'])->orWhere(['salary' => ['$gte' => 70000]])->get() + )); + // Users (Bob, Diana) or salary >= 70000 (John, Eve): key order + $this->assertSame(['John', 'Bob', 'Diana', 'Eve'], $this->names( + $this->q()->where(['role' => 'user'])->orWhere(['salary' => ['$gte' => 70000]])->get() + )); + // Without where(), only the orWhere() groups decide + $this->assertSame(['Jane', 'Charlie'], $this->names( + $this->q()->orWhere(['role' => 'moderator'])->orWhere(['status' => 'pending'])->get() + )); + // where() groups must all match before the OR + $this->assertSame(['John', 'Charlie', 'Eve'], $this->names( + $this->q()->where(['department' => 'Engineering'])->where(['role' => 'admin'])->orWhere(['role' => 'guest'])->get() + )); + // An empty group matches every record (intended) + $this->assertCount(7, $this->q()->where(['role' => 'nobody'])->orWhere([])->get()); + } + + /** + * The "Multiple OR conditions" example + */ + public function testOrWhereMultipleConditionsExample(): void + { + $this->noneDB->insert('products', [ + ['name' => 'tv', 'category' => 'electronics', 'featured' => false, 'discount' => 0], + ['name' => 'sofa', 'category' => 'home', 'featured' => true, 'discount' => 10], + ['name' => 'lamp', 'category' => 'home', 'featured' => false, 'discount' => 60], + ['name' => 'rug', 'category' => 'home', 'featured' => false, 'discount' => 50], // not > 50 + ['name' => 'pan', 'category' => 'kitchen', 'featured' => 1, 'discount' => 0], // 1 is not true + ]); + + $result = $this->q('products') + ->where(['category' => 'electronics']) + ->orWhere(['featured' => true]) + ->orWhere(['discount' => ['$gt' => 50]]) + ->get(); + + $this->assertSame(['tv', 'sofa', 'lamp'], $this->names($result)); + } + + /** + * whereIn(): membership + */ + public function testWhereIn(): void + { + $results = $this->q()->whereIn('department', ['Engineering', 'Design'])->get(); + $this->assertSame(['John', 'Jane', 'Bob', 'Eve'], $this->names($results)); + + $this->assertSame([], $this->q()->whereIn('department', [])->get()); + } + + /** + * whereNotIn(): not a member (records without the field are kept) + */ + public function testWhereNotIn(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'NoRole']); + + $results = $this->q()->whereNotIn('role', ['guest', 'user'])->get(); + + $this->assertSame(['John', 'Jane', 'Alice', 'Eve', 'NoRole'], $this->names($results)); + } + + /** + * whereNot(['role' => 'guest']): all users except guests + */ + public function testWhereNot(): void + { + $results = $this->q()->whereNot(['role' => 'guest'])->get(); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Diana', 'Eve'], $this->names($results)); + } + + /** + * whereNot() rules: any listed field excludes; operators per field AND-ed; $ne/$nin/$exists false match a + * missing field; a plain value never matches a missing field; several calls all apply + */ + public function testWhereNotRules(): void + { + // "drops guests and everyone in HR" (here: Engineering) + $this->assertSame(['Jane', 'Alice', 'Diana'], $this->names( + $this->q()->whereNot(['role' => 'guest', 'department' => 'Engineering'])->get() + )); + + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Kid', 'age' => 17], + ['name' => 'Retired', 'age' => 70], + ]); + // "Everyone except working-age users" + $this->assertSame(['Kid', 'Retired'], $this->names( + $this->q()->whereNot(['age' => ['$gte' => 18, '$lt' => 65]])->get() + )); + // $ne matches a missing field: Kid and Retired (no role) are dropped too + $this->assertSame(['John', 'Eve'], $this->names($this->q()->whereNot(['role' => ['$ne' => 'admin']])->get())); + // $nin and $exists => false as well + $this->assertSame(['John', 'Jane', 'Eve'], $this->names( + $this->q()->whereNot(['role' => ['$nin' => ['admin', 'moderator']]])->get() + )); + $this->assertSame(['John', 'Jane', 'Alice', 'Diana', 'Eve'], $this->names( + $this->q()->whereNot(['tags' => ['$exists' => false]])->get() + )); + // A plain value (even null) doesn't match a missing field: nothing is excluded + $this->assertCount(9, $this->q()->whereNot(['phone' => null])->get()); + // Most operators don't match a missing field: Kid/Retired kept + $this->assertSame(['Bob', 'Charlie', 'Kid', 'Retired'], $this->names( + $this->q()->whereNot(['status' => ['$in' => ['active']]])->get() + )); + // Several calls all apply + $this->assertSame(['John', 'Jane', 'Alice', 'Diana', 'Eve'], $this->names( + $this->q()->whereNot(['role' => 'guest'])->whereNot(['status' => 'inactive'])->whereNot(['age' => ['$lt' => 18]])->whereNot(['age' => ['$gt' => 65]])->get() + )); + } + + /** + * whereNot() checks its operators when it is called (where() checks them when the query runs) + */ + public function testWhereNotValidatesOperatorsAtCall(): void + { + $cases = [ + [['age' => ['$gtee' => 1]], "unknown operator '\$gtee'"], + [['age' => ['$in' => 1]], '$in needs an array'], + [['age' => ['$nin' => 1]], '$nin needs an array'], + [['name' => ['$regex' => '(']], 'invalid $regex pattern'], + [['age' => ['$gt' => 1, 'x' => 2]], 'cannot mix operators'], + ]; + foreach ($cases as [$filter, $message]) { + try { + $this->q()->whereNot($filter); // no get() + $this->fail("whereNot() must throw at once: $message"); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString($message, $e->getMessage()); + } + } + } + + /** + * like('email', 'gmail'): contains + */ + public function testLikeMethod(): void + { + $results = $this->q()->like('email', 'gmail')->get(); + + $this->assertSame(['John', 'Bob', 'Charlie', 'Eve'], $this->names($results)); + $this->assertSame(['Jane', 'Alice', 'Diana'], $this->names($this->q()->notLike('email', 'GMAIL')->get())); + } + + /** + * The like()/notLike() examples of QUERY.md; booleans, arrays and missing fields never match, numbers do + */ + public function testLikeMethodDocExamples(): void + { + $this->noneDB->insert('people', [ + ['name' => 'Anna'], ['name' => 'Jackson'], ['name' => 'john doe'], ['name' => 'Maria'], ['name' => 'Hanna'], + ]); + $this->assertSame(['Anna'], $this->names($this->q('people')->like('name', '^A')->get())); + $this->assertSame(['Jackson'], $this->names($this->q('people')->like('name', 'son$')->get())); + $this->assertSame(['john doe'], $this->names($this->q('people')->like('name', 'john')->get())); + $this->assertSame(['Anna', 'Hanna'], $this->names($this->q('people')->like('name', 'nna$')->get())); + + $this->noneDB->insert('links', [ + ['url' => 'http://a.com'], ['url' => 'https://b.com'], ['url' => 'xhttp://c.com'], ['url' => 'HTTP://D.COM'], + ]); + $this->assertSame(['http://a.com', 'HTTP://D.COM'], array_column($this->q('links')->like('url', '^http://')->get(), 'url')); + + $this->noneDB->insert('posts', [ + ['title' => 'SPAM offer'], ['title' => 'Hello'], ['title' => 'win: spam?'], ['title' => 'Spammy'], ['body' => 'no title'], + ]); + // records without the field never match like(), so notLike() keeps them + $kept = $this->q('posts')->notLike('title', 'SPAM')->get(); + $this->assertSame([1, 4], array_column($kept, 'key')); + + $this->noneDB->insert('codes', [ + ['code' => 12345], ['code' => '0234'], ['code' => true], ['code' => [234]], ['code' => 1234.5], ['other' => 1], + ]); + $this->assertSame([0, 1, 4], array_column($this->q('codes')->like('code', '234')->get(), 'key')); + $this->assertSame([2, 3, 5], array_column($this->q('codes')->notLike('code', '234')->get(), 'key')); + $this->assertSame([], $this->q('codes')->like('code', '1')->where(['code' => true])->get()); + } + + /** + * between(): inclusive range + */ + public function testBetweenMethod(): void + { + $this->assertSame(['Jane', 'Bob', 'Alice'], $this->names($this->q()->between('salary', 50000, 70000)->get())); + // inclusive at both ends: John 25, Bob 35 + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Eve'], $this->names($this->q()->between('age', 25, 35)->get())); + $this->assertSame([], $this->q()->between('age', 36, 44)->get()); + } + + /** + * notBetween(): the complement, including records without the field or with a value that can't be compared + */ + public function testNotBetweenMethod(): void + { + $this->assertSame(['Charlie', 'Diana'], $this->names($this->q()->notBetween('age', 25, 35)->get())); + + $this->noneDB->insert($this->testDbName, [ + ['name' => 'NoAge'], + ['name' => 'NullAge', 'age' => null], + ['name' => 'BoolAge', 'age' => true], + ['name' => 'TextAge', 'age' => 'old'], + ['name' => 'ArrAge', 'age' => [30]], + ['name' => 'NumStr', 'age' => '30'], + ]); + $this->assertSame( + ['Charlie', 'Diana', 'NoAge', 'NullAge', 'BoolAge', 'TextAge', 'ArrAge'], + $this->names($this->q()->notBetween('age', 25, 35)->get()) + ); + $this->assertSame( + ['John', 'Jane', 'Bob', 'Alice', 'Eve', 'NumStr'], + $this->names($this->q()->between('age', 25, 35)->get()) + ); + } + + /** + * search('gmail', ['email']) + */ + public function testSearchMethod(): void + { + $results = $this->q()->search('gmail', ['email'])->get(); + + $this->assertSame(['John', 'Bob', 'Charlie', 'Eve'], $this->names($results)); + $this->assertSame([], $this->q()->search('gmail', ['name'])->get()); + } + + /** + * search(): whole term as a substring of one scalar field, case-insensitive; no word splitting; without a field + * list every field except key + */ + public function testSearchDocRules(): void + { + $this->noneDB->insert('articles', [ + ['title' => 'PHP Tutorial', 'content' => 'Learn step by step', 'tags' => ['php']], + ['title' => 'Tutorial on php', 'content' => 'basics'], + ['title' => 'Advanced', 'content' => 'A PHP tutorial for experts'], + ['title' => 'Cooking', 'content' => 'no code', 'tags' => ['php tutorial']], + ]); + + $this->assertSame([0, 1, 2], array_column($this->q('articles')->search('php')->get(), 'key')); + $this->assertSame([0, 2], array_column($this->q('articles')->search('php tutorial', ['title', 'content', 'tags'])->get(), 'key')); + // only the given fields + $this->assertSame([0], array_column($this->q('articles')->search('php tutorial', ['title'])->get(), 'key')); + // key is not searched: "3" appears in no value + $this->assertSame([], $this->q('articles')->search('3')->get()); + } + + /** + * search() folds Unicode case like like(): İzmir, IZMIR and izmir; straße and STRASSE + */ + public function testSearchUnicodeFolding(): void + { + if (!function_exists('mb_convert_case')) { + $this->markTestSkipped('needs the mbstring extension (without it only ASCII letters are folded)'); + } + $this->noneDB->insert('cities', [['city' => 'İzmir'], ['city' => 'IZMIR'], ['city' => 'izmir'], ['city' => 'straße'], ['city' => 'Ankara']]); + + $this->assertSame([0, 1, 2], array_column($this->q('cities')->search('izmir')->get(), 'key')); + $this->assertSame([0, 1, 2], array_column($this->q('cities')->search('İZMİR')->get(), 'key')); + $this->assertSame([3], array_column($this->q('cities')->search('STRASSE')->get(), 'key')); + } + + // ========== SORTING & PAGINATION ========== + + public function testSortAsc(): void + { + $this->assertSame(['Alice', 'Bob', 'Charlie', 'Diana', 'Eve', 'Jane', 'John'], $this->names($this->q()->sort('name', 'asc')->get())); + } + + public function testSortDesc(): void + { + $this->assertSame(['Eve', 'John', 'Jane', 'Alice', 'Bob', 'Diana', 'Charlie'], $this->names($this->q()->sort('salary', 'desc')->get())); + } + + /** + * sort('name') defaults to ascending; orderBy() is an alias + */ + public function testSortDefaultOrderAndOrderByAlias(): void + { + $expected = ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve', 'Jane', 'John']; + $this->assertSame($expected, $this->names($this->q()->sort('name')->get())); + $this->assertSame($expected, $this->names($this->q()->orderBy('name')->get())); + $this->assertSame(array_reverse($expected), $this->names($this->q()->orderBy('name', 'desc')->get())); + } + + /** + * "Several keys: category first, then price" + */ + public function testSortSeveralKeys(): void + { + $result = $this->q()->sort('department')->sort('salary', 'desc')->get(); + + // Design: Jane; Engineering: Eve 80000, John 75000, Bob 55000; Guest; Marketing; Support + $this->assertSame(['Jane', 'Eve', 'John', 'Bob', 'Charlie', 'Alice', 'Diana'], $this->names($result)); + } + + /** + * The total sort order: missing/null < booleans < numbers < numeric strings (by number, trailing whitespace allowed) + * < other strings (byte order) < arrays; stable (ties keep key order, also for desc); min()/max() use it too + */ + public function testSortTotalOrder(): void + { + $values = [ + ['missing', null], ['a', 'a'], ['null', null], ['3', 3], ['"10"', '10'], ['true', true], ['[1]', [1]], + ['"9"', '9'], ['false', false], ['2.5', 2.5], ['Z', 'Z'], ['"1e3"', '1e3'], ['"1a"', '1a'], ['-1', -1], + ['"7 "', '7 '], ['3.0', 3.0], + ]; + $records = []; + foreach ($values as [$label, $value]) { + $records[] = $label === 'missing' ? ['name' => $label] : ['name' => $label, 'v' => $value]; + } + $this->noneDB->insert('mixed', $records); + + $asc = ['missing', 'null', 'false', 'true', '-1', '2.5', '3', '3.0', '"7 "', '"9"', '"10"', '"1e3"', '"1a"', 'Z', 'a', '[1]']; + $desc = ['[1]', 'a', 'Z', '"1a"', '"1e3"', '"10"', '"9"', '"7 "', '3', '3.0', '2.5', '-1', 'true', 'false', 'missing', 'null']; + $this->assertSame($asc, $this->names($this->q('mixed')->sort('v')->get())); + $this->assertSame($desc, $this->names($this->q('mixed')->sort('v', 'desc')->get())); + // top-k (offset + limit <= 1000) gives the same rows as the full sort + $this->assertSame(array_slice($asc, 3, 5), $this->names($this->q('mixed')->sort('v')->offset(3)->limit(5)->get())); + $this->assertSame(array_slice($desc, 0, 4), $this->names($this->q('mixed')->sort('v', 'desc')->limit(4)->get())); + // noneDB::sort() on a result uses the same order + $this->assertSame($asc, $this->names($this->noneDB->sort($this->noneDB->find('mixed'), 'v'))); + + // min()/max() use the order above (null/missing are not values) + $this->assertSame(false, $this->q('mixed')->min('v')); + $this->assertSame([1], $this->q('mixed')->max('v')); + $this->noneDB->insert('mm', [['p' => 50], ['p' => '10']]); + $this->assertSame('10', $this->q('mm')->max('p')); + $this->assertSame(50, $this->q('mm')->min('p')); + $this->assertSame('10', $this->noneDB->max('mm', 'p')); + $this->assertSame(50, $this->noneDB->min('mm', 'p')); + } + + public function testLimit(): void + { + $this->assertSame(['John', 'Jane', 'Bob'], $this->names($this->q()->limit(3)->get())); + } + + public function testOffset(): void + { + $this->assertSame(['Bob', 'Alice', 'Charlie', 'Diana', 'Eve'], $this->names($this->q()->offset(2)->get())); + $this->assertSame(['Bob', 'Alice', 'Charlie', 'Diana', 'Eve'], $this->names($this->q()->skip(2)->get())); + $this->assertSame([], $this->q()->offset(7)->get()); + } + + /** + * limit + offset, and the paginate() helper of QUERY.md + */ + public function testPagination(): void + { + $this->assertSame(['Bob', 'Alice'], $this->names($this->q()->limit(2)->offset(2)->get())); + + $paginate = function ($db, $dbname, $page, $perPage = 20) { + return $db->query($dbname)->limit($perPage)->offset(($page - 1) * $perPage)->get(); + }; + $this->assertSame(['John', 'Jane', 'Bob'], $this->names($paginate($this->noneDB, $this->testDbName, 1, 3))); + $this->assertSame(['Alice', 'Charlie', 'Diana'], $this->names($paginate($this->noneDB, $this->testDbName, 2, 3))); + $this->assertSame(['Eve'], $this->names($paginate($this->noneDB, $this->testDbName, 3, 3))); + $this->assertSame([], $paginate($this->noneDB, $this->testDbName, 4, 3)); + $this->assertCount(7, $paginate($this->noneDB, $this->testDbName, 1)); + } + + /** + * limit(0) returns nothing: first() is null, exists() is false + */ + public function testLimitZero(): void + { + $this->assertSame([], $this->q()->limit(0)->get()); + $this->assertNull($this->q()->limit(0)->first()); + $this->assertFalse($this->q()->limit(0)->exists()); + $this->assertNull($this->q()->limit(0)->last()); + $this->assertSame(0, $this->q()->limit(0)->count()); + } + + /** + * select(['name', 'email']): only those fields and key + */ + public function testSelect(): void + { + $results = $this->q()->select(['name', 'email'])->get(); + + $expected = []; + foreach ($this->people as $key => $p) { + $expected[] = ['name' => $p['name'], 'email' => $p['email'], 'key' => $key]; + } + $this->assertSameRecords($expected, $results); + } + + /** + * except(['salary', 'age']): every other field + */ + public function testExcept(): void + { + $results = $this->q()->except(['salary', 'age'])->get(); + + $expected = []; + foreach ($this->people as $key => $p) { + unset($p['salary'], $p['age']); + $expected[] = $p + ['key' => $key]; + } + $this->assertSameRecords($expected, $results); + } + + /** + * select()/except() only shape the result: filters and sort see all fields; key is always kept + */ + public function testSelectExceptOnlyShapeTheResult(): void + { + $rows = $this->q()->select(['name'])->where(['salary' => ['$gt' => 60000]])->sort('salary', 'desc')->get(); + $this->assertSame([['name' => 'Eve', 'key' => 6], ['name' => 'John', 'key' => 0], ['name' => 'Jane', 'key' => 1]], $rows); + + // "key is always kept", also when select() names only fields the records don't have + $this->assertSame([['key' => 0], ['key' => 1]], $this->q()->select(['nonexistent'])->limit(2)->get()); + } + + /** + * QUERY.md select()/except(): "`key` is always kept" - also when except() names it + */ + public function testExceptRemovesKeyOnlyWhenListed(): void + { + // QUERY.md select() / except(): select() always keeps key, except() removes it only when listed + $rows = $this->q()->except(['key', 'name'])->limit(2)->get(); + $expected = []; + foreach ([0, 1] as $key) { + $p = $this->people[$key]; + unset($p['name']); + $expected[] = $p; + } + $this->assertSameRecords($expected, $rows); + + $rows = $this->q()->except(['name'])->limit(2)->get(); + foreach ($expected as $i => $p) { + $expected[$i] = $p + ['key' => $i]; + } + $this->assertSameRecords($expected, $rows, 'key stays when it is not listed'); + } + + // ========== AGGREGATION ========== + + /** + * count(): limit()/offset() apply; with groupBy() it counts the groups + */ + public function testCount(): void + { + $this->assertSame(7, $this->q()->count()); + $this->assertSame(5, $this->q()->where(['status' => 'active'])->count()); + + // "offset(2)->count() on 6 matches is 4" + $this->assertSame(6, $this->q()->whereNot(['role' => 'guest'])->count()); + $this->assertSame(4, $this->q()->whereNot(['role' => 'guest'])->offset(2)->count()); + $this->assertSame(3, $this->q()->whereNot(['role' => 'guest'])->limit(3)->count()); + $this->assertSame(2, $this->q()->whereNot(['role' => 'guest'])->offset(4)->limit(10)->count()); + $this->assertSame(5, $this->q()->offset(2)->count()); + // groups: Engineering, Design, Marketing, Guest, Support + $this->assertSame(5, $this->q()->groupBy('department')->count()); + $this->assertSame(1, $this->q()->groupBy('department')->having('count', '>', 1)->count()); + } + + public function testSum(): void + { + // 75000 + 65000 + 55000 + 60000 + 0 + 45000 + 80000 + $this->assertSame(380000.0, $this->q()->sum('salary')); + $this->assertSame(280000.0, $this->q()->where(['status' => 'active'])->where(['department' => ['$ne' => 'Support']])->sum('salary')); + $this->assertSame(0.0, $this->q()->where(['role' => 'nobody'])->sum('salary')); + } + + /** + * The sum()/avg()/min()/max() examples of QUERY.md + */ + public function testAggregationDocExamples(): void + { + $this->noneDB->insert('orders', [ + ['status' => 'completed', 'total' => 100], + ['status' => 'pending', 'total' => 40], + ['status' => 'completed', 'total' => 20.5], + ]); + $this->assertSame(120.5, $this->q('orders')->where(['status' => 'completed'])->sum('total')); + + $this->noneDB->insert('products', [ + ['category' => 'electronics', 'rating' => 4, 'price' => 300], + ['category' => 'books', 'rating' => 2, 'price' => 15], + ['category' => 'electronics', 'rating' => 5, 'price' => 1200], + ]); + $this->assertSame(4.5, $this->q('products')->where(['category' => 'electronics'])->avg('rating')); + $this->assertSame(15, $this->q('products')->min('price')); + $this->assertSame(1200, $this->q('products')->max('price')); + } + + public function testAvg(): void + { + // (25 + 30 + 35 + 28 + 45 + 22 + 33) / 7 + $this->assertSame(218 / 7, $this->q()->avg('age')); + $this->assertSame(0.0, $this->q()->where(['role' => 'nobody'])->avg('age')); + } + + public function testMin(): void + { + $this->assertSame(22, $this->q()->min('age')); + $this->assertNull($this->q()->where(['role' => 'nobody'])->min('age')); + } + + public function testMax(): void + { + $this->assertSame(80000, $this->q()->max('salary')); + $this->assertNull($this->q()->max('nonexistent')); + } + + /** + * distinct(): order of first appearance, null/missing left out, typed identity (-0.0 and 0.0 are one) + */ + public function testDistinct(): void + { + $this->noneDB->insert('users', [ + ['city' => 'Istanbul', 'active' => true], + ['city' => 'Ankara', 'active' => true], + ['city' => 'Istanbul', 'active' => true], + ['city' => 'Izmir', 'active' => false], + ['active' => true], + ['city' => null, 'active' => true], + ['city' => 'Bursa', 'active' => true], + ]); + $this->assertSame(['Istanbul', 'Ankara', 'Bursa'], $this->q('users')->where(['active' => true])->distinct('city')); + $this->assertSame(['Istanbul', 'Ankara', 'Izmir', 'Bursa'], $this->noneDB->distinct('users', 'city')); + + $this->noneDB->insert('typed', [['v' => 1], ['v' => 1.0], ['v' => '1'], ['v' => 1], ['v' => -0.0], ['v' => 0.0]]); + $distinct = $this->q('typed')->distinct('v'); + $this->assertCount(4, $distinct); + $this->assertSame(1, $distinct[0]); + $this->assertSame(1.0, $distinct[1]); + $this->assertSame('1', $distinct[2]); + $this->assertSame(0.0, $distinct[3]); + $this->assertSame([], $this->q('typed')->where(['v' => 'none'])->distinct('v')); + } + + /** + * Aggregates work on the matching records before select()/except() + */ + public function testAggregatesIgnoreSelectAndExcept(): void + { + $this->assertSame(380000.0, $this->q()->select(['name'])->sum('salary')); + $this->assertSame(218 / 7, $this->q()->except(['age'])->avg('age')); + $this->assertSame(22, $this->q()->select(['name'])->min('age')); + $this->assertSame(80000, $this->q()->except(['salary'])->max('salary')); + $this->assertSame(['Engineering', 'Design', 'Marketing', 'Guest', 'Support'], $this->q()->select(['name'])->distinct('department')); + $this->assertSame(7, $this->q()->select(['name'])->count()); + } + + // ========== JOINS ========== + + /** + * join(): the matching record under the foreign database's name (or the alias), null if none; the attached + * record replaces a field of the same name in the result only + */ + public function testJoinDocExamples(): void + { + $this->noneDB->insert('users', [['name' => 'Ann'], ['name' => 'Ben'], ['name' => 'Cid']]); + $this->noneDB->insert('orders', [ + ['item' => 'a', 'user_id' => 0], + ['item' => 'b', 'user_id' => 2, 'users' => 'stored value'], + ['item' => 'c', 'user_id' => 9], + ]); + + $orders = $this->q('orders')->join('users', 'user_id', 'key')->get(); + $this->assertCount(3, $orders); + $this->assertSame(['name' => 'Ann', 'key' => 0], $orders[0]['users']); + $this->assertSame(['name' => 'Cid', 'key' => 2], $orders[1]['users']); + $this->assertNull($orders[2]['users']); + $this->assertSame(['Ann', 'Cid', '-'], array_map(function ($o) { return $o['users']['name'] ?? '-'; }, $orders)); + // stored records are never changed + $this->assertSame('stored value', $this->noneDB->find('orders', ['key' => 1])[0]['users']); + $this->assertArrayNotHasKey('users', $this->noneDB->find('orders', ['key' => 0])[0]); + + // alias + $buyers = $this->q('orders')->join('users', 'user_id', 'key', 'buyer')->get(); + $this->assertSame(['Ann', 'Cid', null], array_map(function ($o) { return $o['buyer']['name'] ?? null; }, $buyers)); + $this->assertArrayNotHasKey('users', $buyers[0]); + $this->assertSame('stored value', $buyers[1]['users']); + } + + /** + * join() matching rules of QUERY.md + */ + public function testJoinMatchingRules(): void + { + $this->noneDB->insert('users', [['name' => 'u0'], ['name' => 'u1'], ['name' => 'u2'], ['name' => 'u3'], ['name' => 'u4'], ['name' => 'u5'], ['name' => 'u6'], ['name' => 'u7']]); + $this->noneDB->insert('orders', [ + ['o' => 'int', 'user_id' => 5], + ['o' => 'string', 'user_id' => '5'], + ['o' => 'float', 'user_id' => 5.0], + ['o' => 'half', 'user_id' => 1.5], + ['o' => 'zeros', 'user_id' => '007'], + ['o' => 'true', 'user_id' => true], + ['o' => 'null', 'user_id' => null], + ['o' => 'array', 'user_id' => [5]], + ['o' => 'missing'], + ]); + $joined = []; + foreach ($this->q('orders')->join('users', 'user_id', 'key')->get() as $row) { + $joined[$row['o']] = $row['users']['name'] ?? null; + } + $this->assertSame([ + 'int' => 'u5', 'string' => 'u5', 'float' => 'u5', // 5, "5" and 5.0 match each other + 'half' => null, // other floats only the same float + 'zeros' => null, // "007" doesn't match 7 + 'true' => null, // true matches only true + 'null' => null, 'array' => null, 'missing' => null, + ], $joined); + + // On another field: floats, booleans, strings exactly; last duplicate wins + $this->noneDB->insert('codes', [ + ['code' => 1, 'label' => 'one'], + ['code' => 1.5, 'label' => 'one and a half'], + ['code' => true, 'label' => 'yes'], + ['code' => false, 'label' => 'no'], + ['code' => '007', 'label' => 'agent'], + ['code' => 7, 'label' => 'seven'], + ['code' => 'x', 'label' => 'first x'], + ['code' => 'x', 'label' => 'last x'], + ['code' => null, 'label' => 'null'], + ]); + $this->noneDB->insert('items', [ + ['c' => 1.0], ['c' => '1'], ['c' => 1.5], ['c' => true], ['c' => false], ['c' => '007'], ['c' => 7.0], ['c' => 'x'], ['c' => null], ['c' => 2], + ]); + $labels = array_map(function ($r) { return $r['codes']['label'] ?? null; }, $this->q('items')->join('codes', 'c', 'code')->get()); + $this->assertSame(['one', 'one', 'one and a half', 'yes', 'no', 'agent', 'seven', 'last x', null, null], $labels); + } + + /** + * Joining a database that doesn't exist attaches null and creates nothing + */ + public function testJoinMissingDatabaseAttachesNullAndCreatesNothing(): void + { + $byKey = $this->q()->join('nope', 'age', 'key')->get(); + $byField = $this->q()->join('nope2', 'name', 'owner')->get(); + + $this->assertCount(7, $byKey); + $this->assertSame(array_fill(0, 7, null), array_column($byKey, 'nope')); + $this->assertSame(array_fill(0, 7, null), array_map(function ($r) { return $r['nope2']; }, $byField)); + $this->assertFalse($this->noneDB->getDBs('nope')); + $this->assertFalse($this->noneDB->getDBs('nope2')); + $this->assertNotContains('nope', $this->noneDB->getDBs(false)); + } + + /** + * "Multiple Joins" example + */ + public function testMultipleJoinsExample(): void + { + $this->noneDB->insert('orders', [['no' => 'A-1'], ['no' => 'A-2']]); + $this->noneDB->insert('products', [['title' => 'Pen'], ['title' => 'Ink']]); + $this->noneDB->insert('order items', [ + ['order_id' => 1, 'product_id' => 0, 'qty' => 2], + ['order_id' => 0, 'product_id' => 1, 'qty' => 1], + ['order_id' => 5, 'product_id' => 1, 'qty' => 3], + ]); + + $data = $this->q('order items') + ->join('orders', 'order_id', 'key') + ->join('products', 'product_id', 'key') + ->get(); + + $this->assertSame(['A-2', 'A-1', null], array_map(function ($i) { return $i['orders']['no'] ?? null; }, $data)); + $this->assertSame(['Pen', 'Ink', 'Ink'], array_map(function ($i) { return $i['products']['title'] ?? null; }, $data)); + } + + // ========== GROUPING & HAVING ========== + + /** + * groupBy('status'): ['_group', '_items', '_count'] per value + */ + public function testGroupByDocExample(): void + { + $groups = $this->q()->groupBy('status')->get(); + + $byValue = []; + foreach ($groups as $group) { + $this->assertSame(['_count', '_group', '_items'], (function ($g) { $k = array_keys($g); sort($k); return $k; })($group)); + $byValue[$group['_group']] = [$group['_count'], array_column($group['_items'], 'name')]; + } + ksort($byValue); + $this->assertSame([ + 'active' => [5, ['John', 'Jane', 'Alice', 'Diana', 'Eve']], + 'inactive' => [1, ['Bob']], + 'pending' => [1, ['Charlie']], + ], $byValue); + // items are the full records + foreach ($groups as $group) { + if ($group['_group'] === 'inactive') { + $this->assertSameRecords([$this->person(2)], $group['_items']); + } + } + } + + /** + * Group values keep their type; missing fields group under null; -0.0 and 0.0 are one group + */ + public function testGroupByTyping(): void + { + $this->noneDB->insert('typed', [ + ['v' => 1], ['v' => '1'], ['v' => 1.5], ['v' => 1], ['x' => 'no v'], ['v' => null], ['v' => -0.0], ['v' => 0.0], + ]); + + $counts = []; + foreach ($this->q('typed')->groupBy('v')->get() as $group) { + $counts[] = [gettype($group['_group']) . ':' . var_export($group['_group'], true), $group['_count'], array_column($group['_items'], 'key')]; + } + usort($counts, function ($a, $b) { return strcmp($a[0], $b[0]); }); + $this->assertSame([ + ['NULL:NULL', 2, [4, 5]], + ['double:-0.0', 2, [6, 7]], + ['double:1.5', 1, [2]], + ['integer:1', 2, [0, 3]], + ["string:'1'", 1, [1]], + ], $counts); + } + + /** + * groupBy()->select(['_group', 'total']): the QUERY.md example (_count dropped, items projected) + */ + public function testGroupBySelectDocExample(): void + { + $this->noneDB->insert('orders', [ + ['status' => 'pending', 'total' => 12.5, 'customer' => 'a'], + ['status' => 'completed', 'total' => 20, 'customer' => 'b'], + ['status' => 'pending', 'total' => 7.25, 'customer' => 'c'], + ]); + + $rows = $this->q('orders')->groupBy('status')->select(['_group', 'total'])->get(); + usort($rows, function ($a, $b) { return strcmp($a['_group'], $b['_group']); }); + + $this->assertSame([ + ['_group' => 'completed', '_items' => [['total' => 20, 'key' => 1]]], + ['_group' => 'pending', '_items' => [['total' => 12.5, 'key' => 0], ['total' => 7.25, 'key' => 2]]], + ], array_map(function ($row) { ksort($row); return $row; }, $rows)); + + // _group/_count/_items select or drop parts of each row; other fields apply to the items + $counts = $this->q('orders')->groupBy('status')->select(['_group', '_count'])->get(); + foreach ($counts as $row) { + $this->assertSame(['_count', '_group'], (function ($r) { $k = array_keys($r); sort($k); return $k; })($row)); + } + $noItems = $this->q('orders')->groupBy('status')->except(['_items'])->get(); + foreach ($noItems as $row) { + $this->assertArrayNotHasKey('_items', $row); + $this->assertArrayHasKey('_group', $row); + $this->assertArrayHasKey('_count', $row); + } + $itemsExcept = $this->q('orders')->groupBy('status')->except(['customer', 'status'])->get(); + foreach ($itemsExcept as $row) { + $this->assertArrayHasKey('_count', $row); + foreach ($row['_items'] as $item) { + $this->assertSame(['key', 'total'], (function ($r) { $k = array_keys($r); sort($k); return $k; })($item)); + } + } + } + + /** + * having(): the QUERY.md examples, every aggregate and operator; ignored without groupBy() or when unknown + */ + public function testHavingDocExamples(): void + { + $orders = []; + foreach (['c1' => 6, 'c2' => 5, 'c3' => 1] as $customer => $n) { + for ($i = 0; $i < $n; $i++) { + $orders[] = ['customer_id' => $customer, 'total' => 10]; + } + } + $this->noneDB->insert('orders', $orders); + $frequent = $this->q('orders')->groupBy('customer_id')->having('count', '>', 5)->get(); + $this->assertSame(['c1'], array_column($frequent, '_group')); + + $this->noneDB->insert('products', [ + ['category' => 'A', 'rating' => 4], ['category' => 'A', 'rating' => 5], + ['category' => 'B', 'rating' => 3], ['category' => 'B', 'rating' => 4], + ['category' => 'C', 'rating' => 4], + ]); + $groups = function ($query) { + $values = array_column($query->get(), '_group'); + sort($values); + return $values; + }; + $p = function () { return $this->q('products')->groupBy('category'); }; + + $this->assertSame(['A', 'C'], $groups($p()->having('avg:rating', '>=', 4))); // A 4.5, B 3.5, C 4 + $this->assertSame(['B'], $groups($p()->having('avg:rating', '<', 4))); + $this->assertSame(['A', 'B'], $groups($p()->having('sum:rating', '>', 4))); // 9, 7, 4 + $this->assertSame(['C'], $groups($p()->having('sum:rating', '<=', 4))); + $this->assertSame(['A', 'C'], $groups($p()->having('min:rating', '=', 4))); // 4, 3, 4 + $this->assertSame(['B'], $groups($p()->having('min:rating', '!=', 4))); + $this->assertSame(['A'], $groups($p()->having('max:rating', '>', 4))); // 5, 4, 4 + $this->assertSame(['A', 'B'], $groups($p()->having('count', '=', 2))); + $this->assertSame(['A'], $groups($p()->having('count', '>=', 2)->having('max:rating', '>=', 5))); // all apply + // unknown aggregate or operator: ignored + $this->assertSame(['A', 'B', 'C'], $groups($p()->having('median:rating', '>', 100))); + $this->assertSame(['A', 'B', 'C'], $groups($p()->having('count', '~', 100))); + // without groupBy(): ignored + $this->assertCount(5, $this->q('products')->having('count', '>', 100)->get()); + } + + // ========== TERMINAL METHODS ========== + + public function testGet(): void + { + $results = $this->q()->where(['status' => 'active'])->get(); + + $this->assertSameRecords([$this->person(0), $this->person(1), $this->person(3), $this->person(5), $this->person(6)], $results); + $this->assertSame([0, 1, 2, 3, 4], array_keys($results), 'get() returns a list'); + } + + public function testFirst(): void + { + $first = $this->q()->where(['email' => 'john@gmail.com'])->first(); + $this->assertSameRecords([$this->person(0)], [$first]); + + // "Use first() instead of get()[0]": the same record + $this->assertSame($this->q()->where(['role' => 'user'])->get()[0], $this->q()->where(['role' => 'user'])->first()); + $this->assertSame('Diana', $this->q()->where(['role' => 'user'])->sort('age')->first()['name']); + $this->assertNull($this->q()->where(['email' => 'nobody@example.com'])->first()); + } + + /** + * last(): the last record of the result after sort(), offset() and limit() + */ + public function testLast(): void + { + $newest = $this->q()->sort('created_at')->last(); + $this->assertSame('Charlie', $newest['name']); + + $this->assertSame('Eve', $this->q()->last()['name']); + $this->assertSame('Jane', $this->q()->sort('created_at')->limit(4)->last()['name']); // Eve, Diana, John, Jane + $this->assertSame('Bob', $this->q()->sort('created_at')->offset(1)->limit(4)->last()['name']); + $this->assertNull($this->q()->where(['role' => 'nobody'])->last()); + } + + public function testExists(): void + { + $this->assertTrue($this->q()->where(['role' => 'admin'])->exists()); + $this->assertFalse($this->q()->where(['role' => 'superadmin'])->exists()); + $this->assertTrue($this->q()->where(['role' => 'admin'])->offset(1)->exists()); + $this->assertFalse($this->q()->where(['role' => 'admin'])->offset(2)->exists()); + } + + /** + * update(): ['n' => updated], stored records changed, others untouched + */ + public function testUpdate(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'UpdateTest', 'status' => 'pending', 'age' => 25]); + + $result = $this->q()->where(['name' => 'UpdateTest'])->update(['status' => 'active']); + $this->assertSame(['n' => 1], $result); + $this->assertSame([['name' => 'UpdateTest', 'status' => 'active', 'age' => 25, 'key' => 7]], + $this->noneDB->find($this->testDbName, ['key' => 7])); + + // QUERY.md example: inactive -> archived + $this->assertSame(['n' => 1], $this->q()->where(['status' => 'inactive'])->update(['status' => 'archived'])); + $this->assertSame(['Bob'], $this->names($this->noneDB->find($this->testDbName, ['status' => 'archived']))); + $this->assertSame(['n' => 0], $this->q()->where(['status' => 'inactive'])->update(['status' => 'archived'])); + // the 5 active fixture users and UpdateTest + $this->assertSame(6, $this->noneDB->count($this->testDbName, ['status' => 'active'])); + } + + /** + * sort(), offset() and limit() choose which records are written; select(), except() and join() don't matter + */ + public function testWritesChooseRecordsBySortOffsetLimit(): void + { + // "the 10 oldest" (here 2): Eve (-20d), Diana (-15d) + $this->assertSame(['n' => 2], $this->q()->sort('created_at')->limit(2)->update(['status' => 'archived'])); + $this->assertSame(['Diana', 'Eve'], $this->names($this->noneDB->find($this->testDbName, ['status' => 'archived']))); + + // offset: the 2nd and 3rd highest salaries (John 75000, Jane 65000) + $this->assertSame(['n' => 2], $this->q()->sort('salary', 'desc')->offset(1)->limit(2)->update(['bonus' => true])); + $this->assertSame(['John', 'Jane'], $this->names($this->noneDB->find($this->testDbName, ['bonus' => true]))); + + // select/except/join change nothing about what is written + $this->assertSame(['n' => 2], $this->q()->where(['role' => 'admin'])->select(['name'])->except(['email'])->join('nope', 'age', 'key')->update(['flag' => 1])); + $this->assertSameRecords([ + array_merge($this->person(0), ['bonus' => true, 'flag' => 1]), + array_merge($this->person(6), ['status' => 'archived', 'flag' => 1]), + ], $this->noneDB->find($this->testDbName, ['flag' => 1])); + + // delete: the youngest record + $this->assertSame(['n' => 1], $this->q()->sort('age')->limit(1)->delete()); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['name' => 'Diana'])); + $this->assertSame(6, $this->noneDB->count($this->testDbName)); + } + + /** + * update(), delete() and removeFields() can't be combined with groupBy(): LogicException, nothing written + */ + public function testWritesWithGroupByThrowLogicException(): void + { + $before = $this->noneDB->find($this->testDbName); + $writes = [ + 'update' => function () { return $this->q()->groupBy('role')->update(['x' => 1]); }, + 'delete' => function () { return $this->q()->groupBy('role')->delete(); }, + 'removeFields' => function () { return $this->q()->groupBy('role')->removeFields(['age']); }, + ]; + foreach ($writes as $name => $write) { + try { + $write(); + $this->fail("$name() with groupBy() must throw"); + } catch (\LogicException $e) { + $this->assertStringContainsString('cannot be combined with groupBy()', $e->getMessage()); + } + } + $this->assertSame($before, $this->noneDB->find($this->testDbName)); + } + + /** + * delete(): ['n' => deleted] + */ + public function testDelete(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'DeleteTest', 'status' => 'temp']); + + $this->assertSame(['n' => 1], $this->q()->where(['name' => 'DeleteTest'])->delete()); + $this->assertFalse($this->q()->where(['name' => 'DeleteTest'])->exists()); + $this->assertSame(7, $this->noneDB->count($this->testDbName)); + + // QUERY.md example: logs older than 30 days + $now = $this->now; + $this->noneDB->insert('logs', [ + ['msg' => 'old', 'created_at' => $now - 40 * 86400], + ['msg' => 'new', 'created_at' => $now - 86400], + ['msg' => 'older', 'created_at' => $now - 400 * 86400], + ]); + $this->assertSame(['n' => 2], $this->q('logs')->where(['created_at' => ['$lt' => $now - 30 * 86400]])->delete()); + $this->assertSame(['new'], array_column($this->noneDB->find('logs'), 'msg')); + } + + /** + * removeFields(): the QUERY.md example: only fields that existed are listed; stored records change + */ + public function testRemoveFieldsDocExample(): void + { + $this->noneDB->insert('users', [ + ['name' => 'a', 'legacy' => true, 'old_password' => 'x'], + ['name' => 'b', 'legacy' => true, 'old_password' => 'y'], + ['name' => 'c', 'legacy' => false, 'old_password' => 'z'], + ['name' => 'd', 'legacy' => true], + ]); + + $result = $this->q('users')->where(['legacy' => true])->removeFields(['old_password', 'tmp']); + + $this->assertSame(['n' => 2, 'fields_removed' => ['old_password']], $result); + $this->assertSame([ + ['name' => 'a', 'legacy' => true, 'key' => 0], + ['name' => 'b', 'legacy' => true, 'key' => 1], + ['name' => 'c', 'legacy' => false, 'old_password' => 'z', 'key' => 2], + ['name' => 'd', 'legacy' => true, 'key' => 3], + ], array_map(function ($r) { $k = $r['key']; unset($r['key']); return $r + ['key' => $k]; }, $this->noneDB->find('users'))); + + // never affected by select()/except()/join(): the stored record keeps its other fields + $this->assertSame(['n' => 1, 'fields_removed' => ['legacy']], + $this->q('users')->where(['name' => 'a'])->select(['name'])->except(['legacy'])->join('nope', 'x', 'key')->removeFields(['legacy'])); + $this->assertSame([['name' => 'a', 'key' => 0]], $this->noneDB->find('users', ['key' => 0])); + } + + // ========== QUERY PLANNING & EXPLAIN ========== + + /** + * The explain() example of QUERY.md: 2000 users in 4 shards of 500, one city per shard, city and age indexed + */ + public function testExplainDocExample(): void + { + $db = new \noneDB([ + 'secretKey' => 'test_secret_key_for_unit_tests', + 'dbDir' => $this->testDbDir, + 'shardSize' => 500, + ]); + $cities = ['Istanbul', 'Ankara', 'Izmir', 'Bursa']; + $records = []; + for ($i = 0; $i < 2000; $i++) { + // 380 of the 500 users of each city are 30 or older + $records[] = ['city' => $cities[intdiv($i, 500)], 'age' => ($i % 500) < 380 ? 30 + $i % 40 : 20 + $i % 10]; + } + $this->assertSame(['n' => 2000], $db->insert('users', $records)); + $this->assertTrue($db->createFieldIndex('users', 'city')['success']); + $this->assertTrue($db->createFieldIndex('users', 'age')['success']); + + $plan = $db->query('users')->where(['city' => 'Istanbul', 'age' => ['$gte' => 30]])->limit(10)->explain(); + $this->assertSame(1, $plan['explain']); + $this->assertSame('users', $plan['db']); + $this->assertTrue($plan['sharded']); + $this->assertSame('index', $plan['access']); + $this->assertSame(['total' => 4, 'skipped' => 3, 'indexed' => 1, 'scanned' => 0], $plan['segments']); + $this->assertSame(['segment' => 0, 'access' => 'index', 'candidates' => 380], $plan['plan'][0]); + foreach ([1, 2, 3] as $i) { + $this->assertSame('skip', $plan['plan'][$i]['access']); + $this->assertSame($i, $plan['plan'][$i]['segment']); + } + $this->assertSame(380, $plan['estimatedRead']); + $this->assertSame('none', $plan['sort']); + $this->assertTrue($plan['earlyExit']); + $this->assertSame([], $plan['joins']); + $this->assertTrue($plan['residual']); + $this->assertNull($plan['groupBy']); + $this->assertFalse($plan['planTruncated']); + + $analyzed = $db->query('users')->where(['city' => 'Istanbul'])->explain(true); + $this->assertSame(500, $analyzed['actual']['recordsRead']); + $this->assertSame(1, $analyzed['actual']['segmentsRead']); + $this->assertSame(500, $analyzed['actual']['returned']); + $this->assertIsFloat($analyzed['actual']['ms']); + + // top-k is shown as sort => top-k; without a limit the sort is full + $this->assertSame('top-k', $db->query('users')->where(['city' => 'Ankara'])->sort('age')->limit(10)->explain()['sort']); + $this->assertSame('full', $db->query('users')->where(['city' => 'Ankara'])->sort('age')->explain()['sort']); + } + + // ========== REAL-WORLD EXAMPLES ========== + + /** + * E-commerce Product Search example + */ + public function testEcommerceSearchPattern(): void + { + $productDb = 'products'; + $this->noneDB->insert($productDb, [ + ['name' => 'Laptop', 'description' => 'Fast laptop', 'category' => 'electronics', 'price' => 999, 'stock' => 10, 'rating' => 4.5, 'status' => 'active'], + ['name' => 'Phone', 'description' => 'Fast phone', 'category' => 'electronics', 'price' => 599, 'stock' => 25, 'rating' => 4.2, 'status' => 'active'], + ['name' => 'Headphones', 'description' => 'Fast sound', 'category' => 'electronics', 'price' => 199, 'stock' => 3, 'rating' => 4.8, 'status' => 'discontinued'], + ['name' => 'Monitor', 'description' => 'FAST refresh', 'category' => 'computers', 'price' => 350, 'stock' => 5, 'rating' => 4.0, 'status' => 'active'], + ['name' => 'Cable', 'description' => 'slow', 'category' => 'computers', 'price' => 150, 'stock' => 5, 'rating' => 4.9, 'status' => 'active'], + ['name' => 'Fast charger', 'description' => 'usb', 'category' => 'computers', 'price' => 99, 'stock' => 5, 'rating' => 5.0, 'status' => 'active'], + ]); + + $page = 0; + $products = $this->q($productDb) + ->where([ + 'category' => ['$in' => ['electronics', 'computers']], + 'price' => ['$gte' => 100, '$lte' => 1000], + 'stock' => ['$gt' => 0], + 'rating' => ['$gte' => 4.0] + ]) + ->whereNot(['status' => 'discontinued']) + ->search('fast', ['name', 'description']) + ->sort('rating', 'desc') + ->limit(20) + ->offset($page * 20) + ->get(); + + $this->assertSame(['Laptop', 'Phone', 'Monitor'], $this->names($products)); + } + + /** + * User Authentication example (first() + password_verify) + */ + public function testUserAuthenticationPattern(): void + { + $authDb = 'auth'; + $passwordHash = password_hash('secret123', PASSWORD_DEFAULT); + $this->noneDB->insert($authDb, [ + ['email' => 'valid@test.com', 'password_hash' => $passwordHash, 'status' => 'active', 'email_verified' => true], + ['email' => 'unverified@test.com', 'password_hash' => $passwordHash, 'status' => 'active', 'email_verified' => false], + ['email' => 'banned@test.com', 'password_hash' => $passwordHash, 'status' => 'banned', 'email_verified' => true], + ['email' => 'string@test.com', 'password_hash' => $passwordHash, 'status' => 'active', 'email_verified' => 'true'], + ]); + $login = function ($email, $password) use ($authDb) { + $user = $this->q($authDb)->where(['email' => $email, 'status' => 'active', 'email_verified' => true])->first(); + return $user !== null && password_verify($password, $user['password_hash']); + }; + + $this->assertTrue($login('valid@test.com', 'secret123')); + $this->assertFalse($login('valid@test.com', 'wrong')); + $this->assertFalse($login('unverified@test.com', 'secret123')); + $this->assertFalse($login('banned@test.com', 'secret123')); + $this->assertFalse($login('string@test.com', 'secret123'), "'true' is not true"); + $this->assertFalse($login('nobody@test.com', 'secret123')); + $this->assertSame(0, $this->q($authDb)->where(['email' => 'valid@test.com'])->first()['key']); + } + + /** + * Analytics Dashboard example: groupBy over a date range, revenue per category + */ + public function testAnalyticsDashboardPattern(): void + { + $this->noneDB->insert('orders', [ + ['status' => 'completed', 'total' => 100, 'category' => 'books', 'created_at' => 1000], + ['status' => 'pending', 'total' => 50, 'category' => 'toys', 'created_at' => 2000], + ['status' => 'completed', 'total' => 30.5, 'category' => 'toys', 'created_at' => 3000], + ['status' => 'cancelled', 'total' => 70, 'category' => 'books', 'created_at' => 4000], + ['status' => 'completed', 'total' => 20, 'category' => 'books', 'created_at' => 5000], + ['status' => 'completed', 'total' => 999, 'category' => 'books', 'created_at' => 9000], + ]); + + $orderStats = $this->q('orders') + ->where(['created_at' => ['$gte' => 1500, '$lte' => 5000]]) + ->groupBy('status') + ->get(); + $stats = []; + foreach ($orderStats as $group) { + $stats[$group['_group']] = [$group['_count'], array_column($group['_items'], 'key')]; + } + ksort($stats); + $this->assertSame(['cancelled' => [1, [3]], 'completed' => [2, [2, 4]], 'pending' => [1, [1]]], $stats); + + $categoryRevenue = []; + foreach ($this->q('orders')->where(['status' => 'completed'])->get() as $order) { + $cat = $order['category']; + $categoryRevenue[$cat] = ($categoryRevenue[$cat] ?? 0) + $order['total']; + } + $this->assertSame(['books' => 1119, 'toys' => 30.5], $categoryRevenue); + } + + /** + * Content Management example + */ + public function testContentManagementPattern(): void + { + $articleDb = 'articles'; + $now = $this->now; + $this->noneDB->insert($articleDb, [ + ['title' => 'PHP Tutorial', 'content' => 'php basics', 'excerpt' => '', 'status' => 'published', 'published_at' => $now - 3600, 'tags' => ['php', 'tutorial', 'featured'], 'category' => 'tech'], + ['title' => 'JavaScript Guide', 'content' => 'js', 'excerpt' => 'php too', 'status' => 'draft', 'published_at' => null, 'tags' => ['javascript', 'featured'], 'category' => 'tech'], + ['title' => 'Science News', 'content' => 'atoms', 'excerpt' => 'PHP in labs', 'status' => 'published', 'published_at' => $now - 7200, 'tags' => ['science', 'featured'], 'category' => 'science'], + ['title' => 'Future', 'content' => 'later', 'excerpt' => '', 'status' => 'published', 'published_at' => $now + 86400, 'tags' => ['featured'], 'category' => 'tech'], + ['title' => 'Plain', 'content' => 'php', 'excerpt' => '', 'status' => 'published', 'published_at' => $now - 60, 'tags' => ['php'], 'category' => 'misc'], + ]); + + $articles = $this->q($articleDb) + ->where([ + 'status' => 'published', + 'published_at' => ['$lte' => $now], + 'tags' => ['$contains' => 'featured'] + ]) + ->sort('published_at', 'desc') + ->limit(10) + ->get(); + $this->assertSame(['PHP Tutorial', 'Science News'], array_column($articles, 'title')); + + $results = $this->q($articleDb) + ->where(['category' => ['$in' => ['tech', 'science']]]) + ->search('php', ['title', 'content', 'excerpt']) + ->sort('published_at', 'desc') + ->get(); + // null published_at sorts first ascending, last descending + $this->assertSame(['PHP Tutorial', 'Science News', 'JavaScript Guide'], array_column($results, 'title')); + } + + /** + * Performance tips: the order of the calls doesn't matter + */ + public function testFilterCallOrderDoesNotMatter(): void + { + $a = $this->q()->where(['salary' => ['$gt' => 40000]])->whereIn('department', ['Engineering', 'Design', 'Marketing'])->between('age', 20, 40)->get(); + $b = $this->q()->between('age', 20, 40)->whereIn('department', ['Engineering', 'Design', 'Marketing'])->where(['salary' => ['$gt' => 40000]])->get(); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Eve'], $this->names($a)); + $this->assertSame($a, $b); + + // the same with a field index on department: an index decides what is read, never what matches + $this->noneDB->createFieldIndex($this->testDbName, 'department'); + $this->assertSame($a, $this->q()->whereIn('department', ['Engineering', 'Design', 'Marketing'])->where(['salary' => ['$gt' => 40000]])->between('age', 20, 40)->get()); + } + + /** + * Complex query with several filter methods, sorted and limited + */ + public function testComplexQueryMultipleFilters(): void + { + $results = $this->q() + ->where(['status' => 'active']) + ->whereIn('role', ['admin', 'moderator']) + ->between('age', 25, 40) + ->sort('salary', 'desc') + ->limit(5) + ->get(); + + $this->assertSame(['Eve', 'John', 'Jane'], $this->names($results)); + } + + /** + * Operators with sort and limit + */ + public function testOperatorsWithSortLimit(): void + { + $results = $this->q() + ->where(['salary' => ['$gte' => 50000, '$lte' => 80000], 'status' => 'active']) + ->sort('salary', 'desc') + ->limit(3) + ->get(); + + // active in range: Eve 80000, John 75000, Jane 65000, Alice 60000 + $this->assertSame(['Eve', 'John', 'Jane'], $this->names($results)); + } + + /** + * Operators + whereIn + between + */ + public function testChainableCompatibility(): void + { + $results = $this->q() + ->where(['salary' => ['$gt' => 40000]]) + ->whereIn('department', ['Engineering', 'Design', 'Marketing']) + ->between('age', 20, 34) + ->get(); + + $this->assertSame(['John', 'Jane', 'Alice', 'Eve'], $this->names($results)); + } + + // ========== EDGE CASES ========== + + public function testEmptyResults(): void + { + $this->assertSame([], $this->q()->where(['name' => 'NonexistentPerson'])->get()); + } + + /** + * A null filter value matches only a field that exists with null + */ + public function testNullValues(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'NullTest', 'email' => null, 'age' => 30], + ['name' => 'NoEmail', 'age' => 31], + ]); + + $results = $this->q()->where(['email' => null])->get(); + + $this->assertSame([['name' => 'NullTest', 'email' => null, 'age' => 30, 'key' => 7]], + array_map(function ($r) { $k = $r['key']; unset($r['key']); return $r + ['key' => $k]; }, $results)); + } + + public function testSpecialCharactersInSearch(): void + { + $this->noneDB->insert($this->testDbName, ['name' => "O'Brien", 'email' => 'obrien@test.com', 'age' => 40]); + + $this->assertSame(["O'Brien"], $this->names($this->q()->where(['name' => "O'Brien"])->get())); + $this->assertSame(["O'Brien"], $this->names($this->q()->search("o'bri")->get())); + $this->assertSame(["O'Brien"], $this->names($this->q()->like('name', "^o'")->get())); + } + + /** + * Numeric strings compare numerically with operators but plain values are strict + */ + public function testNumericStringComparison(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'NumericTest', 'code' => '100', 'value' => 100], + ['name' => 'Small', 'code' => '9', 'value' => 9], + ]); + + $this->assertSame(['NumericTest'], $this->names($this->q()->where(['value' => ['$gt' => 50]])->get())); + // "100" > "50" numerically (lexically it would be smaller) + $this->assertSame(['NumericTest'], $this->names($this->q()->where(['code' => ['$gt' => '50']])->get())); + $this->assertSame(['NumericTest'], $this->names($this->q()->where(['code' => ['$gt' => 50]])->get())); + $this->assertSame([], $this->q()->where(['code' => 100])->get()); + $this->assertSame(['NumericTest'], $this->names($this->q()->where(['code' => '100'])->get())); + } +} diff --git a/tests/Integration/QueryEngineTest.php b/tests/Integration/QueryEngineTest.php new file mode 100644 index 0000000..92f09d2 --- /dev/null +++ b/tests/Integration/QueryEngineTest.php @@ -0,0 +1,279 @@ +db($config); + $db->createFieldIndex('qe', 'a'); + $db->createFieldIndex('qe', 'b'); + $db->insert('qe', array_map(function ($i) { + return ['a' => $i % 10, 'b' => 'b' . ($i % 7), 'c' => $i % 3 === 0 ? null : ($i % 2 ? 'x' : 2.5), 'n' => $i]; + }, range(0, $n - 1))); + return $db; + } + + /** @test */ + public function firstExistsAndLimitReadOnlyWhatTheyNeed(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 500]] as $config) { + $this->cleanTestDirectory(); + $db = $this->filled($config); + $q = function () use ($db) { return $db->query('qe'); }; + foreach ([$q()->where(['c' => 'x'])->limit(1), $q()->where(['n' => ['$gte' => 0]])->limit(5)] as $query) { + $read = $query->explain(true)['actual']['recordsRead']; + $this->assertIsInt($read); + $this->assertGreaterThan(0, $read); + $this->assertLessThanOrEqual(32, $read, 'first batch only'); + // Both are scans (c and n have no index) whose matches (n = 1; n = 0..4) are in the first + // batch of the first segment: exactly 32 records (QUERY.md: batches of 32, then 128, ...) + $this->assertSame(32, $read); + } + $this->assertSame(['a' => 1, 'b' => 'b1', 'c' => 'x', 'n' => 1, 'key' => 1], $q()->where(['c' => 'x'])->first()); + $this->assertTrue($q()->where(['n' => 1999])->exists()); + $this->assertFalse($q()->where(['n' => 5000])->exists()); + // Indexed: only the candidates are read + $explain = $q()->where(['a' => 3, 'b' => 'b3'])->explain(true); + $this->assertSame('index', $explain['access']); + $this->assertSame(count(array_filter(range(0, 1999), function ($i) { return $i % 10 === 3 && $i % 7 === 3; })), $explain['actual']['recordsRead']); + $this->assertSame($explain['actual']['returned'], $explain['actual']['recordsRead']); + // noneDB methods stop early too, and agree with find() + $this->assertSame($db->find('qe', ['c' => 'x'])[0], $db->first('qe', ['c' => 'x'])); + $this->assertSame(count($db->find('qe', ['b' => 'b2', 'c' => 2.5])), $db->count('qe', ['b' => 'b2', 'c' => 2.5])); + $this->assertTrue($db->exists('qe', ['a' => 9, 'n' => 1999])); + } + } + + /** @test */ + public function batchedScansNeverReadARecordTwice(): void + { + // Batches grow (32, 128, ...): a later batch that covers the rest of a segment must not read it all + foreach ([['shardingEnabled' => false], ['shardSize' => 150]] as $config) { + foreach ([33, 100, 160, 700] as $size) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('bt', array_map(function ($i) { return ['n' => $i]; }, range(0, $size - 1))); + foreach ([40, $size - 1, $size + 5] as $limit) { + $keys = array_column($db->query('bt')->where(['n' => ['$gte' => 0]])->limit($limit)->get(), 'key'); + $this->assertSame(range(0, min($limit, $size) - 1), $keys, json_encode([$config, $size, $limit])); + } + $this->assertSame($size, $db->query('bt')->where(['n' => ['$gte' => 0]])->count()); + } + } + } + + /** @test */ + public function readsDontListTheDirectory(): void + { + // Reads of an existing database find its files without listing a directory (a listing per read is + // O(files in dbDir)); only the missing-database path lists dbDir (docs/STORAGE_FORMAT.md §3.4, §12) + $db = $this->db(['shardSize' => 5, 'bufferEnabled' => true]); + $db->insert('dl', array_map(function ($i) { return ['n' => $i]; }, range(0, 29))); + $db->insert('dl', ['n' => 'buffered']); + $db->find('dl', ['n' => 1]); // flushes + $listings = function () { + $property = new \noneDB\Tests\InternalValue('directoryListings'); + return $property->getValue(); + }; + $before = $listings(); + for ($i = 0; $i < 5; $i++) { + $this->assertSame([3], array_column($db->find('dl', ['n' => 3]), 'key')); + $this->assertSame(31, $db->count('dl')); + $this->assertSame(1, $db->query('dl')->where(['n' => ['$gt' => 28]])->count()); + } + $this->assertSame($before, $listings(), 'reads of an existing database list nothing'); + // A missing database: "it lists dbDir directly, every time" (§3.4, foreign entries), once per read + for ($i = 0; $i < 5; $i++) { + $this->assertSame([], $db->find('missing', ['n' => 1])); + $this->assertSame(0, $db->count('missing')); + } + $this->assertSame($before + 10, $listings(), 'one listing per read of a missing database'); + for ($i = 0; $i < 5; $i++) { + $this->assertSame([3], array_column($db->find('dl', ['n' => 3]), 'key')); + $this->assertSame(31, $db->count('dl')); + } + $this->assertSame($before + 10, $listings(), 'still none for the existing database'); + } + + /** @test */ + public function topKEqualsTheFullSort(): void + { + mt_srand(12); + $db = $this->db(['shardingEnabled' => false]); + $pool = [null, true, false, 0, 1, -1, 2.5, '2', '10', '9', '1a', 'abc', '', [1], 'Z']; + $db->insert('tk', array_map(function ($i) use ($pool) { + return ['s' => $pool[mt_rand(0, count($pool) - 1)], 't' => $pool[mt_rand(0, count($pool) - 1)], 'g' => $i % 4]; + }, range(0, 2999))); + for ($round = 0; $round < 40; $round++) { + $field = mt_rand(0, 1) ? 's' : 't'; + $order = mt_rand(0, 1) ? 'asc' : 'desc'; + $offset = mt_rand(0, 20); + $limit = mt_rand(1, 60); + $all = $db->query('tk')->where(['g' => ['$lte' => 2]])->sort($field, $order)->sort('g', 'desc')->get(); + $top = $db->query('tk')->where(['g' => ['$lte' => 2]])->sort($field, $order)->sort('g', 'desc')->offset($offset)->limit($limit); + $this->assertSame('top-k', $top->explain()['sort']); + $this->assertSame(array_slice($all, $offset, $limit), $top->get(), "$field $order $offset $limit"); + } + } + + /** @test */ + public function plansEqualAFullScan(): void + { + mt_srand(2026); + $values = [0, 1, 2, 3, '1', 1.0, 2.5, null, true, 'x', 'y', 'b3']; + foreach ([['shardingEnabled' => false], ['shardSize' => 40]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->createFieldIndex('pl', 'a'); + $db->createFieldIndex('pl', 'b'); + $rows = []; + for ($i = 0; $i < 300; $i++) { + $row = ['a' => $values[mt_rand(0, count($values) - 1)], 'b' => $values[mt_rand(0, count($values) - 1)], 'n' => $i]; + if (mt_rand(0, 9) === 0) { + unset($row['a']); + } + $rows[] = $row; + } + $db->insert('pl', $rows); + $db->delete('pl', ['b' => 'y']); + $plain = $this->db($config + ['fieldIndexEnabled' => false]); + $leaf = function () use ($values) { + $field = ['a', 'b', 'n', 'key'][mt_rand(0, 3)]; + $v = $values[mt_rand(0, count($values) - 1)]; + switch (mt_rand(0, 5)) { + case 0: return [$field => $field === 'key' ? mt_rand(0, 320) : $v]; + case 1: return [$field => ['$in' => [$v, $values[mt_rand(0, count($values) - 1)]]]]; + case 2: return [$field => ['$gte' => $v]]; + case 3: return [$field => ['$lt' => $v, '$gt' => $values[mt_rand(0, count($values) - 1)]]]; + case 4: return [$field => ['$ne' => $v]]; + default: return [$field => ['$eq' => $v]]; + } + }; + for ($round = 0; $round < 120; $round++) { + $build = function (\noneDB $on) use ($leaf, $round) { + mt_srand(9000 + $round); + $q = $on->query('pl'); + for ($w = mt_rand(0, 2); $w > 0; $w--) { + $q->where($leaf() + (mt_rand(0, 2) === 0 ? $leaf() : [])); + } + for ($o = mt_rand(0, 2) === 0 ? mt_rand(1, 2) : 0; $o > 0; $o--) { + $q->orWhere(mt_rand(0, 9) === 0 ? [] : $leaf()); + } + if (mt_rand(0, 3) === 0) { + $q->whereIn(mt_rand(0, 1) ? 'a' : 'b', [1, 'x', 2.5]); + } + if (mt_rand(0, 3) === 0) { + $q->between('a', 0, mt_rand(1, 3)); + } + if (mt_rand(0, 4) === 0) { + $q->whereNot(['b' => 'x']); + } + if (mt_rand(0, 3) === 0) { + $q->sort('n', mt_rand(0, 1) ? 'asc' : 'desc')->limit(mt_rand(1, 20)); + } + return $q; + }; + $expected = $build($plain)->get(); + $this->assertSame($expected, $build($db)->get(), json_encode([$config, $round])); + $this->assertSame(count($expected), $build($db)->count()); + $this->assertSame($expected[0] ?? null, $build($db)->first()); + $this->assertSame(!empty($expected), $build($db)->exists()); + $explain = $build($db)->explain(true); + $this->assertSame(count($expected), $explain['actual']['returned']); + } + } + } + + /** @test */ + public function explainBuildsNothingAndShowsShardSkips(): void + { + $db = $this->db(['shardSize' => 500]); + $db->createFieldIndex('ex', 'u'); + $db->insert('ex', array_map(function ($i) { return ['u' => 'u' . $i, 'v' => $i % 5]; }, range(0, 1999))); + $explain = $db->query('ex')->where(['u' => 'u1234'])->explain(); + $this->assertSame(['total' => 4, 'skipped' => 3, 'indexed' => 1, 'scanned' => 0], $explain['segments']); + $this->assertSame([['segment' => 0, 'access' => 'skip', 'candidates' => 0], ['segment' => 1, 'access' => 'skip', 'candidates' => 0], + ['segment' => 2, 'access' => 'index', 'candidates' => 1], ['segment' => 3, 'access' => 'skip', 'candidates' => 0]], $explain['plan'], 'key 1234 is in shard 2'); + $this->assertSame([true, false, ['eq', 'u', 'u1234']], [$explain['sharded'], $explain['planTruncated'], $explain['filter']]); + $this->assertSame('index', $explain['access']); + $this->assertSame(1, $explain['estimatedRead']); + $this->assertTrue($explain['residual']); + $this->assertSame('scan', $db->query('ex')->where(['v' => 3])->explain()['access']); + $this->assertSame(2000, $db->query('ex')->where(['v' => 3])->explain()['estimatedRead'], 'every record of the 4 shards'); + $this->assertSame('none', $db->query('ex')->where(['u' => 'nothing'])->explain()['access']); + $this->assertSame(['total' => 4, 'skipped' => 4, 'indexed' => 0, 'scanned' => 0], $db->query('ex')->where(['u' => 'nothing'])->explain()['segments']); + // An index a query would build first: explain() counts it as a scan and builds nothing + $db->createFieldIndex('ex', 'v'); + foreach (glob($this->dbPath('ex', 'gen') . 'indexes/s*.v.fx') as $file) { + unlink($file); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 500]); + $this->assertSame('scan', $db->query('ex')->where(['v' => 3])->explain()['access']); + $this->assertSame([], glob($this->dbPath('ex', 'gen') . 'indexes/s*.v.fx')); + $this->assertCount(400, $db->query('ex')->where(['v' => 3])->get()); + $this->assertSame(range(3, 1998, 5), array_column($db->query('ex')->where(['v' => 3])->get(), 'key')); + // The query built the shard indexes: now explain() shows them as indexed + $this->assertCount(4, glob($this->dbPath('ex', 'gen') . 'indexes/s*.v.fx')); + $this->assertSame(['index', 400], [$db->query('ex')->where(['v' => 3])->explain()['access'], $db->query('ex')->where(['v' => 3])->explain()['estimatedRead']]); + } + + /** @test */ + public function joinsMatchByIdentityThroughTheForeignIndex(): void + { + foreach ([true, false] as $indexed) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + if ($indexed) { + $db->createFieldIndex('owners', 'ref'); + } + $db->insert('owners', [ + ['ref' => 'Ada', 'o' => 'string'], ['ref' => true, 'o' => 'bool'], ['ref' => 1, 'o' => 'int1'], + ['ref' => 1.5, 'o' => 'float'], ['ref' => 5, 'o' => 'five'], ['ref' => '007', 'o' => 'zeros'], + ['ref' => 7, 'o' => 'seven'], ['ref' => '5', 'o' => 'five again'], + ]); + $db->insert('items', [ + ['ref' => 'Ada'], ['ref' => true], ['ref' => '1'], ['ref' => 1.5], ['ref' => 5.0], ['ref' => '5'], + ['ref' => '007'], ['ref' => 7], ['ref' => '1.5'], ['ref' => false], ['ref' => null], + ]); + $joined = $db->query('items')->join('owners', 'ref', 'ref', 'owner')->get(); + $names = array_map(function ($r) { return $r['owner']['o'] ?? null; }, $joined); + $this->assertSame(['string', 'bool', 'int1', 'float', 'five again', 'five again', 'zeros', 'seven', null, null, null], $names, + 'indexed ' . var_export($indexed, true) . ': "5", 5.0 and 5 are one key (the last record wins); true only true; "007" not 7'); + $this->assertSame('in-lookup', $db->query('items')->join('owners', 'ref', 'ref', 'owner')->explain()['joins'][0]['strategy']); + $this->assertSame([['db' => 'owners', 'on' => 'ref = ref', 'strategy' => 'in-lookup', 'when' => 'after-slice']], + $db->query('items')->join('owners', 'ref', 'ref', 'owner')->explain()['joins']); + } + } + + /** @test */ + public function aJoinReadsBothDatabasesInOneState(): void + { + $db = $this->db(['shardingEnabled' => false, 'lockTimeout' => 0.5]); + $db->insert('users', [['name' => 'a'], ['name' => 'b']]); + $db->insert('orders', [['user' => 0], ['user' => 1]]); + $blocked = null; + // While the join runs, a writer of the foreign database waits until the join is done + $this->setFaultHook(function ($point) use (&$blocked) { + static $done = false; + if ($point === 'snapshot:held' && !$done) { + $done = true; + $blocked = $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false, 'lockTimeout' => 0.3]) . '; + try { $db->insert("users", ["name" => "late"]); echo "wrote"; } catch (RuntimeException $e) { echo "waited"; }'); + } + }); + $rows = $db->query('orders')->join('users', 'user', 'key', 'u')->get(); + $this->setFaultHook(null); + $this->assertSame('waited', $blocked); + $this->assertSame(['a', 'b'], array_map(function ($r) { return $r['u']['name']; }, $rows)); + } +} diff --git a/tests/Integration/QueryHardeningTest.php b/tests/Integration/QueryHardeningTest.php new file mode 100644 index 0000000..f3a4810 --- /dev/null +++ b/tests/Integration/QueryHardeningTest.php @@ -0,0 +1,137 @@ +db(); + $values = [1, '1', 1.0, 0, -0.0, 0.0, true, false, [1, 'x'], [1, 'x'], ['a' => 1, 'b' => 2], ['b' => 2, 'a' => 1], [1.0], [1], null, '', 'x', 'x']; + $db->insert('dist', array_map(function ($v) { return ['v' => $v]; }, $values)); + $db->insert('dist', ['other' => 1]); + $expected = []; + foreach ($values as $v) { + if ($v !== null && !in_array($v, $expected, true)) { + $expected[] = $v; + } + } + $this->assertSame($expected, $db->distinct('dist', 'v')); + $this->assertSame($expected, $db->query('dist')->distinct('v')); + + // Encoding must not collide (length-prefixed keys) + $collide = [ + ['x1' => 'ab', 'y' => 'AAAAsz12:sBBBBBBBBBBB'], + ['x' => 'absy22:sAAAA', 'z' => 'BBBBBBBBBBB'], + ]; + $db->insert('coll', array_map(function ($v) { return ['v' => $v]; }, $collide)); + $this->assertSame($collide, $db->distinct('coll', 'v')); + $this->assertSame($collide, $db->query('coll')->distinct('v')); + $this->assertCount(2, $db->query('coll')->groupBy('v')->get(), 'groups use the same identity'); + } + + /** + * @test + * A timing over 100k records in a process of its own: neither its time nor its memory depends on what the + * tests before it left in the process + * @runInSeparateProcess + * @preserveGlobalState disabled + */ + public function distinctIsLinear(): void + { + // C-1: in_array over the values seen so far made 100k distinct values take minutes. Measured through + // distinct(), which reads the records segment by segment and keeps a hash set of the values seen + $db = $this->db(['shardSize' => 10000]); + $time = function (int $n) use ($db) { + $name = 'distinct' . $n; + for ($from = 0; $from < $n; $from += 10000) { + $db->insert($name, array_map(function ($i) { return ['v' => 'value-' . $i]; }, range($from, $from + 9999))); + } + $best = INF; + for ($run = 0; $run < 3; $run++) { + $t = hrtime(true); + $this->assertCount($n, $db->distinct($name, 'v')); + $best = min($best, hrtime(true) - $t); + } + return $best; + }; + $ratio = $time(100000) / max(1, $time(10000)); + $this->assertLessThan(30, $ratio, 'linear: ~10, quadratic: ~100 (relative to itself, no absolute time)'); + } + + /** @test */ + public function sortOrderIsTotalForMixedStrings(): void + { + // C-3: "9" < "10" < "1a" < "9" was a cycle: the result depended on the insertion order + $db = $this->db(); + $values = ['9', '10', '1a', '2', '20', 'alpha', 5, 9.5, ' 7', true, null, ['x']]; + $expected = null; + mt_srand(5); + for ($i = 0; $i < 60; $i++) { + $shuffled = $values; + shuffle($shuffled); + $rows = array_map(function ($v) { return ['v' => $v]; }, $shuffled); + $sorted = array_column($db->sort($rows, 'v'), 'v'); + $expected = $expected ?? $sorted; + $this->assertSame($expected, $sorted, 'independent of the input order'); + } + // null < bool < numbers < numeric strings by number < other strings < arrays + $this->assertSame([null, true, 5, 9.5, '2', ' 7', '9', '10', '20', '1a', 'alpha', ['x']], $expected); + // Equal sort keys ("7 " and " 7" are both 7) keep their input order (stable) + $this->assertSame(['7 ', ' 7'], array_column($db->sort([['v' => '7 '], ['v' => ' 7']], 'v'), 'v')); + $this->assertSame([' 7', '7 '], array_column($db->sort([['v' => ' 7'], ['v' => '7 ']], 'v'), 'v')); + + $db->insert('mx', array_map(function ($v) { return ['v' => $v]; }, ['9', '10', '1a', 'alpha', '2'])); + $this->assertSame('alpha', $db->query('mx')->max('v')); + $this->assertSame('2', $db->query('mx')->min('v')); + $this->assertSame(['2', '9', '10', '1a', 'alpha'], array_column($db->query('mx')->sort('v')->get(), 'v')); + // sort + limit + update hits exactly the records the sort shows + $db->query('mx')->sort('v', 'desc')->limit(2)->update(['top' => true]); + $this->assertSame(['1a', 'alpha'], array_column($db->find('mx', ['top' => true]), 'v')); + } + + /** @test */ + public function whereNotAppliesOperators(): void + { + // C-7: operator specs were compared with ===, so nothing was excluded + $db = $this->db(); + $db->insert('wn', [['age' => 10, 'r' => 'a'], ['age' => 20, 'r' => 'b'], ['age' => 70, 'r' => 'a'], ['r' => 'c']]); + $this->assertSame([10, 70, null], array_map(function ($r) { return $r['age'] ?? null; }, + $db->query('wn')->whereNot(['age' => ['$gte' => 18, '$lt' => 65]])->get()), 'AND within the field'); + $this->assertSame(['c'], array_column($db->query('wn')->whereNot(['r' => 'a', 'age' => ['$gt' => 15]])->get(), 'r'), + 'any matching field excludes'); + try { + $db->query('wn')->whereNot(['age' => ['$between' => 1]]); + $this->fail('unknown operator'); + } catch (\InvalidArgumentException $e) { + $this->assertStringStartsWith("noneDB: unknown operator '\$between'", $e->getMessage()); + } + // A builder delete removes exactly what the query shows (the record without "age" is not "age < 65") + $shown = array_column($db->query('wn')->whereNot(['age' => ['$lt' => 65]])->get(), 'key'); + $db->query('wn')->whereNot(['age' => ['$lt' => 65]])->delete(); + $this->assertSame([2, 3], $shown); + $this->assertSame([0, 1], array_column($db->find('wn', 0), 'key')); + } + + /** @test */ + public function nanMatchesNothing(): void + { + // C-10: NAN <=> x is 1 both ways, so $gt NAN matched every number + $db = $this->db(); + $db->insert('nan', [['p' => 1], ['p' => 5.5], ['p' => '7'], ['p' => 'x']]); + foreach (['$gt', '$gte', '$lt', '$lte'] as $op) { + $this->assertSame([], $db->query('nan')->where(['p' => [$op => NAN]])->get(), $op); + } + $this->assertSame([], $db->query('nan')->between('p', NAN, 10)->get()); + $this->assertSame([], $db->between('nan', 'p', NAN, 10)); + $db->query('nan')->where(['p' => ['$gt' => NAN]])->delete(); + $this->assertCount(4, $db->find('nan', 0), 'a delete with a NAN bound deletes nothing'); + } +} diff --git a/tests/Integration/QueryMethodsTest.php b/tests/Integration/QueryMethodsTest.php new file mode 100644 index 0000000..692e053 --- /dev/null +++ b/tests/Integration/QueryMethodsTest.php @@ -0,0 +1,308 @@ + 'John', 'email' => 'john@gmail.com', 'city' => 'Istanbul', 'age' => 25], + ['name' => 'Jane', 'email' => 'jane@yahoo.com', 'city' => 'Ankara', 'age' => 30], + ['name' => 'Bob', 'email' => 'bob@gmail.com', 'city' => 'Istanbul', 'age' => 35], + ['name' => 'Alice', 'email' => 'alice@hotmail.com', 'city' => 'Izmir', 'age' => 28], + ['name' => 'Johnson', 'email' => 'johnson@gmail.com', 'city' => 'Ankara', 'age' => 40], + ]; + + $this->noneDB->insert($this->testDbName, $data); + } + + // ========================================== + // DISTINCT TESTS + // ========================================== + + /** + * @test + */ + public function distinctReturnsUniqueValues(): void + { + $result = $this->noneDB->distinct($this->testDbName, 'city'); + + // In order of first appearance (API.md distinct()) + $this->assertSame(['Istanbul', 'Ankara', 'Izmir'], $result); + } + + /** + * @test + */ + public function distinctReturnsAllValuesWhenUnique(): void + { + $result = $this->noneDB->distinct($this->testDbName, 'name'); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Johnson'], $result); + } + + /** + * @test + */ + public function distinctReturnsEmptyForNonExistentField(): void + { + $result = $this->noneDB->distinct($this->testDbName, 'nonexistent'); + + $this->assertSame([], $result); + } + + /** + * @test + */ + public function distinctReturnsEmptyForEmptyDB(): void + { + $this->noneDB->createDB('emptydb'); + $result = $this->noneDB->distinct('emptydb', 'field'); + + $this->assertSame([], $result); + } + + /** + * @test + */ + public function distinctPreservesDataTypes(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Test', 'active' => true], + ['name' => 'Test2', 'active' => false], + ['name' => 'Test3', 'active' => true], + ]); + + $result = $this->noneDB->distinct($this->testDbName, 'active'); + + $this->assertSame([true, false], $result); + // Distinct by type: 1, 1.0 and "1" are three values; null is left out + $this->noneDB->insert($this->testDbName, [['t' => 1], ['t' => '1'], ['t' => 1.0], ['t' => null], ['t' => 1], ['t' => true]]); + $this->assertSame([1, '1', 1.0, true], $this->noneDB->distinct($this->testDbName, 't')); + } + + // ========================================== + // LIKE TESTS + // ========================================== + + /** + * @test + */ + public function likeFindsContainingPattern(): void + { + $result = $this->noneDB->like($this->testDbName, 'email', 'gmail'); + + $this->assertSame(['John', 'Bob', 'Johnson'], array_column($result, 'name')); + $this->assertSame([0, 2, 4], array_column($result, 'key'), 'records with their key, in key order'); + } + + /** + * @test + */ + public function likeStartsWithPattern(): void + { + $result = $this->noneDB->like($this->testDbName, 'name', '^John'); + + $this->assertSame(['John', 'Johnson'], array_column($result, 'name')); + $this->assertSame([], $this->noneDB->like($this->testDbName, 'name', '^ohn'), 'the start is anchored'); + } + + /** + * @test + */ + public function likeEndsWithPattern(): void + { + $result = $this->noneDB->like($this->testDbName, 'email', 'com$'); + + $this->assertSame(['John', 'Jane', 'Bob', 'Alice', 'Johnson'], array_column($result, 'name')); + $this->assertSame([], $this->noneDB->like($this->testDbName, 'email', 'gmail$'), 'the end is anchored'); + $this->assertSame(['Bob'], array_column($this->noneDB->like($this->testDbName, 'email', '^bob@gmail.com$'), 'name'), 'both anchors: the whole value'); + } + + /** + * @test + */ + public function likeIsCaseInsensitive(): void + { + $result = $this->noneDB->like($this->testDbName, 'name', 'JOHN'); + + $this->assertSame(['John', 'Johnson'], array_column($result, 'name')); + $this->assertSame(['Alice'], array_column($this->noneDB->like($this->testDbName, 'name', 'aLiCe'), 'name')); + } + + /** + * @test + */ + public function likeReturnsEmptyForNoMatch(): void + { + $result = $this->noneDB->like($this->testDbName, 'email', 'outlook'); + + $this->assertSame([], $result); + } + + /** + * @test + */ + public function likeHandlesSpecialChars(): void + { + $this->noneDB->insert($this->testDbName, [ + ['name' => 'Test', 'email' => 'test+tag@gmail.com'], + ]); + + $result = $this->noneDB->like($this->testDbName, 'email', 'test+tag'); + + $this->assertSame(['test+tag@gmail.com'], array_column($result, 'email')); + // Every character but a leading ^ and a trailing $ is literal: "." and "+" are no regex operators + $this->noneDB->insert($this->testDbName, [['name' => 'T2', 'email' => 'testtttag@gmail.com'], ['name' => 'T3', 'email' => 'aXb']]); + $this->assertSame(['test+tag@gmail.com'], array_column($this->noneDB->like($this->testDbName, 'email', 'test+tag'), 'email')); + $this->assertSame([], $this->noneDB->like($this->testDbName, 'email', 'a.b')); + $this->assertSame([], $this->noneDB->like($this->testDbName, 'email', '.*')); + $this->assertSame(['aXb'], array_column($this->noneDB->like($this->testDbName, 'email', '^axb$'), 'email')); + } + + /** + * @test + */ + public function likeWorksWithNonExistentField(): void + { + $result = $this->noneDB->like($this->testDbName, 'nonexistent', 'pattern'); + + $this->assertSame([], $result); + } + + // ========================================== + // BETWEEN TESTS + // ========================================== + + /** + * @test + */ + public function betweenFindsRecordsInRange(): void + { + $result = $this->noneDB->between($this->testDbName, 'age', 25, 30); + + $this->assertSame(['John', 'Jane', 'Alice'], array_column($result, 'name')); // 25, 30, 28 in key order + $this->assertSame([0, 1, 3], array_column($result, 'key')); + } + + /** + * @test + */ + public function betweenIncludesBoundaries(): void + { + $result = $this->noneDB->between($this->testDbName, 'age', 25, 25); + + $this->assertSame(['John'], array_column($result, 'name')); + $this->assertSame(25, $result[0]['age']); + $this->assertSame(['Jane', 'Bob', 'Johnson'], array_column($this->noneDB->between($this->testDbName, 'age', 30, 40), 'name'), 'both ends inclusive'); + } + + /** + * @test + */ + public function betweenReturnsEmptyForNoMatch(): void + { + $result = $this->noneDB->between($this->testDbName, 'age', 50, 60); + + $this->assertSame([], $result); + $this->assertSame([], $this->noneDB->between($this->testDbName, 'age', 40, 25), 'min above max: nothing'); + } + + /** + * @test + */ + public function betweenWithFilter(): void + { + $result = $this->noneDB->between($this->testDbName, 'age', 25, 35, ['city' => 'Istanbul']); + + $this->assertSame(['John', 'Bob'], array_column($result, 'name')); // John (25) and Bob (35) + $this->assertFalse($this->noneDB->between($this->testDbName, 'age', 25, 35, 'Istanbul'), 'an invalid filter: false'); + // A key list keeps its order, as in find() (API.md) + $this->assertSame(['Jane', 'John', 'Bob'], array_column($this->noneDB->between($this->testDbName, 'age', 25, 35, ['key' => [1, '0', 2]]), 'name')); + $this->assertSame(['Jane', 'John'], array_column($this->noneDB->between($this->testDbName, 'age', 25, 30, ['key' => [1, '0', 2]]), 'name')); + } + + /** + * @test + */ + public function betweenWorksWithStrings(): void + { + $result = $this->noneDB->between($this->testDbName, 'name', 'A', 'C'); + + // Strings compare lexically (byte order): Bob and Alice, in key order + $this->assertSame(['Bob', 'Alice'], array_column($result, 'name')); + $this->assertSame([], $this->noneDB->between($this->testDbName, 'name', 'a', 'c'), 'upper case sorts before lower case'); + } + + /** + * @test + */ + public function betweenWithNonExistentField(): void + { + $result = $this->noneDB->between($this->testDbName, 'nonexistent', 0, 100); + + $this->assertSame([], $result); + } + + /** + * @test + */ + public function betweenWithEmptyDB(): void + { + $this->noneDB->createDB('emptydb'); + $result = $this->noneDB->between('emptydb', 'age', 0, 100); + + $this->assertSame([], $result); + } + + /** + * @test + */ + public function betweenIsTypeAware(): void + { + // Numbers and numeric strings numerically; null, booleans, arrays and text against numbers never match + $this->noneDB->insert($this->testDbName, [ + ['name' => 'S', 'age' => '27'], ['name' => 'T', 'age' => true], ['name' => 'N', 'age' => null], + ['name' => 'A', 'age' => [27]], ['name' => 'X', 'age' => 'abc'], ['name' => 'F', 'age' => 26.5], + ]); + $this->assertSame(['Alice', 'S', 'F'], array_column($this->noneDB->between($this->testDbName, 'age', 26, 29), 'name')); + $this->assertSame(['Alice', 'S', 'F'], array_column($this->noneDB->query($this->testDbName)->between('age', 26, 29)->get(), 'name')); + } + + /** + * @test + */ + public function likeNeverMatchesBooleansAndRejectsNonScalarPatterns(): void + { + $this->noneDB->insert($this->testDbName, [['name' => 'B1', 'flag' => true], ['name' => 'B2', 'flag' => 1], ['name' => 'B3', 'flag' => '1']]); + $this->assertSame(['B2', 'B3'], array_column($this->noneDB->like($this->testDbName, 'flag', '1'), 'name')); + $this->assertFalse($this->noneDB->like($this->testDbName, 'name', null)); + $this->assertFalse($this->noneDB->like($this->testDbName, 'name', ['J'])); + $this->assertSame(['Jane'], array_column($this->noneDB->like($this->testDbName, 'age', 30), 'name'), 'numbers are matched as text'); + } + + /** + * @test + */ + public function queryMethodsOnAMissingDatabase(): void + { + $before = glob($this->testDbDir . '*'); + $this->assertSame([], $this->noneDB->distinct('ghostdb', 'a')); + $this->assertSame([], $this->noneDB->like('ghostdb', 'a', 'x')); + $this->assertSame([], $this->noneDB->between('ghostdb', 'a', 1, 2)); + $this->assertSame($before, glob($this->testDbDir . '*'), 'reads create nothing'); + $strict = new \noneDB(['secretKey' => 'test_secret_key_for_unit_tests', 'dbDir' => $this->testDbDir, 'autoCreateDB' => false]); + $this->assertFalse($strict->distinct('ghostdb', 'a')); + $this->assertFalse($strict->like('ghostdb', 'a', 'x')); + $this->assertFalse($strict->between('ghostdb', 'a', 1, 2)); + } +} diff --git a/tests/Regression/QueryRegressionTest.php b/tests/Integration/QueryRegressionTest.php similarity index 73% rename from tests/Regression/QueryRegressionTest.php rename to tests/Integration/QueryRegressionTest.php index 1d1c9c7..6e2c2dd 100644 --- a/tests/Regression/QueryRegressionTest.php +++ b/tests/Integration/QueryRegressionTest.php @@ -1,6 +1,8 @@ 'Bob', 'age' => '30', 'city' => 'Ankara', 'score' => 90], ['name' => 'Charlie', 'age' => 35, 'city' => 'Istanbul', 'score' => null], ['name' => 'abc', 'age' => 'abc', 'city' => 'Izmir'], - ['name' => 'AYDOĞDU', 'age' => [1], 'city' => 'İzmir'], + ['name' => 'AĞUSTOS', 'age' => [1], 'city' => 'İzmir'], ]); } @@ -37,8 +39,13 @@ public function removeFieldsOnlyRemovesTheGivenFields(): void $this->assertArrayNotHasKey('_distance', $record); $this->assertArrayNotHasKey('dept', $record); } - $this->assertSame(12, $db->query('rf')->except(['key'])->removeFields(['name'])['n']); - $this->assertArrayHasKey('pw', $db->find('rf', ['key' => 11])[0]); + $this->assertSame(['n' => 12, 'fields_removed' => ['name']], $db->query('rf')->except(['key'])->removeFields(['name'])); + foreach ($db->find('rf', 0) as $i => $record) { + $this->assertSame($i, $record['key']); + $this->assertArrayNotHasKey('name', $record); + $this->assertSame('secret' . $i, $record['pw'], 'other fields keep their values'); + } + $this->assertSame([['id' => 'x', 'title' => 'X', 'key' => 0]], $db->find('dept', 0), 'the joined database is untouched'); } /** @test Y-15 */ @@ -60,9 +67,14 @@ public function nullFiltersBehaveTheSameEverywhere(): void $name = empty($config) ? 'nu' : 'nus'; $db = $this->db($config); $this->seed($db, $name); - $this->assertCount(1, $db->find($name, ['score' => null])); - $this->assertSame(1, $db->update($name, [['score' => null], ['set' => ['flag' => 1]]])['n']); - $this->assertSame(1, $db->delete($name, ['score' => null])['n']); + $this->assertSame(!empty($config), $db->getShardInfo($name)['sharded']); + // null matches only a field that exists with the value null: Charlie (abc and AĞUSTOS have no score) + $this->assertSame(['Charlie'], array_column($db->find($name, ['score' => null]), 'name')); + $this->assertSame(['n' => 1], $db->update($name, [['score' => null], ['set' => ['flag' => 1]]])); + $this->assertSame(['Charlie'], array_column($db->find($name, ['flag' => 1]), 'name')); + $this->assertSame(['Charlie'], array_column($db->query($name)->where(['score' => null])->get(), 'name')); + $this->assertSame(['n' => 1], $db->delete($name, ['score' => null])); + $this->assertSame(['Alice', 'Bob', 'abc', 'AĞUSTOS'], array_column($db->find($name, 0), 'name')); } } @@ -93,18 +105,19 @@ public function likePatternsAreLiteralAndSafe(): void $this->assertSame(['http://a.com'], array_column($db->query('lk')->like('url', '^http://')->get(), 'url')); $this->assertSame(['http://a.com'], array_column($db->query('lk')->where(['url' => ['$like' => '^http://']])->get(), 'url')); $this->assertSame(['http://a.com', 'https://b.com'], array_column($db->query('lk')->where(['url' => ['$regex' => '^https?://[a-z]']])->get(), 'url')); - $this->assertCount(2, $db->query('lk')->where(['url' => ['$regex' => 'a.com|b.com']])->get(), 'slashes inside $regex are fine'); + $this->assertSame(['http://a.com', 'https://b.com'], array_column($db->query('lk')->where(['url' => ['$regex' => 'a.com|b.com']])->get(), 'url'), 'slashes inside $regex are fine'); + $this->assertSame(['http://a.com', 'https://b.com', 'h1tp://c'], array_column($db->query('lk')->where(['url' => ['$regex' => '^H.TP']])->get(), 'url'), '$regex is case-insensitive'); try { $db->query('lk')->where(['url' => ['$regex' => '(unclosed']])->get(); $this->fail('invalid regex must be reported'); } catch (\InvalidArgumentException $e) { - $this->assertStringContainsString('invalid', $e->getMessage()); + $this->assertSame("noneDB: invalid \$regex pattern '(unclosed'", $e->getMessage()); } try { $db->query('lk')->where(['url' => ['$like' => ['not', 'a', 'string']]])->get(); $this->fail('non-string $like must be reported'); } catch (\InvalidArgumentException $e) { - $this->assertStringContainsString('like', $e->getMessage()); + $this->assertSame('noneDB: like pattern must be a string', $e->getMessage()); } } @@ -117,7 +130,8 @@ public function whereAndOrWhereLogic(): void $this->assertSame(['Alice', 'Charlie'], array_column($db->query('q')->where(['age' => ['$gt' => 20]])->where(['age' => ['$lt' => 40]])->where(['city' => 'Istanbul'])->get(), 'name')); $this->assertSame(['Alice', 'Bob'], array_column($db->query('q')->where(['age' => ['$gt' => 20]])->where(['age' => ['$lt' => 31]])->get(), 'name')); $db->insert('num', [['100' => 'x'], ['100' => 'y']]); - $this->assertCount(1, $db->query('num')->where(['100' => 'x'])->where(['100' => ['$ne' => 'z']])->get(), 'numeric field names'); + $this->assertSame([['100' => 'x', 'key' => 0]], $db->query('num')->where(['100' => 'x'])->where(['100' => ['$ne' => 'z']])->get(), 'numeric field names'); + $this->assertSame([1], array_column($db->query('num')->where(['100' => ['$ne' => 'x']])->get(), 'key')); $db->insert('sp', [ ['n' => 'a', 'loc' => ['type' => 'Point', 'coordinates' => [29, 41]]], @@ -148,7 +162,8 @@ public function sortOrderMatchesReferenceOnMixedTypes(): void { $db = $this->db(); mt_srand(77); - $pool = [null, true, false, 0, 1, -1, 2.5, -0.5, 10, '10', '9', '1e1', 'abc', 'ABC', 'b', '', ['x'], ['a' => 1], [1, 2], 3.0]; + $pool = [null, true, false, 0, 1, -1, 2.5, -0.5, 10, '10', '9', '1e1', 'abc', 'ABC', 'b', '', ['x'], ['a' => 1], [1, 2], 3.0, + '1a', '9 ', ' 7', '1e3', '0x1A', '-0', 'a10']; $rows = []; for ($i = 0; $i < 120; $i++) { $row = ['i' => $i, 'g' => mt_rand(0, 2)]; @@ -159,19 +174,28 @@ public function sortOrderMatchesReferenceOnMixedTypes(): void } $db->insert('mix', $rows); - // Reference: missing/null < booleans < numbers < strings < arrays; same kind with <=> (arrays by JSON); ties by insertion order - $kind = function ($v) { + // Reference (order B): missing/null < booleans < numbers < numeric strings (by their number; + // trailing whitespace allowed on every PHP version) < other strings (byte order) < arrays (by JSON); + // ties by insertion order. Comparing all strings with <=> would not be transitive ("9" < "10" < "1a" < "9"). + $numericString = function ($v) { + return is_string($v) && is_numeric(rtrim($v, " \t\n\r\v\f")); + }; + $kind = function ($v) use ($numericString) { if ($v === null) return 0; if (is_bool($v)) return 1; if (is_int($v) || is_float($v)) return 2; - if (is_string($v)) return 3; - return 4; + if ($numericString($v)) return 3; + if (is_string($v)) return 4; + return 5; }; $cmp = function ($a, $b) use ($kind) { $ka = $kind($a); $kb = $kind($b); if ($ka !== $kb) return $ka <=> $kb; - return $ka === 4 ? strcmp(json_encode($a), json_encode($b)) <=> 0 : $a <=> $b; + if ($ka === 3) return (rtrim($a, " \t\n\r\v\f") + 0) <=> (rtrim($b, " \t\n\r\v\f") + 0); + if ($ka === 4) return strcmp($a, $b) <=> 0; + if ($ka === 5) return strcmp(json_encode($a), json_encode($b)) <=> 0; + return $a <=> $b; }; $reference = function (array $keys) use ($rows, $cmp) { $sorted = $rows; @@ -222,7 +246,11 @@ public function countMatchesGet(): void $this->assertSame(2, $db->query('ct')->groupBy('g')->count(), 'groups'); $this->assertSame(2, $db->query('ct')->limit(2)->count()); $this->assertSame(3, $db->query('ct')->offset(2)->count()); - $this->assertSame(count($db->query('ct')->withinDistance('loc', 0, 0, 350000)->get()), $db->query('ct')->withinDistance('loc', 0, 0, 350000)->count()); + // Points at 0, 1, 2, 3 and 4 degrees on the equator: 111.2 km per degree, so 0..3 are within 350 km + $this->assertSame([0, 1, 2, 3], array_column($db->query('ct')->withinDistance('loc', 0, 0, 350000)->get(), 'i')); + $this->assertSame(4, $db->query('ct')->withinDistance('loc', 0, 0, 350000)->count()); + $this->assertSame(2, $db->query('ct')->withinDistance('loc', 0, 0, 350000)->where(['g' => 0])->count()); + $this->assertSame(3, $db->query('ct')->withinDistance('loc', 0, 0, 350000)->offset(1)->count()); } /** @test O-03 */ @@ -236,7 +264,7 @@ public function getDBsListsEachDatabaseOnce(): void $info = $db->getDBs('sh'); $this->assertSame('sh', $info['name']); $this->assertNotSame('0 B', $info['size']); - $this->assertCount(2, $db->getDBs(true)); + $this->assertEqualsCanonicalizing(['users', 'sh'], array_column($db->getDBs(true), 'name')); } /** @test O-04 */ @@ -251,7 +279,11 @@ public function withDistanceIsComputedBeforeSortAndKeptBySelect(): void ]); $r = $db->query('wd')->withDistance('loc', 0, 0)->sort('_distance')->limit(2)->select(['n'])->get(); $this->assertSame(['c', 'b'], array_column($r, 'n')); - $this->assertArrayHasKey('_distance', $r[0]); + // select() keeps withDistance()'s _distance (and key): meters, R * longitude in radians on the equator + foreach ($r as $i => $row) { + $this->assertEqualsCanonicalizing(['n', 'key', '_distance'], array_keys($row)); + $this->assertEqualsWithDelta(6371000 * deg2rad([0.1, 0.5][$i]), $row['_distance'], 0.0002); + } $m = $db->query('wd')->where(['n' => 'm'])->withDistance('loc', 2.0005, 2.0005)->first(); $this->assertLessThan(100, $m['_distance'], 'MultiPolygon centroid'); $own = $db->query('wd')->withinDistance('loc', 1, 0, 200000)->withDistance('loc', 0, 0)->sort('_distance')->get(); @@ -333,13 +365,18 @@ public function writesAfterProjectionWorkAndGroupByIsRejected(): void $db = $this->db(); $this->seed($db); $this->assertSame(2, $db->query('q')->where(['city' => 'Istanbul'])->except(['key'])->update(['seen' => 1])['n']); - try { - $db->query('q')->groupBy('city')->delete(); - $this->fail('groupBy + delete must be rejected'); - } catch (\LogicException $e) { - $this->assertStringContainsString('groupBy', $e->getMessage()); + $this->assertSame(['Alice', 'Charlie'], array_column($db->find('q', ['seen' => 1]), 'name')); + foreach (['delete', 'update', 'removeFields'] as $write) { + try { + $q = $db->query('q')->groupBy('city'); + $write === 'delete' ? $q->delete() : ($write === 'update' ? $q->update(['x' => 1]) : $q->removeFields(['name'])); + $this->fail("groupBy + $write must be rejected"); + } catch (\LogicException $e) { + $this->assertSame('noneDB: update(), delete() and removeFields() cannot be combined with groupBy()', $e->getMessage()); + } } - $this->assertCount(5, $db->find('q', 0)); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'abc', 'AĞUSTOS'], array_column($db->find('q', 0), 'name')); + $this->assertSame([], $db->find('q', ['x' => 1])); } /** @test O-08 */ @@ -348,10 +385,11 @@ public function searchIgnoresKeyAndIsMultibyteAware(): void $db = $this->db(); $this->seed($db); $this->assertSame([], $db->query('q')->search('4')->get(), '"4" only appears as a record key'); - $this->assertSame(['AYDOĞDU'], array_column($db->query('q')->search('aydoğdu')->get(), 'name')); - $this->assertCount(2, $db->query('q')->search('izmir')->get(), 'Izmir and İzmir'); - $this->assertCount(2, $db->query('q')->search('İZMİR')->get()); - $this->assertCount(2, $db->query('q')->search('ZMIR')->get()); + $this->assertSame(['AĞUSTOS'], array_column($db->query('q')->search('ağustos')->get(), 'name')); + $this->assertSame(['abc', 'AĞUSTOS'], array_column($db->query('q')->search('izmir')->get(), 'name'), 'Izmir and İzmir'); + $this->assertSame(['abc', 'AĞUSTOS'], array_column($db->query('q')->search('İZMİR')->get(), 'name')); + $this->assertSame(['abc', 'AĞUSTOS'], array_column($db->query('q')->search('ZMIR')->get(), 'name')); + $this->assertSame(['Alice', 'Charlie'], array_column($db->query('q')->search('istanbul', ['city'])->get(), 'name')); } /** @test O-09 */ @@ -361,7 +399,8 @@ public function joinDoesNotCreateDatabases(): void $this->seed($db); $before = glob($this->testDbDir . '*'); $r = $db->query('q')->join('nothere', 'city', 'id')->get(); - $this->assertNull($r[0]['nothere']); + $this->assertSame([null, null, null, null, null], array_column($r, 'nothere')); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'abc', 'AĞUSTOS'], array_column($r, 'name')); $this->assertSame($before, glob($this->testDbDir . '*')); } @@ -377,25 +416,26 @@ public function operatorsAreTypeAware(): void $db->query('q')->where(['age' => ['$gtee' => 1]])->get(); $this->fail('unknown operator'); } catch (\InvalidArgumentException $e) { - $this->assertStringContainsString('$gtee', $e->getMessage()); + $this->assertStringStartsWith("noneDB: unknown operator '\$gtee'", $e->getMessage()); } try { - $db->query('q')->where(['age' => ['$gt' => 1, 'plain' => 2]])->get(); - $this->fail('mixed operator spec'); + $db->query('q')->where(['age' => ['$gt' => 1, 'plain' => 2]]); + $this->fail('mixed operator spec: rejected when where() is called'); } catch (\InvalidArgumentException $e) { - $this->assertStringContainsString('mix', $e->getMessage()); + $this->assertSame('noneDB: a filter value cannot mix operators ($gt, $in, ...) with plain keys', $e->getMessage()); } // Filters are compiled before records are scanned: an invalid query fails the same way // whether or not there are records to look at $db->insert('empty-q', ['a' => 1]); $db->delete('empty-q', []); - foreach ([['$gtee' => 1], ['$in' => 'notarray'], ['$regex' => '(']] as $spec) { - foreach (['q', 'empty-q'] as $name) { + $messages = ["noneDB: unknown operator '\$gtee'", 'noneDB: $in needs an array', "noneDB: invalid \$regex pattern '('"]; + foreach ([['$gtee' => 1], ['$in' => 'notarray'], ['$regex' => '(']] as $i => $spec) { + foreach (['q', 'empty-q', 'no-such-db'] as $name) { try { $db->query($name)->where(['age' => $spec])->get(); $this->fail('invalid spec ' . json_encode($spec) . " on $name"); } catch (\InvalidArgumentException $e) { - $this->assertNotSame('', $e->getMessage()); + $this->assertStringStartsWith($messages[$i], $e->getMessage()); } } } @@ -406,14 +446,16 @@ public function insertAndNameEdgeCases(): void { $db = $this->db(); $r = $db->insert('ie', [0 => ['a' => 1], 'name' => 'x']); - $this->assertSame(0, $r['n']); - $this->assertArrayHasKey('error', $r); - $this->assertSame(0, $db->insert('ie', [])['n']); + $this->assertSame(['n' => 0, 'error' => 'insert data must be either one record or a list of records'], $r); + $this->assertSame(['n' => 0, 'error' => 'insert data must not be empty'], $db->insert('ie', [])); $this->assertSame(['n' => 2], $db->insert('ie', [['a' => 1], ['a' => 2]])); $this->assertSame(['n' => 1], $db->insert('0', ['z' => 1])); $this->assertSame([['z' => 1, 'key' => 0]], $db->find('0', 0)); - $this->assertCount(2, $db->find('ie', null)); + $this->assertSame([['a' => 1, 'key' => 0], ['a' => 2, 'key' => 1]], $db->find('ie', null)); + $this->assertSame($db->find('ie', null), $db->find('ie', 7), 'any integer means every record'); $this->assertFalse($db->find('ie', 'not-a-filter')); + $this->assertFalse($db->find('ie', 1.5)); + $this->assertFalse($db->find('ie', true)); } /** @test D-05 D-06 */ diff --git a/tests/Integration/RTreeTest.php b/tests/Integration/RTreeTest.php new file mode 100644 index 0000000..c18dad4 --- /dev/null +++ b/tests/Integration/RTreeTest.php @@ -0,0 +1,262 @@ +getValue(); + } + + private function randomGeometry(): array + { + $x = mt_rand(-1800, 1790) / 10; + $y = mt_rand(-890, 880) / 10; + switch (mt_rand(0, 3)) { + case 0: + return ['type' => 'Point', 'coordinates' => [$x, $y]]; + case 1: + return ['type' => 'LineString', 'coordinates' => [[$x, $y], [$x + mt_rand(1, 50) / 10, $y + mt_rand(1, 50) / 10]]]; + case 2: + $d = mt_rand(1, 30) / 10; + return ['type' => 'Polygon', 'coordinates' => [[[$x, $y], [$x + $d, $y], [$x + $d, $y + $d], [$x, $y + $d], [$x, $y]]]]; + default: + return ['type' => 'Point', 'coordinates' => [round($x, 1), round($y, 1)]]; // on box edges + } + } + + /** Valid per docs/SPATIAL.md for what randomGeometry() makes: positions in range, no flat coordinates */ + private static function modelValid($g): bool + { + if (!is_array($g) || ($g['type'] !== 'Point' && !is_array($g['coordinates'][0] ?? null))) { + return false; + } + foreach (SpecSpatialTest::positions($g) as [$x, $y]) { + if ($x < -180 || $x > 180 || $y < -90 || $y > 90) { + return false; + } + } + return true; + } + + /** @test */ + public function queriesEqualAScanThroughChanges(): void + { + mt_srand(77); + foreach ([['shardingEnabled' => false], ['shardSize' => 40]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->createSpatialIndex('sp', 'g'); + for ($round = 0; $round < 6; $round++) { + $rows = []; + for ($i = 0; $i < 60; $i++) { + $rows[] = mt_rand(0, 12) === 0 ? ['g' => ['type' => 'LineString', 'coordinates' => [1, 2]]] : ['g' => $this->randomGeometry()]; + } + $db->insert('sp', $rows); + $all = $db->find('sp', 0); + for ($i = 0; $i < 10; $i++) { + $pick = $all[mt_rand(0, count($all) - 1)]['key']; + if (mt_rand(0, 1)) { + $db->delete('sp', ['key' => $pick]); + } else { + $db->update('sp', [['key' => $pick], ['set' => ['g' => $this->randomGeometry()]]]); + } + } + $plain = $this->db($config + ['spatialIndexEnabled' => false]); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + for ($q = 0; $q < 8; $q++) { + $x = mt_rand(-1800, 1700) / 10; + $y = mt_rand(-900, 800) / 10; + $w = mt_rand(1, 400) / 10; + $expected = array_column($plain->withinBBox('sp', 'g', $x, $y, $x + $w, $y + $w), 'key'); + $actual = array_column($db->withinBBox('sp', 'g', $x, $y, $x + $w, $y + $w), 'key'); + sort($expected); + sort($actual); + $this->assertSame($expected, $actual, json_encode([$config, $round, $x, $y, $w])); + // and the scan equals the documented rule: bounding boxes overlap or touch + $model = []; + foreach ($plain->find('sp', 0) as $record) { + if (self::modelValid($record['g']) && SpecSpatialTest::inBBoxQuery($record['g'], SpecSpatialTest::bboxQueryBoxes($x, $y, $x + $w, $y + $w))) { + $model[] = $record['key']; + } + } + sort($model); + $this->assertSame($model, $expected, 'model ' . json_encode([$config, $round, $x, $y, $w])); + $radius = (float)mt_rand(0, 2000000); + $this->assertSame( + array_column($plain->withinDistance('sp', 'g', $x, $y, $radius, ['includeDistance' => true]), '_distance', 'key'), + array_column($db->withinDistance('sp', 'g', $x, $y, $radius, ['includeDistance' => true]), '_distance', 'key'), + 'withinDistance ' . json_encode([$config, $round, $x, $y, $radius])); + $this->assertSame(array_column($plain->nearest('sp', 'g', $x, $y, 3), 'key'), array_column($db->nearest('sp', 'g', $x, $y, 3), 'key')); + } + } + } + $this->assertTrue($db->checkIntegrity('sp')['ok']); + } + } + + /** @test */ + public function treeStaysConsistentThroughInsertsAndRemovals(): void + { + mt_srand(5); + $db = $this->db(['shardingEnabled' => false]); + $tree = (function () { return $this->newRTree('g', 4); })->call(\noneDB\Tests\Support\Testing::service($db, 'newRTree')); + $boxes = []; + $op = function (string $method, ...$args) use ($db, &$tree) { + return (function () use ($method, $args, &$tree) { + return $this->$method($tree, ...$args); + })->call(\noneDB\Tests\Support\Testing::service($db, 'newRTree')); + }; + for ($i = 0; $i < 400; $i++) { + $key = mt_rand(0, 150); + if (mt_rand(0, 2) === 0) { + $op('rtreeRemove', $key); + unset($boxes[$key]); + } else { + $x = mt_rand(-1000, 1000); + $y = mt_rand(-1000, 1000); + $box = [$x, $y, $x + mt_rand(0, 50), $y + mt_rand(0, 50)]; + $op('rtreeInsert', $key, $box); + $boxes[$key] = $box; + } + } + // Every node's box is the union of its items; every key is in exactly one leaf + $seen = []; + foreach ($tree['nodes'] as $id => [$level, $box, $items]) { + $union = null; + foreach ($items as $item) { + $itemBox = $level === 0 ? array_slice($item, 1) : $tree['nodes'][$item][1]; + if ($itemBox === null) { + continue; + } + $union = $union === null ? $itemBox : [min($union[0], $itemBox[0]), min($union[1], $itemBox[1]), max($union[2], $itemBox[2]), max($union[3], $itemBox[3])]; + if ($level === 0) { + $this->assertArrayNotHasKey($item[0], $seen); + $seen[$item[0]] = array_slice($item, 1); + } + } + $this->assertSame($union, $box, "node $id"); + } + ksort($seen); + ksort($boxes); + $this->assertSame($boxes, $seen); + $this->assertSame(count($boxes), $tree['cnt']); + // Bulk load gives the same answers as the tree built one by one + $bulk = (function () { return $this->newRTree('g', 4); })->call(\noneDB\Tests\Support\Testing::service($db, 'newRTree')); + (function () use (&$bulk, $boxes) { + // The entries as columns: keys, then the boxes' minX, minY, maxX, maxY + $columns = [[], [], [], [], []]; + foreach ($boxes as $key => $box) { + $columns[0][] = $key; + for ($c = 0; $c < 4; $c++) { + $columns[$c + 1][] = $box[$c]; + } + } + $this->rtreeBulkLoad($bulk, ...$columns); + })->call(\noneDB\Tests\Support\Testing::service($db, 'newRTree')); + for ($q = 0; $q < 50; $q++) { + $x = mt_rand(-1000, 1000); + $y = mt_rand(-1000, 1000); + $query = [[$x, $y, $x + 200, $y + 200]]; + $a = $op('rtreeSearch', $query); + $b = (function () use ($bulk, $query) { return $this->rtreeSearch($bulk, $query); })->call(\noneDB\Tests\Support\Testing::service($db, 'newRTree')); + sort($a); + sort($b); + $expected = []; + foreach ($boxes as $key => $box) { + if ($box[0] <= $query[0][2] && $query[0][0] <= $box[2] && $box[1] <= $query[0][3] && $query[0][1] <= $box[3]) { + $expected[] = $key; + } + } + $this->assertSame($expected, $a); + $this->assertSame($expected, $b); + } + } + + /** @test */ + public function boxEdgesAndIniSettingsDontLosePoints(): void + { + $before = ini_get('serialize_precision'); + ini_set('serialize_precision', '5'); + try { + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('qe', 'g'); + $db->insert('qe', [ + ['g' => ['type' => 'Point', 'coordinates' => [28.980345, 41.0086]]], + ['g' => ['type' => 'Point', 'coordinates' => [29.0, 41.0]]], + ['g' => ['type' => 'Point', 'coordinates' => [29.00000001, 41.0]]], + ['g' => ['type' => 'Point', 'coordinates' => [-180, -90]]], + ['g' => ['type' => 'Point', 'coordinates' => [180, 90]]], + ]); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0], array_column($db->withinBBox('qe', 'g', 28.980345, 41.0086, 28.980345, 41.0086), 'key')); + $this->assertSame([1], array_column($db->withinBBox('qe', 'g', 28.99, 40.99, 29.0, 41.0), 'key'), 'the box edge belongs to the box'); + $this->assertSame([2], array_column($db->withinBBox('qe', 'g', 29.000000005, 40.99, 29.1, 41.01), 'key'), 'closer than the 1e-7 grid'); + $this->assertSame([0, 1, 2, 3, 4], array_column($db->withinBBox('qe', 'g', -INF, -INF, INF, INF), 'key'), 'the corners of the globe too'); + } finally { + ini_set('serialize_precision', $before); + } + } + + /** @test */ + public function indexedCandidatesAreNotValidatedAgain(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('nv', 'g'); + $ring = []; + for ($i = 0; $i < 200; $i++) { + $ring[] = [cos($i / 100 * M_PI), sin($i / 100 * M_PI)]; + } + $ring[] = $ring[0]; + $db->insert('nv', array_fill(0, 20, ['g' => ['type' => 'Polygon', 'coordinates' => [$ring]]])); + $db->withinBBox('nv', 'g', -2, -2, 2, 2); // loads the index + $before = $this->stat('geometryValidations'); + $this->assertCount(20, $db->withinBBox('nv', 'g', -2, -2, 2, 2)); + $this->assertCount(20, $db->withinDistance('nv', 'g', 0, 0, 500000)); + $this->assertSame(0, $this->stat('geometryValidations') - $before, 'R-tree entries were validated when indexed'); + $plain = $this->db(['shardingEnabled' => false, 'spatialIndexEnabled' => false]); + $before = $this->stat('geometryValidations'); + $this->assertCount(20, $plain->withinBBox('nv', 'g', -2, -2, 2, 2)); + $this->assertSame(20, $this->stat('geometryValidations') - $before, 'a scan validates each'); + } + + /** @test */ + public function writesDontRewriteTheTree(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('wt', 'g'); + $db->insert('wt', array_map(function ($n) { return ['g' => ['type' => 'Point', 'coordinates' => [$n / 10, 1]]]; }, range(0, 99))); + $file = $this->dbPath('wt', 'rx', 'g'); + $bytes = file_get_contents($file); + for ($i = 0; $i < 10; $i++) { + $db->insert('wt', ['g' => ['type' => 'Point', 'coordinates' => [50 + $i, 1]]]); + $db->update('wt', [['key' => $i], ['set' => ['g' => ['type' => 'Point', 'coordinates' => [-50, -$i]]]]]); + } + clearstatcache(); + $this->assertSame($bytes, file_get_contents($file)); + // the in-memory tree answers the new state before any checkpoint + $this->assertCount(10, $db->withinBBox('wt', 'g', 49, 0, 60, 2)); + $this->assertSame($bytes, file_get_contents($file), 'reads below the checkpoint threshold write nothing either'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertCount(10, $db->withinBBox('wt', 'g', 49, 0, 60, 2)); + $this->assertCount(10, $db->withinBBox('wt', 'g', -51, -10, -49, 0)); + $this->assertCount(0, $db->withinBBox('wt', 'g', 0, 0.5, 0.95, 1.5), 'keys 0-9 moved'); + $header = json_decode(strtok(file_get_contents($file), "\n"), true); + $this->assertSame(['rx', 1], [$header['kind'], $header['v']]); // STORAGE_FORMAT.md §6.5 + $this->assertSame(10000000, $header['q']); + } +} diff --git a/tests/Integration/RemoveFieldsTest.php b/tests/Integration/RemoveFieldsTest.php new file mode 100644 index 0000000..5ff66b1 --- /dev/null +++ b/tests/Integration/RemoveFieldsTest.php @@ -0,0 +1,752 @@ + 12, 'fields_removed' => ['old_password']]` for removeFields(['old_password', 'tmp']): + * "n" counts the records that changed, "fields_removed" lists the requested fields that were really + * removed somewhere, in the order given; sort/offset/limit choose the records; groupBy() throws + * LogicException) and from an independent model: the setUp() records with the fields unset by hand. + * The stored records are always re-read from disk by a fresh instance. + */ +class RemoveFieldsTest extends noneDBTestCase +{ + /** The records of setUp(), keys 0-4 */ + private const DATA = [ + ['name' => 'Alice', 'age' => 25, 'city' => 'Istanbul', 'email' => 'alice@gmail.com', 'score' => 85, 'department' => 'IT', 'active' => true, 'notes' => 'Some notes'], + ['name' => 'Bob', 'age' => 30, 'city' => 'Ankara', 'email' => 'bob@yahoo.com', 'score' => 90, 'department' => 'HR', 'active' => true, 'notes' => 'Other notes'], + ['name' => 'Charlie', 'age' => 35, 'city' => 'Istanbul', 'email' => 'charlie@gmail.com', 'score' => 75, 'department' => 'IT', 'active' => false, 'notes' => null], + ['name' => 'David', 'age' => 28, 'city' => 'Izmir', 'email' => 'david@hotmail.com', 'score' => 95, 'department' => 'Sales', 'active' => true], + ['name' => 'Eve', 'age' => 22, 'city' => 'Istanbul', 'email' => 'eve@gmail.com', 'score' => 80, 'department' => 'IT', 'active' => true, 'notes' => 'Eve notes'], + ]; + + protected function setUp(): void + { + parent::setUp(); + + $this->noneDB->insert($this->testDbName, self::DATA); + } + + /** + * The stored database as the model expects it: setUp() data with $fields removed from the records + * whose key is in $keys (all keys when null); [key => record with "key"], fields sorted + */ + private function model(array $fields, ?array $keys = null, array $data = self::DATA): array + { + $out = []; + foreach ($data as $key => $record) { + if ($keys === null || in_array($key, $keys, true)) { + foreach ($fields as $field) { + unset($record[$field]); + } + } + $record['key'] = $key; + ksort($record); + $out[$key] = $record; + } + return $out; + } + + /** + * The stored records as a fresh instance reads them from disk: [key => record] (fields sorted) + */ + private function stored(?string $dbName = null, ?\noneDB $db = null): array + { + \noneDB::clearStaticCache(); + $db = $db ?? $this->createTestInstance(); + $out = []; + foreach ($db->find($dbName ?? $this->testDbName, 0) as $record) { + ksort($record); + $out[$record['key']] = $record; + } + return $out; + } + + // ========================================== + // BASIC FUNCTIONALITY + // ========================================== + + /** + * @test + */ + public function removeFieldsBasicUsage(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + // 4 records have a 'notes' field (Charlie's is null but present); David has none + $this->assertSame(['n' => 4, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsMultipleFields(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes', 'score']); + + // Every record has 'score', so all 5 change + $this->assertSame(['n' => 5, 'fields_removed' => ['notes', 'score']], $result); + $this->assertSame($this->model(['notes', 'score']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsListsRemovedFieldsInTheOrderGiven(): void + { + $result = $this->noneDB->query($this->testDbName)->removeFields(['score', 'missing', 'notes']); + + $this->assertSame(['n' => 5, 'fields_removed' => ['score', 'notes']], $result); + $this->assertSame($this->model(['score', 'notes']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsPreservesOtherFields(): void + { + $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + // Every other field keeps its value and type + $this->assertSame($this->model(['notes']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsReturnsCorrectCount(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Istanbul']) + ->removeFields(['notes']); + + // Istanbul has 3 records: Alice, Charlie, Eve. All three have the field (Charlie's is null) + $this->assertSame(['n' => 3, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes'], [0, 2, 4]), $this->stored()); + } + + // ========================================== + // EDGE CASES - EMPTY AND INVALID INPUT + // ========================================== + + /** + * @test + */ + public function removeFieldsWithEmptyArray(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields([]); + + $this->assertSame(0, $result['n']); + $this->assertSame([], $result['fields_removed']); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('No fields specified', $result['error']); + $this->assertSame($this->model([]), $this->stored()); // nothing changed + } + + /** + * @test + */ + public function removeFieldsWithNonExistentField(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['nonexistent_field']); + + // No records have this field, so no updates and nothing removed + $this->assertSame(['n' => 0, 'fields_removed' => []], $result); + $this->assertSame($this->model([]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithMixedExistentNonExistent(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes', 'nonexistent']); + + // Only 'notes' should be in fields_removed + $this->assertSame(['n' => 4, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsCannotRemoveKeyField(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['key']); + + $this->assertSame(0, $result['n']); + $this->assertSame([], $result['fields_removed']); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('key', $result['error']); + + // Every record, with its key, is unchanged + $this->assertSame($this->model([]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsKeyWithOtherFields(): void + { + // Try to remove key along with valid fields + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['key', 'notes']); + + // key is ignored, notes is removed + $this->assertSame(['n' => 4, 'fields_removed' => ['notes']], $result); + + // Keys unchanged, notes removed + $this->assertSame($this->model(['notes']), $this->stored()); + } + + // ========================================== + // WITH FILTERS + // ========================================== + + /** + * @test + */ + public function removeFieldsWithWhereFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->removeFields(['notes']); + + // IT: Alice, Charlie, Eve; Bob (HR) keeps his notes + $this->assertSame(['n' => 3, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes'], [0, 2, 4]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithWhereInFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->removeFields(['score']); + + // Istanbul (3) + Ankara (1) = 4 records affected; Izmir keeps its score + $this->assertSame(['n' => 4, 'fields_removed' => ['score']], $result); + $this->assertSame($this->model(['score'], [0, 1, 2, 4]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithBetweenFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->between('age', 25, 30) + ->removeFields(['email']); + + // Alice(25), Bob(30), David(28) = 3 records (inclusive); Charlie(35) and Eve(22) keep email + $this->assertSame(['n' => 3, 'fields_removed' => ['email']], $result); + $this->assertSame($this->model(['email'], [0, 1, 3]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithOrWhereFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['city' => 'Izmir']) + ->orWhere(['city' => 'Ankara']) + ->removeFields(['department']); + + // Izmir (1) + Ankara (1) = 2 records; Istanbul records keep their department + $this->assertSame(['n' => 2, 'fields_removed' => ['department']], $result); + $this->assertSame($this->model(['department'], [1, 3]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithSearchFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->search('gmail') + ->removeFields(['active']); + + // Alice, Charlie, Eve have gmail = 3 records + $this->assertSame(['n' => 3, 'fields_removed' => ['active']], $result); + $this->assertSame($this->model(['active'], [0, 2, 4]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithNoMatchingRecords(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['city' => 'NonExistent']) + ->removeFields(['notes']); + + // No record matched: nothing changed and nothing was removed ("fields_removed" lists removed + // fields, QUERY.md example), the same answer as when matching records lack the field + $this->assertSame(['n' => 0, 'fields_removed' => []], $result); + $this->assertSame($this->model([]), $this->stored()); + } + + // ========================================== + // SORT / OFFSET / LIMIT CHOOSE THE RECORDS + // ========================================== + + /** + * @test + */ + public function removeFieldsWithSortAndLimitChoosesTheRecords(): void + { + // QUERY.md: sort(), offset() and limit() choose which records are written. Two oldest: Charlie, Bob + $result = $this->noneDB->query($this->testDbName)->sort('age', 'desc')->limit(2)->removeFields(['notes']); + + $this->assertSame(['n' => 2, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes'], [1, 2]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithSortOffsetAndLimitChoosesTheRecords(): void + { + // Ages ascending: Eve 22, Alice 25, David 28, Bob 30, Charlie 35 -> offset 1, limit 2: Alice, David. + // David has no notes: one record changes + $result = $this->noneDB->query($this->testDbName)->sort('age')->offset(1)->limit(2)->removeFields(['notes']); + + $this->assertSame(['n' => 1, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes'], [0, 3]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithLimitWithoutSortUsesKeyOrder(): void + { + $result = $this->noneDB->query($this->testDbName)->limit(2)->removeFields(['email']); + + $this->assertSame(['n' => 2, 'fields_removed' => ['email']], $result); + $this->assertSame($this->model(['email'], [0, 1]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithOffsetPastTheEndRemovesNothing(): void + { + $result = $this->noneDB->query($this->testDbName)->offset(5)->removeFields(['notes']); + + $this->assertSame(0, $result['n']); + $this->assertSame($this->model([]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithGroupByThrowsLogicException(): void + { + // API.md: update(), delete() and removeFields() throw LogicException together with groupBy() + try { + $this->noneDB->query($this->testDbName)->groupBy('city')->removeFields(['notes']); + $this->fail('removeFields() with groupBy() must throw'); + } catch (\LogicException $e) { + $this->assertSame(\LogicException::class, get_class($e)); + $this->assertStringContainsString('cannot be combined with groupBy()', $e->getMessage()); + } + $this->assertSame($this->model([]), $this->stored()); // nothing written + } + + // ========================================== + // PROJECTION, JOINS AND DISTANCES NEVER REACH STORAGE + // ========================================== + + /** + * @test + */ + public function selectAndExceptDoNotChangeWhatIsStored(): void + { + // select() keeps only "name" in the result: the other fields must not be dropped from storage + $result = $this->noneDB->query($this->testDbName)->select(['name'])->where(['name' => 'Alice'])->removeFields(['notes']); + $this->assertSame(['n' => 1, 'fields_removed' => ['notes']], $result); + + // except() drops "email" from the result only + $result = $this->noneDB->query($this->testDbName)->except(['email'])->where(['name' => 'Bob'])->removeFields(['notes']); + $this->assertSame(['n' => 1, 'fields_removed' => ['notes']], $result); + + $this->assertSame($this->model(['notes'], [0, 1]), $this->stored()); + } + + /** + * @test + */ + public function joinedRecordsAreNeverStored(): void + { + $this->noneDB->insert('departments', [ + ['department' => 'IT', 'floor' => 3], + ['department' => 'HR', 'floor' => 1], + ]); + + $result = $this->noneDB->query($this->testDbName) + ->join('departments', 'department', 'department') + ->join('departments', 'department', 'department', 'dept') + ->where(['department' => 'IT']) + ->removeFields(['notes']); + + $this->assertSame(['n' => 3, 'fields_removed' => ['notes']], $result); + // Neither "departments" nor "dept" reached the stored records + $this->assertSame($this->model(['notes'], [0, 2, 4]), $this->stored()); + // The foreign database is unchanged too + $this->assertSame([ + 0 => ['department' => 'IT', 'floor' => 3, 'key' => 0], + 1 => ['department' => 'HR', 'floor' => 1, 'key' => 1], + ], $this->stored('departments')); + } + + /** + * @test + */ + public function distancesAreNeverStored(): void + { + $places = [ + ['name' => 'A', 'tag' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [28.98, 41.01]]], + ['name' => 'B', 'tag' => 'y', 'loc' => ['type' => 'Point', 'coordinates' => [29.50, 41.50]]], + ]; + $this->noneDB->insert('places', $places); + + // withDistance() adds _distance to the result; withinDistance() adds one to its candidates + $result = $this->noneDB->query('places')->withDistance('loc', 28.97, 41.0)->removeFields(['tag']); + $this->assertSame(['n' => 2, 'fields_removed' => ['tag']], $result); + + $this->noneDB->update('places', [[], ['set' => ['tag' => 'z']]]); + $result = $this->noneDB->query('places')->withinDistance('loc', 28.97, 41.0, 5000)->removeFields(['tag']); + $this->assertSame(['n' => 1, 'fields_removed' => ['tag']], $result); // only A is within 5 km + + $stored = $this->stored('places'); + $this->assertSame(['key' => 0, 'loc' => $places[0]['loc'], 'name' => 'A'], $stored[0]); + $this->assertSame(['key' => 1, 'loc' => $places[1]['loc'], 'name' => 'B', 'tag' => 'z'], $stored[1]); + } + + // ========================================== + // SPECIAL FIELD VALUES + // ========================================== + + /** + * @test + */ + public function removeFieldsWithNullValue(): void + { + // Charlie has notes = null + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'Charlie']) + ->removeFields(['notes']); + + // The field is removed even though its value is null + $this->assertSame(['n' => 1, 'fields_removed' => ['notes']], $result); + $this->assertSame($this->model(['notes'], [2]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithBooleanValue(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['active']); + + $this->assertSame(['n' => 5, 'fields_removed' => ['active']], $result); + $this->assertSame($this->model(['active']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithNumericValue(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['age']); + + $this->assertSame(['n' => 5, 'fields_removed' => ['age']], $result); + $this->assertSame($this->model(['age']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsWithNestedArrayValue(): void + { + $this->noneDB->update($this->testDbName, [['name' => 'Eve'], ['set' => ['address' => ['city' => 'Istanbul', 'zip' => '34000']]]]); + + $result = $this->noneDB->query($this->testDbName)->removeFields(['address']); + + $this->assertSame(['n' => 1, 'fields_removed' => ['address']], $result); + $this->assertSame($this->model([]), $this->stored()); + } + + // ========================================== + // PARTIAL FIELD EXISTENCE + // ========================================== + + /** + * @test + */ + public function removeFieldsWhenOnlySomeRecordsHaveField(): void + { + // David doesn't have 'notes' field + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + // Only 4 records have notes (Alice, Bob, Charlie, Eve) + $this->assertSame(4, $result['n']); + + // David is unchanged, field by field + $this->assertSame($this->model([])[3], $this->stored()[3]); + } + + // ========================================== + // MULTIPLE OPERATIONS + // ========================================== + + /** + * @test + */ + public function removeFieldsMultipleTimes(): void + { + $result1 = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + $result2 = $this->noneDB->query($this->testDbName) + ->removeFields(['score']); + // Third removal: the records match, but none has notes any more + $result3 = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + $this->assertSame(['n' => 4, 'fields_removed' => ['notes']], $result1); + $this->assertSame(['n' => 5, 'fields_removed' => ['score']], $result2); + $this->assertSame(['n' => 0, 'fields_removed' => []], $result3); + + $this->assertSame($this->model(['notes', 'score']), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsThenUpdate(): void + { + $this->noneDB->query($this->testDbName) + ->where(['name' => 'Alice']) + ->removeFields(['notes']); + + $this->assertSame(['n' => 1], $this->noneDB->update($this->testDbName, [ + ['name' => 'Alice'], + ['set' => ['status' => 'updated']] + ])); + + $expected = $this->model(['notes'], [0]); + $expected[0]['status'] = 'updated'; + ksort($expected[0]); + $this->assertSame($expected, $this->stored()); + } + + /** + * @test + */ + public function removeFieldsThenDelete(): void + { + $this->assertSame(5, $this->noneDB->count($this->testDbName)); + + $this->noneDB->query($this->testDbName) + ->where(['department' => 'IT']) + ->removeFields(['notes']); + + $this->assertSame(['n' => 1], $this->noneDB->query($this->testDbName) + ->where(['department' => 'HR']) + ->delete()); + + $this->assertSame(4, $this->noneDB->count($this->testDbName)); + $expected = $this->model(['notes'], [0, 2, 4]); + unset($expected[1]); // Bob (HR) deleted + $this->assertSame($expected, $this->stored()); + } + + // ========================================== + // DATA INTEGRITY + // ========================================== + + /** + * @test + */ + public function removeFieldsPreservesRecordCount(): void + { + $this->noneDB->query($this->testDbName) + ->removeFields(['notes', 'score']); + + $this->assertSame(5, $this->noneDB->count($this->testDbName)); + $this->assertSame(5, $this->createTestInstance()->count($this->testDbName)); + } + + /** + * @test + */ + public function removeFieldsPreservesKeyValues(): void + { + $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + $this->assertSame([0, 1, 2, 3, 4], array_column($this->noneDB->find($this->testDbName, 0), 'key')); + $this->assertSame([0, 1, 2, 3, 4], array_keys($this->stored())); + // A new record still gets the next key (keys are never reused or changed, API.md) + $this->noneDB->insert($this->testDbName, ['name' => 'Frank']); + $this->assertSame(5, $this->noneDB->first($this->testDbName, ['name' => 'Frank'])['key']); + } + + /** + * @test + */ + public function removeFieldsPreservesFieldValues(): void + { + $this->noneDB->query($this->testDbName) + ->where(['name' => 'Alice']) + ->removeFields(['notes']); + + $aliceAfter = $this->noneDB->first($this->testDbName, ['name' => 'Alice']); + ksort($aliceAfter); + + $this->assertSame($this->model(['notes'], [0])[0], $aliceAfter); + } + + /** + * @test + */ + public function removeFieldsKeepsFieldIndexConsistent(): void + { + $this->noneDB->createFieldIndex($this->testDbName, 'department'); + + $result = $this->noneDB->query($this->testDbName)->where(['department' => 'IT'])->limit(2)->removeFields(['department']); + $this->assertSame(['n' => 2, 'fields_removed' => ['department']], $result); // Alice, Charlie + + // API.md: indexes always give exactly the results of a full scan + $this->assertSame([4], array_column($this->noneDB->find($this->testDbName, ['department' => 'IT']), 'key')); + $this->assertSame([4], array_column($this->noneDB->query($this->testDbName)->where(['department' => 'IT'])->get(), 'key')); + $this->assertSame([0, 2], array_column($this->noneDB->query($this->testDbName)->where(['department' => ['$exists' => false]])->get(), 'key')); + $this->assertSame($this->model(['department'], [0, 2]), $this->stored()); + } + + // ========================================== + // SHARDED DATABASE + // ========================================== + + /** + * @test + */ + public function removeFieldsOnShardedDatabase(): void + { + $config = ['secretKey' => 'test_secret_key_for_unit_tests', 'dbDir' => $this->testDbDir, 'autoCreateDB' => true, 'shardSize' => 2]; + $db = new \noneDB($config); + $data = []; + for ($i = 0; $i < 9; $i++) { + $data[] = ['n' => $i, 'even' => $i % 2 === 0, 'tmp' => 't' . $i]; + } + $db->insert('shardedrf', $data); + $this->assertTrue($db->getShardInfo('shardedrf')['sharded']); + $this->assertSame(5, $db->getShardInfo('shardedrf')['shards']); + + // Even records across every shard; sort desc + limit 3 picks n = 8, 6, 4 + $result = $db->query('shardedrf')->where(['even' => true])->sort('n', 'desc')->limit(3)->removeFields(['tmp']); + $this->assertSame(['n' => 3, 'fields_removed' => ['tmp']], $result); + + $expected = []; + foreach ($data as $key => $record) { + if (in_array($key, [4, 6, 8], true)) { + unset($record['tmp']); + } + $record['key'] = $key; + ksort($record); + $expected[$key] = $record; + } + $this->assertSame($expected, $this->stored('shardedrf', new \noneDB($config))); + } + + // ========================================== + // COMPLEX SCENARIOS + // ========================================== + + /** + * @test + */ + public function removeFieldsComplexFilter(): void + { + $result = $this->noneDB->query($this->testDbName) + ->whereIn('city', ['Istanbul', 'Ankara']) + ->between('age', 20, 30) + ->where(['active' => true]) + ->removeFields(['notes', 'score']); + + // Istanbul: Alice(25,active), Eve(22,active); Ankara: Bob(30,active) = 3 records + $this->assertSame(['n' => 3, 'fields_removed' => ['notes', 'score']], $result); + $this->assertSame($this->model(['notes', 'score'], [0, 1, 4]), $this->stored()); + } + + /** + * @test + */ + public function removeFieldsAllFieldsExceptRequired(): void + { + $fields = ['age', 'city', 'email', 'score', 'department', 'active', 'notes']; + $result = $this->noneDB->query($this->testDbName) + ->removeFields($fields); + + $this->assertSame(['n' => 5, 'fields_removed' => $fields], $result); + + // All records now only have 'name' and 'key' + $expected = []; + foreach (self::DATA as $key => $record) { + $expected[$key] = ['key' => $key, 'name' => $record['name']]; + } + $this->assertSame($expected, $this->stored()); + } + + /** + * @test + */ + public function removeFieldsReturnValueStructure(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + // Exactly "n" (int) and "fields_removed" (list); no "error" on success + $this->assertSame(['n', 'fields_removed'], array_keys($result)); + $this->assertSame(4, $result['n']); + $this->assertSame(['notes'], $result['fields_removed']); + } + + // ========================================== + // CHAINABLE METHOD CHECK + // ========================================== + + /** + * @test + */ + public function removeFieldsIsTerminalMethod(): void + { + // removeFields returns the result array, not the builder + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['notes']); + + $this->assertIsArray($result); + $this->assertSame(4, $result['n']); + } +} diff --git a/tests/Regression/SecurityConfigRegressionTest.php b/tests/Integration/SecurityConfigRegressionTest.php similarity index 55% rename from tests/Regression/SecurityConfigRegressionTest.php rename to tests/Integration/SecurityConfigRegressionTest.php index dab8d5e..8f901a4 100644 --- a/tests/Regression/SecurityConfigRegressionTest.php +++ b/tests/Integration/SecurityConfigRegressionTest.php @@ -1,6 +1,8 @@ assertSame($raw, $this->sealRecord(json_decode($raw)), $copy . ': encoding and checksum'); + return json_decode($raw, true); + } + + /** Entries of a directory (without . and ..), sorted */ + private function entries(string $dir): array + { + return $this->sorted(array_diff(scandir($dir) ?: [], ['.', '..'])); + } + + private function sorted(array $names): array + { + $names = array_values($names); + sort($names, SORT_STRING); + return $names; + } + protected function tearDown(): void { foreach (glob(sys_get_temp_dir() . '/nonedb_sec_*') ?: [] as $dir) { @@ -27,8 +50,12 @@ public function wrongKeyCannotReadData(): void { $dir = $this->scratch('s01') . '/'; $this->runChild('(new \noneDB(["secretKey" => "alpha-key", "dbDir" => ' . var_export($dir, true) . ']))->insert("users", ["pin" => "1234"]);'); - $wrong = $this->runChild('echo json_encode((new \noneDB(["secretKey" => "WRONG", "dbDir" => ' . var_export($dir, true) . ', "autoCreateDB" => false]))->find("users", 0));'); - $this->assertSame('false', $wrong); + // The store does not list the wrong key: KEY_NOT_REGISTERED (STORAGE_FORMAT.md §3.2), no data, no file + $before = $this->entries($dir); + $wrong = $this->runChild('try { echo json_encode((new \noneDB(["secretKey" => "WRONG", "dbDir" => ' . var_export($dir, true) . ', "autoCreateDB" => false]))->find("users", 0)); } + catch (\RuntimeException $e) { echo $e->getCode() === \noneDB::KEY_NOT_REGISTERED ? "KEY_NOT_REGISTERED" : get_class($e) . $e->getCode(); }'); + $this->assertSame('KEY_NOT_REGISTERED', $wrong); + $this->assertSame($before, $this->entries($dir), 'nothing created'); foreach (['null', "''", 'false'] as $bad) { $out = $this->runChild('try { new \noneDB(["secretKey" => ' . $bad . ', "dbDir" => ' . var_export($dir, true) . ']); echo "ACCEPTED"; } catch (\InvalidArgumentException $e) { echo "REJECTED"; }'); $this->assertSame('REJECTED', $out, "key $bad"); @@ -41,16 +68,10 @@ public function wrongKeyRunDoesNotPoisonTheCache(): void $dir = $this->scratch('s02') . '/'; $mk = function ($key) use ($dir) { return '(new \noneDB(["secretKey" => "' . $key . '", "dbDir" => ' . var_export($dir, true) . ']))'; }; $this->runChild($mk('alpha-key') . '->insert("users", ["pin" => "1234"]);'); - foreach (glob($dir . '.nonedb_hash_cache*') as $f) { - unlink($f); - } $this->runChild($mk('WRONG') . '->find("users", 0);'); $right = $this->runChild('echo json_encode(' . $mk('alpha-key') . '->find("users", 0));'); $this->assertSame('[{"pin":"1234","key":0}]', $right); $this->runChild($mk('alpha-key') . '->insert("users", ["pin" => "5678"]);'); - foreach (glob($dir . '.nonedb_hash_cache*') as $f) { - unlink($f); - } $this->assertSame('[{"pin":"1234","key":0},{"pin":"5678","key":1}]', $this->runChild('echo json_encode(' . $mk('alpha-key') . '->find("users", 0));')); } @@ -67,15 +88,18 @@ public function tenantsInOneProcessStayIsolated(): void $B->insert("users", ["u" => "b"]); echo json_encode($B->find("users", 0)), json_encode($A->find("users", 0));'); $this->assertSame('[{"u":"b","key":0}][{"u":"a","key":0}]', $out); - $expectedB = hash_pbkdf2('sha256', 'users', 'tenantB-key', 1000, 20); - $this->assertFileExists("$base/B/$expectedB-users.nonedb"); - $cache = ''; - foreach (glob("$base/B/.nonedb_hash_cache_*") as $f) { - $cache .= file_get_contents($f); - $this->assertSame('0600', substr(sprintf('%o', fileperms($f)), -4)); + // Each store holds its own databases, under its key's locators (STORAGE_FORMAT.md §1, §3) + $this->assertFileExists("$base/B/" . $this->locatorOf('users', 'tenantB-key') . '.ndb4/FORMAT'); + $this->assertSame($this->sorted(['.htaccess', $this->locatorOf('orders', 'tenantB-key') . '.ndb4', $this->locatorOf('users', 'tenantB-key') . '.ndb4', + 'STORE', 'STORE.2', 'STORE.LOCK', 'index.html']), $this->entries("$base/B"), 'nothing of tenant A in B\'s store'); + $this->assertSame([$this->keyCheckOf('tenantB-key')], $this->storeRecord("$base/B/")['keys']); + // No name of A's databases anywhere in B's store (file names are locators, not names) + $files = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator("$base/B", \FilesystemIterator::SKIP_DOTS)); + foreach ($files as $file) { + $this->assertStringNotContainsString('payroll', $file->getPathname()); + $this->assertStringNotContainsString('payroll', (string)file_get_contents($file->getPathname()), $file->getPathname()); } - $this->assertStringNotContainsString('payroll', $cache); - $this->assertFileDoesNotExist("$base/B/.nonedb_hash_cache"); + $this->assertDirectoryExists("$base/A/" . $this->locatorOf('payroll-2026-ceo-bonus', 'tenantA-key') . '.ndb4'); } /** @test S-04 */ @@ -112,7 +136,7 @@ public function configFileResolution(): void $this->assertSame($web, $cron, 'web and cron share one database'); $this->assertSame($web, $fromParent, 'relative script path, other cwd'); $this->assertSame($web, $mixed, 'array config merges over the file (secretKey from file)'); - $this->assertCount(1, glob("$p/db/*-orders.nonedb")); + $this->assertSame([$p . '/db/' . $this->locatorOf('orders', 's3cr3t-long-random') . '.ndb4'], glob("$p/db/*.ndb4")); $this->assertFileExists("$p/db/.htaccess"); $this->assertFileExists("$p/db/index.html"); $this->assertSame([], glob("$p/public/db") ?: []); @@ -135,37 +159,13 @@ public function plainRelativeDbDirFollowsTheConfigFile(): void $cron = trim(shell_exec('cd ' . escapeshellarg(sys_get_temp_dir()) . " && $php " . escapeshellarg("$p/bin/cron.php") . ' 2>&1')); $this->assertSame('[{"from":"web","key":0}]', $web); $this->assertSame($web, $cron, 'a cwd elsewhere does not matter'); - $this->assertCount(1, glob("$p/data/*-orders.nonedb")); + $this->assertSame([$p . '/data/' . $this->locatorOf('orders', 's3cr3t-long-random') . '.ndb4'], glob("$p/data/*.ndb4")); $this->assertSame([], glob("$p/public/data") ?: []); $this->assertSame('ok', trim(shell_exec('cd ' . escapeshellarg($p) . " && $php bin/array.php 2>&1"))); - $this->assertCount(1, glob("$p/bin/store/*-a.nonedb")); + $this->assertSame([$p . '/bin/store/' . $this->locatorOf('a', 'other-key-123') . '.ndb4'], glob("$p/bin/store/*.ndb4")); $this->assertSame([], glob("$p/store") ?: []); } - /** @test S-05: data in a pre-3.2 location keeps being used (with a deprecation notice) */ - public function oldDbDirLocationsKeepWorking(): void - { - foreach (['./old/' => 'script', 'old/' => 'cwd'] as $dbDir => $oldBase) { - $p = realpath($this->scratch('legacy')); - mkdir("$p/app/web", 0777, true); - mkdir("$p/cwd"); - copy(__DIR__ . '/../../noneDB.php', "$p/app/noneDB.php"); - file_put_contents("$p/app/.nonedb", json_encode(['secretKey' => 's3cr3t-long-random', 'dbDir' => $dbDir])); - $script = 'insert("t", ["v" => 1]); echo json_encode(array_column($db->find("t", 0), "v"));'; - file_put_contents("$p/app/web/run.php", $script); - // Where v3.1 put the data: next to the entry script ("./old/") or in the working directory ("old/") - $old = $oldBase === 'script' ? "$p/app/web/old" : "$p/cwd/old"; - mkdir($old); - $out = trim(shell_exec('cd ' . escapeshellarg("$p/cwd") . ' && ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg("$p/app/web/run.php") . ' 2>&1')); - $this->assertStringContainsString('NOTICE:noneDB: dbDir', $out, $dbDir); - $this->assertStringContainsString("using the old location $old", $out, $dbDir); - $this->assertStringEndsWith('[1]', $out, $dbDir); - $this->assertCount(1, glob("$old/*-t.nonedb"), $dbDir); - $this->assertDirectoryDoesNotExist("$p/app/old", $dbDir); - } - } - /** @test S-04 S-11: an explicitly given invalid secretKey is rejected whatever the environment */ public function givenInvalidKeyIsRejectedWithoutConfigFileAndInDevMode(): void { @@ -229,6 +229,51 @@ public function clonesKeepTheirKey(): void $this->assertCount(1, $copy->find('cl', 0)); } + /** + * @test an instance is freed when its last reference goes: its engine (kernel and services) holds no cycle + * the cycle collector would have to find, and a new instance builds only the services it uses + */ + public function instancesAreFreedAtOnce(): void + { + $config = ['secretKey' => 'free-key', 'dbDir' => $this->testDbDir]; + (new \noneDB($config))->insert('fr', ['a' => 1]); + gc_collect_cycles(); + $live = \noneDB\Tests\Support\Testing::liveInstances(); + $runs = gc_status()['runs']; + $memory = memory_get_usage(); + for ($i = 0; $i < 300; $i++) { + $db = new \noneDB($config); + $db->find('fr', 0); + $copy = clone $db; + $copy->insert('fr', ['a' => 2]); + unset($db, $copy); + $this->assertSame($live, \noneDB\Tests\Support\Testing::liveInstances(), "round $i"); + } + $this->assertSame($runs, gc_status()['runs'], 'nothing left for the cycle collector'); + $this->assertLessThan(256 * 1024, memory_get_usage() - $memory, 'no memory kept per instance'); + $this->assertCount(301, (new \noneDB($config))->find('fr', 0)); + } + + /** @test a clone is an independent instance: its own settings, the same data and shared caches */ + public function clonesHaveTheirOwnSettings(): void + { + $db = new \noneDB(['secretKey' => 'clone-key', 'dbDir' => $this->testDbDir, 'shardingEnabled' => false]); + $db->insert('cs', ['a' => 1]); + $copy = clone $db; + $copy->enableBuffering(true); + $copy->setBufferCountLimit(50); + $this->assertFalse($db->isBufferingEnabled(), 'the original keeps its settings'); + $this->assertTrue($copy->isBufferingEnabled()); + $this->assertSame(10000, $db->getBufferInfo('cs')['countLimit']); + $this->assertSame(50, $copy->getBufferInfo('cs')['countLimit']); + $copy->insert('cs', ['a' => 2]); + $copy->flush('cs'); + $db->insert('cs', ['a' => 3]); + $this->assertSame([1, 2, 3], array_column($db->find('cs', 0), 'a'), 'one database'); + $this->assertSame([1, 2, 3], array_column($copy->find('cs', 0), 'a')); + $this->assertSame(\noneDB\Tests\Support\Testing::getOf($db, 'indexCache'), \noneDB\Tests\Support\Testing::getOf($copy, 'indexCache'), 'caches stay shared'); + } + /** @test S-11 Y-25 */ public function devModeInstancesAreConfigured(): void { @@ -238,7 +283,9 @@ public function devModeInstancesAreConfigured(): void $a = new noneDB(); $b = new noneDB(); $b->insert("t", ["a" => 1]); echo count($a->find("t", 0));'); $out = trim(shell_exec('cd ' . escapeshellarg(sys_get_temp_dir()) . ' && env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg("$dir/run.php") . ' 2>&1')); $this->assertSame('1', $out); - $this->assertCount(1, glob("$dir/db/*-t.nonedb"), 'data next to noneDB.php, not in the working directory'); + $databases = glob("$dir/db/*.ndb4"); + $this->assertCount(1, $databases, 'data next to noneDB.php, not in the working directory'); + $this->assertSame('t', json_decode((string)file_get_contents($databases[0] . '/FORMAT'), true)['name']); } /** @test S-12 */ @@ -266,19 +313,18 @@ public function dropDbRemovesEveryFile(): void $db->createSpatialIndex('dd', 'loc'); $r = $db->dropDB('dd'); $this->assertTrue($r['success']); - $hash = hash_pbkdf2('sha256', 'dd', self::SECRET, 1000, 20); - $left = array_filter(glob($this->testDbDir . $hash . '*'), function ($f) { return substr($f, -5) !== '.lock'; }); - $this->assertSame([], array_values($left)); + // Everything but LOCK goes; LOCK and the directory stay (STORAGE_FORMAT.md §8.2) + $this->assertSame(['LOCK'], $this->dbFiles('dd')); + $this->assertSame(['LOCK'], $this->entries($this->dbPath('dd', 'home'))); $this->assertSame([], $db->find('dd', 0)); $this->assertFalse($db->dropDB('dd')['success']); $db->insert('dd', ['fresh' => true]); $this->assertSame([['fresh' => true, 'key' => 0]], $db->find('dd', 0)); } - /** @test S-04 migrateKey */ - public function migrateKeyMovesEverything(): void + /** Databases of the migrateKey tests: a plain one with a field index, a spatial one, a sharded one */ + private function keyRotationStore(string $dir): void { - $dir = $this->scratch('mk') . '/'; $old = new \noneDB(['secretKey' => 'old-key-abc', 'dbDir' => $dir, 'shardSize' => 5]); $old->insert('plain', [['city' => 'A'], ['city' => 'B']]); $old->createFieldIndex('plain', 'city'); @@ -291,49 +337,125 @@ public function migrateKeyMovesEverything(): void $old->insert('sharded', $rows); unset($old); \noneDB::clearStaticCache(); + } - $r = \noneDB::migrateKey($dir, 'old-key-abc', 'brand-new-key-0123456789abcdef'); - sort($r['databases']); - $this->assertTrue($r['success']); - $this->assertSame(['geo', 'plain', 'sharded'], $r['databases']); + /** + * Everything the rotation must leave (STORAGE_FORMAT.md §9.3): the databases under the new key's locators + * with their contents, nothing under the old ones, STORE listing the new key instead of the old one, the + * old key refused with KEY_NOT_REGISTERED + */ + private function assertRotated(string $dir, array $storeBefore, array $result, string $label): void + { + $this->assertSame(['code', 'databases', 'error', 'storeManifest', 'success'], $this->sortedKeys($result), $label); + $this->assertTrue($result['success'], $label . ': ' . json_encode($result)); + $this->assertNull($result['error'], $label); + $this->assertNull($result['code'], $label); + $databases = $result['databases']; + sort($databases); + $this->assertSame(['geo', 'plain', 'sharded'], $databases, $label); + + foreach (['plain', 'geo', 'sharded'] as $name) { + $this->assertDirectoryDoesNotExist($dir . $this->locatorOf($name, 'old-key-abc') . '.ndb4', "$label: nothing of $name left under the old key"); + $this->assertFileExists($dir . $this->locatorOf($name, 'brand-new-key-0123456789abcdef') . '.ndb4/FORMAT', "$label: $name renamed"); + } + $this->assertSame([], preg_grep('/\.ndb4$/', array_diff($this->entries($dir), array_map(function ($name) { + return $this->locatorOf($name, 'brand-new-key-0123456789abcdef') . '.ndb4'; + }, ['plain', 'geo', 'sharded']))), "$label: no other database directory"); + $this->assertFileDoesNotExist($dir . '.nonedb-maintenance/journal.json', "$label: journal closed"); + // STORE rewritten: both copies, the new keyCheck instead of the old one, seq one higher (§3.1) + $store = $this->storeRecord($dir, 'STORE'); + $this->assertSame($store, $this->storeRecord($dir, 'STORE.2'), "$label: both copies identical"); + $this->assertSame([$this->keyCheckOf('brand-new-key-0123456789abcdef')], $store['keys'], $label); + $this->assertSame($storeBefore['seq'] + 1, $store['seq'], $label); + $this->assertSame($storeBefore['storeId'], $store['storeId'], $label); + $this->assertSame($store, $result['storeManifest'], "$label: the new STORE record is returned (§9.4)"); + + \noneDB::clearStaticCache(); $new = new \noneDB(['secretKey' => 'brand-new-key-0123456789abcdef', 'dbDir' => $dir, 'autoCreateDB' => false]); - $this->assertSame([1], array_column($new->find('plain', ['city' => 'B']), 'key')); - $this->assertSame(['city'], $new->getFieldIndexes('plain')['fields']); - $this->assertSame(11, $new->find('sharded', ['key' => 11])[0]['i']); - $this->assertCount(1, $new->withinDistance('geo', 'loc', 29.0, 41.0, 100)); + $this->assertSame([1], array_column($new->find('plain', ['city' => 'B']), 'key'), $label); + $this->assertSame(['city'], $new->getFieldIndexes('plain')['fields'], $label); + $this->assertSame(11, $new->find('sharded', ['key' => 11])[0]['i'], $label); + $this->assertSame(range(0, 11), array_column($new->find('sharded', 0), 'i'), $label); + $this->assertCount(1, $new->withinDistance('geo', 'loc', 29.0, 41.0, 100), $label); + $this->assertSame([], preg_grep('/ m( |$)/', [trim((string)file_get_contents($dir . $this->locatorOf('plain', 'brand-new-key-0123456789abcdef') . '.ndb4/LOCK'))]), "$label: m cleared"); \noneDB::clearStaticCache(); - $this->assertFalse((new \noneDB(['secretKey' => 'old-key-abc', 'dbDir' => $dir, 'autoCreateDB' => false]))->find('plain', 0)); - $this->assertFalse(\noneDB::migrateKey($dir, 'old-key-abc', 'CHANGE_THIS_TO_A_SECURE_RANDOM_STRING')['success']); - foreach (['plain', 'geo', 'sharded'] as $name) { - $this->assertSame([], glob($dir . hash_pbkdf2('sha256', $name, 'old-key-abc', 1000, 20) . '*'), "nothing of $name left under the old key"); + try { + (new \noneDB(['secretKey' => 'old-key-abc', 'dbDir' => $dir, 'autoCreateDB' => false]))->find('plain', 0); + $this->fail("$label: the old key must be refused"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $e->getCode(), "$label: " . $e->getMessage()); } } - /** @test S-04: databases from before v3.2 (no lock files) leave nothing behind under the old key */ - public function migrateKeyOfPreLockDatabases(): void + private function sortedKeys(array $result): array { - $dir = $this->scratch('mk31') . '/'; - $old = new \noneDB(['secretKey' => 'old-key-abc', 'dbDir' => $dir, 'shardSize' => 5]); - $rows = []; - for ($i = 0; $i < 12; $i++) { - $rows[] = ['i' => $i]; - } - $old->insert('users', $rows); - unset($old); - \noneDB::clearStaticCache(); // closes kept lock handles - foreach (glob($dir . '*.nonedb.lock') as $lock) { - $this->assertTrue(unlink($lock)); // pre-3.2 versions had no lock files + $keys = array_keys($result); + sort($keys); + return $keys; + } + + /** @test S-04 migrateKey: the database directories move to the new key's locators, STORE lists the new key */ + public function migrateKeyMovesEverything(): void + { + $dir = $this->scratch('mk') . '/'; + $this->keyRotationStore($dir); + $storeBefore = $this->storeRecord($dir); + $this->assertSame([$this->keyCheckOf('old-key-abc')], $storeBefore['keys'], 'set up'); + $formats = []; + foreach (['plain', 'geo', 'sharded'] as $name) { + $formats[$name] = file_get_contents($dir . $this->locatorOf($name, 'old-key-abc') . '.ndb4/FORMAT'); } - $oldHash = hash_pbkdf2('sha256', 'users', 'old-key-abc', 1000, 20); - $before = count(glob($dir . $oldHash . '*')); $r = \noneDB::migrateKey($dir, 'old-key-abc', 'brand-new-key-0123456789abcdef'); - $this->assertTrue($r['success']); - $this->assertSame($before, $r['files']); - $this->assertSame([], glob($dir . $oldHash . '*'), 'no lock file (or anything else) left under the old key'); - $new = new \noneDB(['secretKey' => 'brand-new-key-0123456789abcdef', 'dbDir' => $dir, 'autoCreateDB' => false]); - $this->assertSame(range(0, 11), array_column($new->find('users', 0), 'i')); + $this->assertRotated($dir, $storeBefore, $r, 'rotated'); + foreach ($formats as $name => $format) { + // FORMAT holds no key material: the directory is renamed, its identity unchanged (§4.1, §9.3) + $this->assertSame($format, file_get_contents($dir . $this->locatorOf($name, 'brand-new-key-0123456789abcdef') . '.ndb4/FORMAT'), $name); + } + + // A placeholder key is refused, and the old key is not registered any more + $placeholder = \noneDB::migrateKey($dir, 'brand-new-key-0123456789abcdef', 'CHANGE_THIS_TO_A_SECURE_RANDOM_STRING'); + $this->assertFalse($placeholder['success']); + $again = \noneDB::migrateKey($dir, 'old-key-abc', 'yet-another-key-0123456789'); + $this->assertFalse($again['success']); + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $again['code']); + $this->assertSame($r['storeManifest'], $this->storeRecord($dir), 'nothing changed by the refused runs'); + } + + /** + * @test S-04 migrateKey: a rotation interrupted at any step (process killed) keeps the store closed + * (MAINTENANCE_IN_PROGRESS, STORAGE_FORMAT.md §9.1) and is finished by the next run (§9.2 "On re-run") + */ + public function anInterruptedKeyRotationIsFinishedByTheNextRun(): void + { + foreach (['rotate:marked', 'rotate:renamed', 'rotate:committed', 'maintenance:store-written'] as $point) { + $dir = $this->scratch('mkr') . '/'; + $this->keyRotationStore($dir); + $storeBefore = $this->storeRecord($dir); + $out = $this->runChild(' + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === ' . var_export($point, true) . ') { exit(0); } }); + \noneDB::migrateKey(' . var_export($dir, true) . ', "old-key-abc", "brand-new-key-0123456789abcdef"); + echo "not reached"; + '); + $this->assertSame('', $out, $point); + $this->assertFileExists($dir . '.nonedb-maintenance/journal.json', "$point: the journal stays open"); + + // Until the rotation is finished, neither key opens the store + foreach (['old-key-abc', 'brand-new-key-0123456789abcdef'] as $key) { + \noneDB::clearStaticCache(); + try { + (new \noneDB(['secretKey' => $key, 'dbDir' => $dir, 'autoCreateDB' => false]))->find('plain', 0); + $this->fail("$point: the store must stay closed for $key"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $e->getCode(), "$point, $key: " . $e->getMessage()); + } + } + + \noneDB::clearStaticCache(); + $r = \noneDB::migrateKey($dir, 'old-key-abc', 'brand-new-key-0123456789abcdef'); + $this->assertRotated($dir, $storeBefore, $r, $point); + } } /** @test O-19 */ diff --git a/tests/Feature/ShardedFieldIndexTest.php b/tests/Integration/ShardedFieldIndexTest.php similarity index 51% rename from tests/Feature/ShardedFieldIndexTest.php rename to tests/Integration/ShardedFieldIndexTest.php index 329bfa0..c79bfee 100644 --- a/tests/Feature/ShardedFieldIndexTest.php +++ b/tests/Integration/ShardedFieldIndexTest.php @@ -1,12 +1,12 @@ setPrivateProperty('shardSize', 100); $this->setPrivateProperty('shardingEnabled', true); - $this->setPrivateProperty('autoMigrate', true); - $this->cleanupTestFiles(); } - protected function tearDown(): void + private function isDbSharded() { - $this->cleanupTestFiles(); - parent::tearDown(); + return $this->invokePrivateMethod('isSharded', [$this->testDbName]); } - private function cleanupTestFiles() + private function getGlobalIndexPath($field) { - $files = glob($this->testDbDir . '*' . $this->testDbName . '*'); - foreach ($files as $file) { - @unlink($file); - } - noneDB::clearStaticCache(); + return $this->dbPath($this->testDbName, 'gfx', $field); } - private function isDbSharded() + /** + * The global field index checkpoint, decoded by the test's own reader (STORAGE_FORMAT.md §6.5.2): its header, + * the filter of each shard, and the shards whose filter may hold a value + */ + private function readGlobalIndex($field) { - return $this->invokePrivateMethod('isSharded', [$this->testDbName]); + [$header, $filters] = $this->readGlobalFieldFile($this->getGlobalIndexPath($field)); + return ['header' => $header, 'filters' => $filters, 'holders' => function ($value) use ($filters) { + $shards = []; + foreach ($filters as $shard => $filter) { + if ($this->filterMayHold($filter, $value)) { + $shards[] = $shard; + } + } + return $shards; + }]; } - private function getGlobalIndexPath($field) + /** + * Shards a find() on city = $value reads: the global index's holders, the shards whose newer + * lines may hold it, and the shards it has no valid entry for + */ + private function shardsFor($value) { - // Must use sanitized dbname (createFieldIndex sanitizes the name) - $sanitizedName = $this->invokePrivateMethod('normalizeDbName', [$this->testDbName]); - $hash = $this->invokePrivateMethod('hashDBName', [$sanitizedName]); - return $this->testDbDir . $hash . "-" . $sanitizedName . ".nonedb.gfidx." . $field; + $name = $this->invokePrivateMethod('normalizeDbName', [$this->testDbName]); + $shards = $this->invokePrivateMethod('targetShardsFromGlobalIndex', [$name, ['city' => $value]]); + sort($shards); + return $shards; + } + + /** + * Access per shard of a find on city = $value (explain(): skip / index / scan) + */ + private function planFor($value) + { + return array_column($this->noneDB->query($this->testDbName)->where(['city' => $value])->explain()['plan'], 'access', 'segment'); } // ==================== GLOBAL FIELD INDEX CREATE TESTS ==================== @@ -71,23 +89,33 @@ public function testCreateFieldIndexCreatesGlobalMetadata() $this->assertTrue($result['success']); $this->assertEquals(3, $result['shards']); // 3 shards indexed + $this->assertSame(['success' => true, 'indexed' => 300, 'error' => null, 'shards' => 3], $result); // Check global field index file exists - $gfidxPath = $this->getGlobalIndexPath('city'); - $this->assertFileExists($gfidxPath); - - // Verify global metadata structure - $content = file_get_contents($gfidxPath); - $globalMeta = json_decode($content, true); - - $this->assertEquals(2, $globalMeta['v']); // v2 since 3.2.0 (v1 files may be incomplete and are rebuilt) - $this->assertEquals('city', $globalMeta['field']); - $this->assertArrayHasKey('shardMap', $globalMeta); - - // Check shardMap - Istanbul only in shard 0, Ankara only in shard 1, Izmir only in shard 2 - $this->assertEquals([0], $globalMeta['shardMap']['Istanbul']); - $this->assertEquals([1], $globalMeta['shardMap']['Ankara']); - $this->assertEquals([2], $globalMeta['shardMap']['Izmir']); + $gfxPath = $this->getGlobalIndexPath('city'); + $this->assertFileExists($gfxPath); + + // Verify global metadata structure (a checkpoint with an entry per shard; header kind "gfx", v 5: + // docs/STORAGE_FORMAT.md §5.1, §6.5, §6.5.2) + $global = $this->readGlobalIndex('city'); + + $this->assertEquals(1, $global['header']['v']); + $this->assertEquals('gfx', $global['header']['kind']); + $this->assertEquals('city', $global['header']['field']); + $this->assertEquals([0, 1, 2], array_keys($global['header']['sh'])); + foreach ($global['header']['sh'] as $shard => $entry) { + // Each entry covers its whole shard: the bytes of its data file + $this->assertSame(filesize($this->dbPath($this->testDbName, 'data', $shard)), $entry['s']); + } + // One value per shard: 64 bits (the minimum, ≥ 32 per value), 8 shards wide (the minimum) + $this->assertSame([64, 8], [$global['header']['bits'], $global['header']['width']]); + $this->assertSame([0 => $this->shardFilter(['Istanbul'], 64), 1 => $this->shardFilter(['Ankara'], 64), 2 => $this->shardFilter(['Izmir'], 64)], $global['filters']); + + // Istanbul only in shard 0, Ankara only in shard 1, Izmir only in shard 2 + $this->assertEquals([0], $global['holders']('Istanbul')); + $this->assertEquals([1], $global['holders']('Ankara')); + $this->assertEquals([2], $global['holders']('Izmir')); + $this->assertEquals([1], $this->shardsFor('Ankara')); } public function testGlobalFieldIndexWithValueInMultipleShards() @@ -107,14 +135,9 @@ public function testGlobalFieldIndexWithValueInMultipleShards() $this->noneDB->createFieldIndex($this->testDbName, 'city'); - // Read global metadata - $gfidxPath = $this->getGlobalIndexPath('city'); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - // Istanbul should be in all 3 shards - $this->assertContains(0, $globalMeta['shardMap']['Istanbul']); - $this->assertContains(1, $globalMeta['shardMap']['Istanbul']); - $this->assertContains(2, $globalMeta['shardMap']['Istanbul']); + $this->assertEquals([0, 1, 2], $this->readGlobalIndex('city')['holders']('Istanbul')); + $this->assertEquals([0, 1, 2], $this->shardsFor('Istanbul')); } // ==================== SHARD-SKIP FIND TESTS ==================== @@ -139,6 +162,9 @@ public function testFindUsesShardSkipOptimization() foreach ($result as $record) { $this->assertEquals('Istanbul', $record['city']); } + $this->assertSame(range(0, 99), array_column($result, 'key')); + $this->assertEquals([0], $this->shardsFor('Istanbul')); + $this->assertSame([0 => 'index', 1 => 'skip', 2 => 'skip'], $this->planFor('Istanbul'), 'shards 1 and 2 are skipped'); } public function testFindWithValueInMultipleShards() @@ -159,6 +185,9 @@ public function testFindWithValueInMultipleShards() $result = $this->noneDB->find($this->testDbName, ['city' => 'Istanbul']); $this->assertCount(200, $result); // 100 from shard 0 + 100 from shard 2 + $this->assertSame(array_merge(range(0, 99), range(200, 299)), array_column($result, 'key')); + $this->assertEquals([0, 2], $this->shardsFor('Istanbul')); + $this->assertSame([0 => 'index', 1 => 'skip', 2 => 'index'], $this->planFor('Istanbul')); } public function testFindWithNonExistentValue() @@ -176,7 +205,9 @@ public function testFindWithNonExistentValue() // Find non-existent value - should return empty immediately $result = $this->noneDB->find($this->testDbName, ['city' => 'Izmir']); - $this->assertEmpty($result); + $this->assertSame([], $result); + $this->assertSame([], $this->shardsFor('Izmir'), 'no shard is read'); + $this->assertSame('none', $this->noneDB->query($this->testDbName)->where(['city' => 'Izmir'])->explain()['access']); } // ==================== INSERT UPDATES GLOBAL INDEX ==================== @@ -195,20 +226,20 @@ public function testInsertUpdatesGlobalFieldIndex() $this->noneDB->createFieldIndex($this->testDbName, 'city'); // Verify Istanbul is only in shard 0 - $gfidxPath = $this->getGlobalIndexPath('city'); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - $this->assertEquals([0], $globalMeta['shardMap']['Istanbul']); + $gfxPath = $this->getGlobalIndexPath('city'); + $this->assertEquals([0], $this->readGlobalIndex('city')['holders']('Istanbul')); + $file = file_get_contents($gfxPath); // Insert Istanbul record - will go to shard 2 (shards 0 and 1 are full) $this->noneDB->insert($this->testDbName, ['name' => 'NewUser', 'city' => 'Istanbul']); - // Refresh and check global index was updated + // A write doesn't rewrite the global index; shard 2 has no entry yet, so it is read noneDB::clearStaticCache(); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - - // Istanbul should now be in shards 0 and 2 (not 1, because new shard 2 was created) - $this->assertContains(0, $globalMeta['shardMap']['Istanbul']); - $this->assertContains(2, $globalMeta['shardMap']['Istanbul']); + $this->assertSame($file, file_get_contents($gfxPath)); + $this->assertEquals([0, 2], $this->shardsFor('Istanbul')); + $this->assertEquals([1, 2], $this->shardsFor('Ankara'), 'shard 2 has no entry: read for every value'); + $this->assertCount(101, $this->noneDB->find($this->testDbName, ['city' => 'Istanbul'])); + $this->assertSame(array_merge(range(0, 99), [200]), array_column($this->noneDB->find($this->testDbName, ['city' => 'Istanbul']), 'key')); } public function testInsertNewValueCreatesGlobalEntry() @@ -226,20 +257,17 @@ public function testInsertNewValueCreatesGlobalEntry() // Insert new city $this->noneDB->insert($this->testDbName, ['name' => 'IzmirUser', 'city' => 'Izmir']); - // Check global index - $gfidxPath = $this->getGlobalIndexPath('city'); + // Izmir is found in shard 2 (no entry yet: read) and nowhere else noneDB::clearStaticCache(); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - - // Izmir should be in the global index - $this->assertArrayHasKey('Izmir', $globalMeta['shardMap']); + $this->assertEquals([2], $this->shardsFor('Izmir')); + $this->assertSame(['Izmir'], array_column($this->noneDB->find($this->testDbName, ['city' => 'Izmir']), 'city')); } // ==================== DELETE UPDATES GLOBAL INDEX ==================== /** - * Test that deleting all records with a specific value from a shard - * removes that shard from the global field index's shardMap. + * Test that deleting all records with a specific value from a shard keeps lookups exact, and + * that the value leaves the global field index once the shard's index is checkpointed (rebuilt). */ public function testDeleteUpdatesGlobalFieldIndex() { @@ -263,32 +291,35 @@ public function testDeleteUpdatesGlobalFieldIndex() $this->noneDB->createFieldIndex($this->testDbName, 'city'); // Verify global index structure before delete - $gfidxPath = $this->getGlobalIndexPath('city'); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); + $global = $this->readGlobalIndex('city'); // Istanbul should only be in shard 0 - $this->assertEquals([0], $globalMeta['shardMap']['Istanbul']); + $this->assertEquals([0], $global['holders']('Istanbul')); // Ankara should only be in shard 0 - $this->assertEquals([0], $globalMeta['shardMap']['Ankara']); + $this->assertEquals([0], $global['holders']('Ankara')); // Izmir should only be in shard 1 - $this->assertEquals([1], $globalMeta['shardMap']['Izmir']); + $this->assertEquals([1], $global['holders']('Izmir')); // Delete ALL Istanbul records from shard 0 $this->noneDB->delete($this->testDbName, ['city' => 'Istanbul']); - // Re-read global index - Istanbul should be removed since no more Istanbul records exist + // A value leaves the global index when the shard's index is checkpointed; until then the + // shard is read for it (and holds none) noneDB::clearStaticCache(); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - - // Istanbul should no longer be in shardMap (or empty array) - $this->assertTrue( - !isset($globalMeta['shardMap']['Istanbul']) || empty($globalMeta['shardMap']['Istanbul']), - 'Istanbul should be removed from global index after deleting all Istanbul records' - ); - - // Ankara and Izmir should still exist - $this->assertEquals([0], $globalMeta['shardMap']['Ankara']); - $this->assertEquals([1], $globalMeta['shardMap']['Izmir']); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['city' => 'Istanbul'])); + $this->assertContains($this->shardsFor('Istanbul'), [[], [0]]); + + // Ankara and Izmir still exist + $this->assertEquals([0], $this->shardsFor('Ankara')); + $this->assertEquals([1], $this->shardsFor('Izmir')); + + // A rebuilt global index has the data's values only + $this->noneDB->rebuildFieldIndex($this->testDbName, 'city'); + $global = $this->readGlobalIndex('city'); + $this->assertSame([0 => $this->shardFilter(['Ankara'], 64), 1 => $this->shardFilter(['Izmir'], 64)], $global['filters']); + $this->assertSame([], $global['holders']('Istanbul')); + $this->assertEquals([0], $global['holders']('Ankara')); + $this->assertEquals([1], $global['holders']('Izmir')); } // ==================== UPDATE UPDATES GLOBAL INDEX ==================== @@ -312,12 +343,11 @@ public function testUpdateUpdatesGlobalFieldIndex() $this->noneDB->createFieldIndex($this->testDbName, 'city'); // Verify global index structure before update - $gfidxPath = $this->getGlobalIndexPath('city'); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); + $global = $this->readGlobalIndex('city'); - $this->assertEquals([0], $globalMeta['shardMap']['Istanbul']); - $this->assertEquals([1], $globalMeta['shardMap']['Ankara']); - $this->assertFalse(isset($globalMeta['shardMap']['Izmir'])); + $this->assertEquals([0], $global['holders']('Istanbul')); + $this->assertEquals([1], $global['holders']('Ankara')); + $this->assertSame([], $global['holders']('Izmir')); // Update ALL Istanbul records in shard 0 to Izmir $this->noneDB->update($this->testDbName, [ @@ -325,21 +355,23 @@ public function testUpdateUpdatesGlobalFieldIndex() ['set' => ['city' => 'Izmir']] ]); - // Re-read global index + // Shard 0's newer lines (or its new data file, after compaction) are read for Izmir noneDB::clearStaticCache(); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); - - // Istanbul should be removed (no more Istanbul records in shard 0) - $this->assertTrue( - !isset($globalMeta['shardMap']['Istanbul']) || empty($globalMeta['shardMap']['Istanbul']), - 'Istanbul should be removed from global index after updating all Istanbul records' - ); - - // Izmir should now be in shard 0 - $this->assertContains(0, $globalMeta['shardMap']['Izmir']); - - // Ankara should still be in shard 1 - $this->assertEquals([1], $globalMeta['shardMap']['Ankara']); + $this->assertEquals([0], $this->shardsFor('Izmir')); + $this->assertCount(100, $this->noneDB->find($this->testDbName, ['city' => 'Izmir'])); + $this->assertSame(range(0, 99), array_column($this->noneDB->find($this->testDbName, ['city' => 'Izmir']), 'key')); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['city' => 'Istanbul'])); + + // Ankara should still be in shard 1 only + $this->assertEquals([1], $this->shardsFor('Ankara')); + + // A rebuilt global index has the data's values only + $this->noneDB->rebuildFieldIndex($this->testDbName, 'city'); + $global = $this->readGlobalIndex('city'); + $this->assertSame([0 => $this->shardFilter(['Izmir'], 64), 1 => $this->shardFilter(['Ankara'], 64)], $global['filters']); + $this->assertSame([], $global['holders']('Istanbul')); + $this->assertEquals([0], $global['holders']('Izmir')); + $this->assertEquals([1], $global['holders']('Ankara')); } // ==================== DROP INDEX TESTS ==================== @@ -357,14 +389,23 @@ public function testDropFieldIndexDeletesGlobalIndex() $this->noneDB->createFieldIndex($this->testDbName, 'city'); // Verify global index exists - $gfidxPath = $this->getGlobalIndexPath('city'); - $this->assertFileExists($gfidxPath); + $gfxPath = $this->getGlobalIndexPath('city'); + $this->assertFileExists($gfxPath); + $shardFiles = glob($this->dbPath($this->testDbName, 'gen') . 'indexes/s*.city.fx'); + $this->assertSame([$this->dbPath($this->testDbName, 'fx', 'city', 0), $this->dbPath($this->testDbName, 'fx', 'city', 1)], $shardFiles); // Drop index - $this->noneDB->dropFieldIndex($this->testDbName, 'city'); + $this->assertSame(['success' => true, 'error' => null], $this->noneDB->dropFieldIndex($this->testDbName, 'city')); - // Global index should be deleted - $this->assertFileDoesNotExist($gfidxPath); + // Global index should be deleted, the shard indexes too + $this->assertFileDoesNotExist($gfxPath); + foreach ($shardFiles as $file) { + $this->assertFileDoesNotExist($file); + } + $this->assertFalse($this->noneDB->hasFieldIndex($this->testDbName, 'city')); + // Lookups still exact, without skipping shards + $this->assertSame(range(100, 199), array_column($this->noneDB->find($this->testDbName, ['city' => 'Ankara']), 'key')); + $this->assertSame([0 => 'scan', 1 => 'scan'], $this->planFor('Ankara')); } // ==================== REBUILD INDEX TESTS ==================== @@ -381,17 +422,24 @@ public function testRebuildFieldIndexRebuildsGlobalIndex() $this->noneDB->insert($this->testDbName, $records); $this->noneDB->createFieldIndex($this->testDbName, 'city'); + // A broken global index: rebuildFieldIndex() must write it again from the data + file_put_contents($this->getGlobalIndexPath('city'), 'garbage'); + noneDB::clearStaticCache(); + // Rebuild index $result = $this->noneDB->rebuildFieldIndex($this->testDbName, 'city'); $this->assertTrue($result['success']); + $this->assertSame(['success' => true, 'indexed' => 200, 'error' => null, 'shards' => 2], $result); // Check global index is valid - $gfidxPath = $this->getGlobalIndexPath('city'); - $globalMeta = json_decode(file_get_contents($gfidxPath), true); + $global = $this->readGlobalIndex('city'); - $this->assertEquals([0], $globalMeta['shardMap']['Istanbul']); - $this->assertEquals([1], $globalMeta['shardMap']['Ankara']); + $this->assertEquals([0], $global['holders']('Istanbul')); + $this->assertEquals([1], $global['holders']('Ankara')); + $this->assertSame([0 => $this->shardFilter(['Istanbul'], 64), 1 => $this->shardFilter(['Ankara'], 64)], $global['filters']); + $this->assertEquals([0, 1], array_keys($global['header']['sh'])); + $this->assertSame([0 => 'skip', 1 => 'index'], $this->planFor('Ankara')); } // ==================== PERFORMANCE TESTS ==================== @@ -425,9 +473,13 @@ public function testShardSkipPerformanceImprovement() // Verify correct results $this->assertCount(10, $result); + $this->assertSame(range(0, 9), array_column($result, 'key')); - // Index should be faster (at least in large datasets) - // For small test, we just verify functionality works + // Timing is not asserted (too noisy); what the optimization guarantees: shards 1..4 are skipped and + // only the 10 matching records of shard 0 are read + $explain = $this->noneDB->query($this->testDbName)->where(['city' => 'Rare'])->explain(true); + $this->assertSame(['total' => 5, 'skipped' => 4, 'indexed' => 1, 'scanned' => 0], $explain['segments']); + $this->assertSame(['recordsRead' => 10, 'segmentsRead' => 1, 'returned' => 10], array_diff_key($explain['actual'], ['ms' => 0])); } // ==================== HELPER METHODS ==================== diff --git a/tests/Feature/ShardingTest.php b/tests/Integration/ShardingTest.php similarity index 59% rename from tests/Feature/ShardingTest.php rename to tests/Integration/ShardingTest.php index 4cf6799..02d54c1 100644 --- a/tests/Feature/ShardingTest.php +++ b/tests/Integration/ShardingTest.php @@ -1,6 +1,6 @@ setPrivateProperty('shardSize', 5); $this->setPrivateProperty('shardingEnabled', true); - $this->setPrivateProperty('autoMigrate', true); + } + + /** Shard file of the test database */ + private function shardFile(int $shard): string + { + return $this->dbPath($this->testDbName, 'data', $shard); + } + + /** Keys stored in a shard file (record lines only) */ + private function shardKeys(int $shard): array + { + $keys = []; + foreach (explode("\n", (string)@file_get_contents($this->shardFile($shard))) as $line) { + $record = json_decode($line, true); + if (is_array($record) && isset($record['key'])) { + $keys[] = $record['key']; + } + } + return $keys; } // ========================================== - // MIGRATION TESTS + // THE SWITCH TO SHARDS // ========================================== /** @@ -37,30 +55,31 @@ public function insertTriggersShardingAtThreshold(): void $result = $this->noneDB->insert($this->testDbName, $data); $this->assertEquals(5, $result['n']); - // Check that database is now sharded + // docs/CONFIGURATION.md shardingEnabled: switched to shards once shardSize (5) keys are assigned: one shard [0, 5) $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertTrue($info['sharded']); - $this->assertEquals(1, $info['shards']); - $this->assertEquals(5, $info['totalRecords']); + $this->assertSame(['sharded' => true, 'shards' => 1, 'totalRecords' => 5, 'deletedCount' => 0, 'shardSize' => 5, 'nextKey' => 5], $info); + $this->assertDatabaseNotExists($this->testDbName); + $this->assertSame(range(0, 4), $this->shardKeys(0)); + $this->assertSame($data, array_map(function ($r) { unset($r['key']); return $r; }, $this->noneDB->find($this->testDbName, 0))); } /** * @test */ - public function legacyDatabaseNotSharded(): void + public function smallDatabaseIsNotSharded(): void { // Insert less than threshold $this->noneDB->insert($this->testDbName, ['name' => 'Test', 'age' => 25]); $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertFalse($info['sharded']); - $this->assertEquals(1, $info['totalRecords']); + // docs/API.md: not sharded: ["sharded" => false, "shards" => 0, "totalRecords" => n, "shardSize" => size, "nextKey" => k] + $this->assertSame(['sharded' => false, 'shards' => 0, 'totalRecords' => 1, 'shardSize' => 5, 'nextKey' => 1], $info); } /** * @test */ - public function manualMigration(): void + public function convertToShardedSwitchesTheDatabase(): void { // Insert some records $this->noneDB->insert($this->testDbName, [ @@ -68,22 +87,26 @@ public function manualMigration(): void ['name' => 'User2', 'age' => 30], ]); - // Manually migrate - $result = $this->noneDB->migrate($this->testDbName); - $this->assertTrue($result['success']); - $this->assertEquals('migrated', $result['status']); + // Switch by hand + $result = $this->noneDB->convertToSharded($this->testDbName); + $this->assertSame(['success' => true, 'status' => 'converted'], $result); - // Verify sharded + // Verify sharded, keys and records kept $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertTrue($info['sharded']); + $this->assertSame(['sharded' => true, 'shards' => 1, 'totalRecords' => 2, 'deletedCount' => 0, 'shardSize' => 5, 'nextKey' => 2], $info); + $this->assertSame([ + ['name' => 'User1', 'age' => 25, 'key' => 0], + ['name' => 'User2', 'age' => 30, 'key' => 1], + ], $this->noneDB->find($this->testDbName, 0)); + $this->assertDatabaseNotExists($this->testDbName); } /** * @test */ - public function migrationPreservesAllData(): void + public function theSwitchToShardsPreservesAllData(): void { - // Insert records before migration + // Insert records before the switch $data = [ ['name' => 'Alice', 'email' => 'alice@test.com'], ['name' => 'Bob', 'email' => 'bob@test.com'], @@ -91,15 +114,16 @@ public function migrationPreservesAllData(): void ]; $this->noneDB->insert($this->testDbName, $data); - // Migrate - $this->noneDB->migrate($this->testDbName); + // Switch + $this->noneDB->convertToSharded($this->testDbName); - // Verify all records are accessible + // Verify all records are accessible, with their keys $all = $this->noneDB->find($this->testDbName, 0); - $this->assertCount(3, $all); - $this->assertEquals('Alice', $all[0]['name']); - $this->assertEquals('Bob', $all[1]['name']); - $this->assertEquals('Charlie', $all[2]['name']); + $this->assertSame([ + ['name' => 'Alice', 'email' => 'alice@test.com', 'key' => 0], + ['name' => 'Bob', 'email' => 'bob@test.com', 'key' => 1], + ['name' => 'Charlie', 'email' => 'charlie@test.com', 'key' => 2], + ], $all); } // ========================================== @@ -120,9 +144,11 @@ public function insertCreatesNewShardWhenFull(): void $this->noneDB->insert($this->testDbName, $data); $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertTrue($info['sharded']); - $this->assertEquals(2, $info['shards']); // 5 + 3 = 2 shards - $this->assertEquals(8, $info['totalRecords']); + // key K lives in shard intdiv(K, 5): keys 0-4 in shard 0, 5-7 in shard 1 + $this->assertSame(['sharded' => true, 'shards' => 2, 'totalRecords' => 8, 'deletedCount' => 0, 'shardSize' => 5, 'nextKey' => 8], $info); + $this->assertSame(range(0, 4), $this->shardKeys(0)); + $this->assertSame([5, 6, 7], $this->shardKeys(1)); + $this->assertFileDoesNotExist($this->shardFile(2)); } /** @@ -137,13 +163,17 @@ public function insertSingleRecordToShardedDB(): void } $this->noneDB->insert($this->testDbName, $data); - // Insert a single record + // Insert a single record: key 5, the first of shard 1 $result = $this->noneDB->insert($this->testDbName, ['name' => 'NewUser']); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); // Verify $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertEquals(6, $info['totalRecords']); + $this->assertSame(6, $info['totalRecords']); + $this->assertSame(6, $info['nextKey']); + $this->assertSame(2, $info['shards']); + $this->assertSame([['name' => 'NewUser', 'key' => 5]], $this->noneDB->find($this->testDbName, ['name' => 'NewUser'])); + $this->assertSame([5], $this->shardKeys(1)); } // ========================================== @@ -166,10 +196,9 @@ public function findAllInShardedDB(): void $all = $this->noneDB->find($this->testDbName, 0); $this->assertCount(12, $all); - // Verify order is preserved - for ($i = 0; $i < 12; $i++) { - $this->assertEquals($i, $all[$i]['index']); - } + // Verify order is preserved: key order across shards, record i has key i + $this->assertSame(range(0, 11), array_column($all, 'index')); + $this->assertSame(range(0, 11), array_column($all, 'key')); } /** @@ -184,20 +213,12 @@ public function findByKeyInShardedDB(): void } $this->noneDB->insert($this->testDbName, $data); - // Find by key in first shard - $result = $this->noneDB->find($this->testDbName, ['key' => 2]); - $this->assertCount(1, $result); - $this->assertEquals(2, $result[0]['index']); - - // Find by key in second shard - $result = $this->noneDB->find($this->testDbName, ['key' => 7]); - $this->assertCount(1, $result); - $this->assertEquals(7, $result[0]['index']); - - // Find by key in third shard - $result = $this->noneDB->find($this->testDbName, ['key' => 10]); - $this->assertCount(1, $result); - $this->assertEquals(10, $result[0]['index']); + // Find by key in first, second and third shard + foreach ([2, 7, 10] as $key) { + $this->assertSame([['name' => 'User' . $key, 'index' => $key, 'key' => $key]], $this->noneDB->find($this->testDbName, ['key' => $key])); + } + $this->assertSame([], $this->noneDB->find($this->testDbName, ['key' => 12]), 'beyond the last shard'); + $this->assertSame([], $this->noneDB->find($this->testDbName, ['key' => 99]), 'a shard that does not exist'); } /** @@ -212,9 +233,10 @@ public function findByMultipleKeysAcrossShards(): void } $this->noneDB->insert($this->testDbName, $data); - // Find multiple keys across shards - $result = $this->noneDB->find($this->testDbName, ['key' => [1, 6, 11]]); - $this->assertCount(3, $result); + // Find multiple keys across shards: in the order of the list + $result = $this->noneDB->find($this->testDbName, ['key' => [11, 1, 6]]); + $this->assertSame([11, 1, 6], array_column($result, 'key')); + $this->assertSame([11, 1, 6], array_column($result, 'index')); } /** @@ -231,10 +253,10 @@ public function findByFilterInShardedDB(): void // Find by filter $evens = $this->noneDB->find($this->testDbName, ['type' => 'even']); - $this->assertCount(5, $evens); + $this->assertSame([0, 2, 4, 6, 8], array_column($evens, 'key')); $odds = $this->noneDB->find($this->testDbName, ['type' => 'odd']); - $this->assertCount(5, $odds); + $this->assertSame([1, 3, 5, 7, 9], array_column($odds, 'key')); } // ========================================== @@ -258,11 +280,12 @@ public function updateInShardedDB(): void ['name' => 'User5'], ['set' => ['status' => 'inactive']] ]); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); - // Verify update + // Verify update: only User5 (key 5, shard 1) $user = $this->noneDB->find($this->testDbName, ['name' => 'User5']); - $this->assertEquals('inactive', $user[0]['status']); + $this->assertSame([['name' => 'User5', 'status' => 'inactive', 'key' => 5]], $user); + $this->assertSame([5], array_column($this->noneDB->find($this->testDbName, ['status' => 'inactive']), 'key')); } /** @@ -282,13 +305,12 @@ public function updateMultipleRecordsAcrossShards(): void [], ['set' => ['updated' => true]] ]); - $this->assertEquals(12, $result['n']); + $this->assertSame(['n' => 12], $result); - // Verify all updated + // Verify all updated, keys unchanged $all = $this->noneDB->find($this->testDbName, 0); - foreach ($all as $record) { - $this->assertTrue($record['updated']); - } + $this->assertSame(array_fill(0, 12, true), array_column($all, 'updated')); + $this->assertSame(range(0, 11), array_column($all, 'key')); } // ========================================== @@ -309,15 +331,16 @@ public function deleteInShardedDB(): void // Delete specific record $result = $this->noneDB->delete($this->testDbName, ['name' => 'User3']); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); // Verify deletion $all = $this->noneDB->find($this->testDbName, 0); - $this->assertCount(9, $all); + $this->assertSame([0, 1, 2, 4, 5, 6, 7, 8, 9], array_column($all, 'key')); $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertEquals(9, $info['totalRecords']); - $this->assertEquals(1, $info['deletedCount']); + $this->assertSame(9, $info['totalRecords']); + $this->assertSame(1, $info['deletedCount']); + $this->assertSame(10, $info['nextKey'], 'a delete never lowers the next key'); } /** @@ -334,14 +357,13 @@ public function deleteMultipleRecordsAcrossShards(): void // Delete all even types $result = $this->noneDB->delete($this->testDbName, ['type' => 'even']); - $this->assertEquals(6, $result['n']); + $this->assertSame(['n' => 6], $result); // Verify $remaining = $this->noneDB->find($this->testDbName, 0); - $this->assertCount(6, $remaining); - foreach ($remaining as $record) { - $this->assertEquals('odd', $record['type']); - } + $this->assertSame([1, 3, 5, 7, 9, 11], array_column($remaining, 'key')); + $this->assertSame(array_fill(0, 6, 'odd'), array_column($remaining, 'type')); + $this->assertSame(6, $this->noneDB->getShardInfo($this->testDbName)['deletedCount']); } // ========================================== @@ -369,15 +391,20 @@ public function compactRemovesDeletedRecords(): void $infoBefore = $this->noneDB->getShardInfo($this->testDbName); $this->assertEquals(3, $infoBefore['deletedCount']); - // Compact + // Compact (docs/API.md result; no shard became empty) $result = $this->noneDB->compact($this->testDbName); - $this->assertTrue($result['success']); - $this->assertEquals(3, $result['freedSlots']); + $this->assertSame(['success' => true, 'freedSlots' => 3, 'totalRecords' => 7, 'sharded' => true, 'newShardCount' => 2], $result); - // After compact + // After compact: keys never change $infoAfter = $this->noneDB->getShardInfo($this->testDbName); - $this->assertEquals(0, $infoAfter['deletedCount']); - $this->assertEquals(7, $infoAfter['totalRecords']); + $this->assertSame(0, $infoAfter['deletedCount']); + $this->assertSame(7, $infoAfter['totalRecords']); + $this->assertSame(10, $infoAfter['nextKey']); + $all = $this->noneDB->find($this->testDbName, 0); + $this->assertSame([0, 1, 3, 4, 6, 7, 9], array_column($all, 'key')); + $this->assertSame([0, 1, 3, 4, 6, 7, 9], array_column($all, 'index')); + $this->assertSame([0, 1, 3, 4], $this->shardKeys(0), 'only live records are left in the data'); + $this->assertSame([6, 7, 9], $this->shardKeys(1)); } /** @@ -401,10 +428,17 @@ public function compactRebuildsShards(): void $result = $this->noneDB->compact($this->testDbName); $this->assertTrue($result['success']); - // Should now have fewer shards since we only have 2 records + // docs/API.md: shards that became empty (0 and 1) are removed, the shard of the newest key (2) stays + $this->assertSame(1, $result['newShardCount']); $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertEquals(1, $info['shards']); // 2 records fits in 1 shard - $this->assertEquals(2, $info['totalRecords']); + $this->assertSame(1, $info['shards']); + $this->assertSame(2, $info['totalRecords']); + $this->assertSame(12, $info['nextKey']); + $this->assertFileDoesNotExist($this->shardFile(0)); + $this->assertFileDoesNotExist($this->shardFile(1)); + $this->assertSame([['name' => 'User10', 'key' => 10], ['name' => 'User11', 'key' => 11]], $this->noneDB->find($this->testDbName, 0), 'keys never change'); + $this->noneDB->insert($this->testDbName, ['name' => 'User12']); + $this->assertSame([10, 11, 12], array_column($this->noneDB->find($this->testDbName, 0), 'key')); } // ========================================== @@ -425,11 +459,7 @@ public function getShardInfoReturnsCorrectData(): void $info = $this->noneDB->getShardInfo($this->testDbName); - $this->assertTrue($info['sharded']); - $this->assertEquals(2, $info['shards']); - $this->assertEquals(8, $info['totalRecords']); - $this->assertEquals(0, $info['deletedCount']); - $this->assertEquals(5, $info['shardSize']); + $this->assertSame(['sharded' => true, 'shards' => 2, 'totalRecords' => 8, 'deletedCount' => 0, 'shardSize' => 5, 'nextKey' => 8], $info); } /** @@ -528,7 +558,7 @@ public function getShardSizeReturnsCorrectValue(): void /** * @test */ - public function migrateAlreadyShardedReturnsAlreadySharded(): void + public function convertingAShardedDatabaseReturnsAlreadySharded(): void { // Create sharded database $data = []; @@ -537,24 +567,23 @@ public function migrateAlreadyShardedReturnsAlreadySharded(): void } $this->noneDB->insert($this->testDbName, $data); - // Try to migrate again - $result = $this->noneDB->migrate($this->testDbName); - $this->assertTrue($result['success']); - $this->assertEquals('already_sharded', $result['status']); + // Try to switch again + $result = $this->noneDB->convertToSharded($this->testDbName); + $this->assertSame(['success' => true, 'status' => 'already_sharded'], $result); // Verify still works $all = $this->noneDB->find($this->testDbName, 0); - $this->assertCount(5, $all); + $this->assertSame(range(0, 4), array_column($all, 'key')); } /** * @test */ - public function migrateNonExistentDatabaseReturnsFalse(): void + public function convertingAMissingDatabaseFails(): void { - $result = $this->noneDB->migrate('nonexistent-db'); - $this->assertFalse($result['success']); - $this->assertEquals('database_not_found', $result['status']); + $result = $this->noneDB->convertToSharded('nonexistent-db'); + $this->assertSame(['success' => false, 'status' => 'database_not_found'], $result); + $this->assertDatabaseNotExists('nonexistent-db'); } // ========================================== @@ -563,7 +592,7 @@ public function migrateNonExistentDatabaseReturnsFalse(): void /** * @test - * v3.0: JSONL format has auto-compaction, so freedSlots may be 0 or 1 + * Deletes compact automatically past the garbage threshold, so compact() finds nothing left to free */ public function compactWorksOnNonShardedDatabase(): void { @@ -577,21 +606,20 @@ public function compactWorksOnNonShardedDatabase(): void ['name' => 'User3'], ]); - // Delete one record (may trigger auto-compaction in JSONL) + // Delete one record: 1 garbage line of 3 record versions is more than the 30% threshold, so the delete + // compacts by itself (docs/CONFIGURATION.md "Compaction Settings") $this->noneDB->delete($this->testDbName, ['name' => 'User2']); - // Compact + // Compact: nothing left to free $result = $this->noneDB->compact($this->testDbName); - $this->assertTrue($result['success']); - // JSONL may have already auto-compacted, so freedSlots can be 0 or 1 - $this->assertGreaterThanOrEqual(0, $result['freedSlots']); - $this->assertEquals(2, $result['totalRecords']); - $this->assertFalse($result['sharded']); + $this->assertSame(['success' => true, 'freedSlots' => 0, 'totalRecords' => 2, 'sharded' => false], $result); - // Verify data is still accessible + // Verify data is still accessible, keys unchanged, and the data file holds only the two records $all = $this->noneDB->find($this->testDbName, 0); - $this->assertCount(2, $all); + $this->assertSame([['name' => 'User1', 'key' => 0], ['name' => 'User3', 'key' => 2]], $all); + $lines = array_values(array_filter(explode("\n", file_get_contents($this->getDbFilePath($this->testDbName))), 'strlen')); + $this->assertCount(2, $lines); } /** @@ -600,8 +628,8 @@ public function compactWorksOnNonShardedDatabase(): void public function compactOnNonExistentDatabaseReturnsFalse(): void { $result = $this->noneDB->compact('nonexistent-db'); - $this->assertFalse($result['success']); - $this->assertEquals('database_not_found', $result['status']); + $this->assertSame(['success' => false, 'freedSlots' => 0, 'status' => 'database_not_found'], $result); + $this->assertDatabaseNotExists('nonexistent-db'); } /** @@ -622,7 +650,6 @@ public function compactReturnsShardedFlag(): void // Compact sharded $result = $this->noneDB->compact($this->testDbName); - $this->assertTrue($result['success']); - $this->assertTrue($result['sharded']); + $this->assertSame(['success' => true, 'freedSlots' => 1, 'totalRecords' => 9, 'sharded' => true, 'newShardCount' => 2], $result); } } diff --git a/tests/Feature/SpatialIndexTest.php b/tests/Integration/SpatialIndexTest.php similarity index 70% rename from tests/Feature/SpatialIndexTest.php rename to tests/Integration/SpatialIndexTest.php index 82e74d8..96fcf39 100644 --- a/tests/Feature/SpatialIndexTest.php +++ b/tests/Integration/SpatialIndexTest.php @@ -2,15 +2,57 @@ /** * Spatial Index Feature Tests * Tests spatial index creation, CRUD integration, and queries - * @version 3.1.0 */ -namespace noneDB\Tests\Feature; +namespace noneDB\Tests\Integration; use noneDB\Tests\noneDBTestCase; +use noneDB\Tests\Integration\SpecSpatialTest as Model; class SpatialIndexTest extends noneDBTestCase { + /** The records of setUp(), in key order */ + private const PLACES = [ + 'Hagia Sophia' => [28.9803, 41.0086], + 'Blue Mosque' => [28.9768, 41.0054], + 'Topkapi Palace' => [28.9833, 41.0115], + 'Grand Bazaar' => [28.9680, 41.0106], + 'Galata Tower' => [28.9741, 41.0256], + ]; + + /** Names within $meters of a point, nearest first (independent haversine, docs: R = 6,371 km) */ + private function expectedWithin(float $lon, float $lat, float $meters, array $places = self::PLACES): array + { + $distances = []; + foreach ($places as $name => [$x, $y]) { + $d = Model::hav($lon, $lat, $x, $y); + if ($d <= $meters) { + $distances[$name] = $d; + } + } + asort($distances); + return $distances; + } + + /** Names whose point lies in the closed box */ + private function expectedInBox(float $minLon, float $minLat, float $maxLon, float $maxLat): array + { + $names = []; + foreach (self::PLACES as $name => [$x, $y]) { + if ($minLon <= $x && $x <= $maxLon && $minLat <= $y && $y <= $maxLat) { + $names[] = $name; + } + } + sort($names); + return $names; + } + + private static function sortedNames(array $rows): array + { + $names = array_column($rows, 'name'); + sort($names); + return $names; + } protected function setUp(): void { parent::setUp(); @@ -36,6 +78,7 @@ public function testCreateSpatialIndex(): void $this->assertTrue($result['success']); $this->assertEmpty($result['error'] ?? ''); + $this->assertSame(['success' => true, 'indexed' => 5, 'error' => null], $result); } /** @@ -48,6 +91,7 @@ public function testCreateSpatialIndexDuplicate(): void $this->assertFalse($result['success']); $this->assertArrayHasKey('error', $result); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field 'location'"], $result); } /** @@ -61,6 +105,7 @@ public function testGetSpatialIndexes(): void $this->assertIsArray($indexes); $this->assertContains('location', $indexes); + $this->assertSame(['location'], $indexes); } /** @@ -75,6 +120,11 @@ public function testDropSpatialIndex(): void $indexes = $this->noneDB->getSpatialIndexes($this->testDbName); $this->assertNotContains('location', $indexes); + $this->assertSame(['success' => true, 'error' => null], $result); + $this->assertSame([], $indexes); + $this->assertFalse($this->noneDB->hasSpatialIndex($this->testDbName, 'location')); + // the data stays and is still queried (by a scan) + $this->assertSame(['Hagia Sophia'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', 28.9803, 41.0086, 10), 'name')); } /** @@ -87,6 +137,9 @@ public function testHasSpatialIndex(): void $this->noneDB->createSpatialIndex($this->testDbName, 'location'); $this->assertTrue($this->noneDB->hasSpatialIndex($this->testDbName, 'location')); + $this->assertFalse($this->noneDB->hasSpatialIndex($this->testDbName, 'name')); + $this->noneDB->dropSpatialIndex($this->testDbName, 'location'); + $this->assertFalse($this->noneDB->hasSpatialIndex($this->testDbName, 'location')); } // ========== WithinDistance Queries ========== @@ -107,6 +160,9 @@ public function testWithinDistance(): void // Hagia Sophia should be in results (distance 0) $names = array_column($results, 'name'); $this->assertContains('Hagia Sophia', $names); + // exactly the places within 1000 m, nearest first (Grand Bazaar is about 1060 m away) + $this->assertSame(array_keys($this->expectedWithin(28.9803, 41.0086, 1000)), $names); + $this->assertSame(['Hagia Sophia', 'Topkapi Palace', 'Blue Mosque'], $names); } /** @@ -140,6 +196,8 @@ public function testWithinBBox(): void $this->assertIsArray($results); $this->assertGreaterThan(0, count($results)); + $this->assertSame($this->expectedInBox(28.97, 41.00, 28.99, 41.02), self::sortedNames($results)); + $this->assertSame(['Blue Mosque', 'Hagia Sophia', 'Topkapi Palace'], self::sortedNames($results)); } /** @@ -175,6 +233,7 @@ public function testNearest(): void // First result should be Hagia Sophia itself (distance 0) $this->assertEquals('Hagia Sophia', $results[0]['name']); + $this->assertSame(array_slice(array_keys($this->expectedWithin(28.9803, 41.0086, INF)), 0, 3), array_column($results, 'name')); } /** @@ -191,6 +250,12 @@ public function testNearestWithDistance(): void $this->assertIsArray($results); $this->assertArrayHasKey('_distance', $results[0]); $this->assertEquals(0, $results[0]['_distance']); // Hagia Sophia distance to itself + $expected = array_slice($this->expectedWithin(28.9803, 41.0086, INF), 0, 3, true); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + foreach ($results as $i => $row) { + $this->assertEqualsWithDelta(array_values($expected)[$i], $row['_distance'], 0.006, $row['name'] . ': meters, 2 decimals'); + $this->assertSame(round($row['_distance'], 2), $row['_distance']); + } } // ========== WithinPolygon Queries ========== @@ -218,6 +283,9 @@ public function testWithinPolygon(): void $this->assertIsArray($results); $this->assertGreaterThan(0, count($results)); + // a rectangle: the same points as the box (docs) + $this->assertSame($this->expectedInBox(28.96, 41.00, 28.99, 41.02), self::sortedNames($results)); + $this->assertSame(['Blue Mosque', 'Grand Bazaar', 'Hagia Sophia', 'Topkapi Palace'], self::sortedNames($results)); } // ========== CRUD Integration ========== @@ -240,6 +308,8 @@ public function testInsertUpdatesSpatialIndex(): void $names = array_column($results, 'name'); $this->assertContains('Maiden Tower', $names); + $this->assertSame(['Maiden Tower'], $names); + $this->assertSame('Maiden Tower', $this->noneDB->nearest($this->testDbName, 'location', 29.0041, 41.0211, 1)[0]['name']); } /** @@ -260,6 +330,8 @@ public function testDeleteRemovesFromSpatialIndex(): void } } $this->assertNotNull($hagiaSophiaKey); + $this->assertSame(['Hagia Sophia'], array_column($results, 'name')); + $this->assertSame(0, $hagiaSophiaKey); // Delete it $this->noneDB->delete($this->testDbName, ['key' => $hagiaSophiaKey]); @@ -268,6 +340,8 @@ public function testDeleteRemovesFromSpatialIndex(): void $results = $this->noneDB->withinDistance($this->testDbName, 'location', 28.9803, 41.0086, 10); $names = array_column($results, 'name'); $this->assertNotContains('Hagia Sophia', $names); + $this->assertSame([], $results); + $this->assertSame(['Topkapi Palace', 'Blue Mosque'], array_column($this->noneDB->nearest($this->testDbName, 'location', 28.9803, 41.0086, 2), 'name')); } /** @@ -298,11 +372,17 @@ public function testUpdateUpdatesSpatialIndex(): void $results = $this->noneDB->withinDistance($this->testDbName, 'location', 32.8597, 39.9334, 1000); $names = array_column($results, 'name'); $this->assertContains('Hagia Sophia', $names); + $this->assertSame(['Hagia Sophia'], $names); // Should NOT find it at old location $results = $this->noneDB->withinDistance($this->testDbName, 'location', 28.9803, 41.0086, 10); $names = array_column($results, 'name'); $this->assertNotContains('Hagia Sophia', $names); + $this->assertSame([], $results); + $moved = self::PLACES; + $moved['Hagia Sophia'] = [32.8597, 39.9334]; + $this->assertSame(array_keys($this->expectedWithin(28.9803, 41.0086, 5000, $moved)), + array_column($this->noneDB->withinDistance($this->testDbName, 'location', 28.9803, 41.0086, 5000), 'name')); } // ========== Query Builder Integration ========== @@ -320,6 +400,9 @@ public function testQueryBuilderWithinDistance(): void $this->assertIsArray($results); $this->assertGreaterThan(0, count($results)); + $expected = array_keys($this->expectedWithin(28.9803, 41.0086, 1000)); + sort($expected); + $this->assertSame($expected, self::sortedNames($results)); } /** @@ -335,6 +418,7 @@ public function testQueryBuilderWithinBBox(): void $this->assertIsArray($results); $this->assertGreaterThan(0, count($results)); + $this->assertSame($this->expectedInBox(28.97, 41.00, 28.99, 41.02), self::sortedNames($results)); } /** @@ -350,6 +434,7 @@ public function testQueryBuilderNearest(): void $this->assertIsArray($results); $this->assertCount(2, $results); + $this->assertSame(array_slice(array_keys($this->expectedWithin(28.9803, 41.0086, INF)), 0, 2), array_column($results, 'name')); } /** @@ -376,6 +461,7 @@ public function testQueryBuilderWithinPolygon(): void $this->assertIsArray($results); $this->assertGreaterThan(0, count($results)); + $this->assertSame($this->expectedInBox(28.96, 41.00, 28.99, 41.02), self::sortedNames($results)); } /** @@ -408,6 +494,7 @@ public function testQueryBuilderSpatialWithWhere(): void $this->assertEquals('restaurant', $r['category']); } } + $this->assertSame(['Restaurant A'], array_column($results, 'name'), 'the only restaurant; records without the field never match'); } /** @@ -432,5 +519,10 @@ public function testQueryBuilderWithDistance(): void $this->assertGreaterThanOrEqual($prevDistance, $r['_distance']); $prevDistance = $r['_distance']; } + $expected = $this->expectedWithin(28.9803, 41.0086, 2000); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + foreach ($results as $r) { + $this->assertEqualsWithDelta($expected[$r['name']], $r['_distance'], 0.00006, 'withDistance: meters, 4 decimals'); + } } } diff --git a/tests/Feature/SpatialOperatorCombinationTest.php b/tests/Integration/SpatialOperatorCombinationTest.php similarity index 73% rename from tests/Feature/SpatialOperatorCombinationTest.php rename to tests/Integration/SpatialOperatorCombinationTest.php index fbf1029..bdfdf73 100644 --- a/tests/Feature/SpatialOperatorCombinationTest.php +++ b/tests/Integration/SpatialOperatorCombinationTest.php @@ -2,15 +2,43 @@ /** * Spatial + Comparison Operator Combination Tests * Tests combining spatial queries with MongoDB-style comparison operators - * @version 3.1.0 */ -namespace noneDB\Tests\Feature; +namespace noneDB\Tests\Integration; use noneDB\Tests\noneDBTestCase; +use noneDB\Tests\Integration\SpecSpatialTest as Model; class SpatialOperatorCombinationTest extends noneDBTestCase { + /** @var array The records of setUp() */ + private $places = []; + + /** + * The places within $meters of (28.978, 41.006) that pass $filter: name => distance, nearest first + * (independent haversine, docs: R = 6,371 km) + */ + private function expected(float $meters, callable $filter): array + { + $out = []; + foreach ($this->places as $place) { + $d = Model::hav(28.978, 41.006, $place['location']['coordinates'][0], $place['location']['coordinates'][1]); + if ($d <= $meters && $filter($place)) { + $out[$place['name']] = $d; + } + } + asort($out); + return $out; + } + + private function assertNames(array $expected, array $results, string $message = ''): void + { + $names = array_column($results, 'name'); + sort($names); + sort($expected); + $this->assertSame($expected, $names, $message); + } + protected function setUp(): void { parent::setUp(); @@ -19,7 +47,7 @@ protected function setUp(): void $this->noneDB->createSpatialIndex($this->testDbName, 'location'); // Insert realistic test data - restaurants in Istanbul - $this->noneDB->insert($this->testDbName, [ + $this->noneDB->insert($this->testDbName, $this->places = [ // Sultanahmet area [ 'name' => 'Ottoman Kitchen', @@ -113,6 +141,7 @@ protected function setUp(): void 'location' => ['type' => 'Point', 'coordinates' => [29.0050, 41.0430]] ] ]); + $this->assertCount(8, $this->noneDB->find($this->testDbName, 0), 'setup'); } // ========== SPATIAL + $gt/$gte COMBINATIONS ========== @@ -133,6 +162,8 @@ public function testWithinDistanceWithGteRating(): void foreach ($results as $record) { $this->assertGreaterThanOrEqual(4.5, $record['rating']); } + $this->assertNames(array_keys($this->expected(2000, function ($p) { return $p['rating'] >= 4.5; })), $results); + $this->assertNames(['Ottoman Kitchen', 'Sultan Kebab'], $results); } /** @@ -149,6 +180,8 @@ public function testWithinDistanceWithGtReviews(): void foreach ($results as $record) { $this->assertGreaterThan(100, $record['review_count']); } + $this->assertNames(array_keys($this->expected(2000, function ($p) { return $p['review_count'] > 100; })), $results); + $this->assertNotEmpty($results); } // ========== SPATIAL + $lt/$lte COMBINATIONS ========== @@ -169,6 +202,15 @@ public function testWithinBBoxWithLtePriceRange(): void foreach ($results as $record) { $this->assertLessThanOrEqual(2, $record['price_range']); } + $inBox = []; + foreach ($this->places as $p) { + [$x, $y] = $p['location']['coordinates']; + if ($x >= 28.97 && $x <= 28.99 && $y >= 41.00 && $y <= 41.02 && $p['price_range'] <= 2) { + $inBox[] = $p['name']; + } + } + $this->assertNames($inBox, $results); + $this->assertNames(['Blue Cafe', 'Sultan Kebab'], $results); } /** @@ -196,6 +238,12 @@ public function testNearestWithLtPrice(): void $results[$i]['_distance'] ); } + // the 10 nearest among the cheap ones (all of them), nearest first; withDistance: 4 decimals + $expected = $this->expected(INF, function ($p) { return $p['price_range'] < 3; }); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + foreach ($results as $record) { + $this->assertEqualsWithDelta($expected[$record['name']], $record['_distance'], 0.00006); + } } // ========== SPATIAL + $ne COMBINATIONS ========== @@ -214,6 +262,8 @@ public function testWithinDistanceWithNeCategory(): void foreach ($results as $record) { $this->assertNotEquals('fast_food', $record['category']); } + $this->assertNames(array_keys($this->expected(5000, function ($p) { return $p['category'] !== 'fast_food'; })), $results); + $this->assertCount(6, $results); } // ========== SPATIAL + $in COMBINATIONS ========== @@ -234,6 +284,8 @@ public function testWithinDistanceWithInCategories(): void foreach ($results as $record) { $this->assertContains($record['category'], ['restaurant', 'cafe']); } + $this->assertNames(array_keys($this->expected(2000, function ($p) { return in_array($p['category'], ['restaurant', 'cafe'], true); })), $results); + $this->assertNames(['Blue Cafe', 'Ottoman Kitchen', 'Sultan Kebab'], $results); } /** @@ -256,6 +308,15 @@ public function testWithinPolygonWithInCuisine(): void foreach ($results as $record) { $this->assertContains($record['cuisine'], ['turkish', 'international']); } + $inside = []; + foreach ($this->places as $p) { + [$x, $y] = $p['location']['coordinates']; + if (Model::inClosedPolygon($x, $y, $polygon['coordinates']) && in_array($p['cuisine'], ['turkish', 'international'], true)) { + $inside[] = $p['name']; + } + } + $this->assertNames($inside, $results); + $this->assertNames(['Blue Cafe', 'Ottoman Kitchen', 'Sultan Kebab'], $results); } // ========== SPATIAL + $nin COMBINATIONS ========== @@ -274,6 +335,8 @@ public function testWithinDistanceWithNinCuisine(): void foreach ($results as $record) { $this->assertNotContains($record['cuisine'], ['turkish', 'french']); } + $this->assertNames(array_keys($this->expected(10000, function ($p) { return !in_array($p['cuisine'], ['turkish', 'french'], true); })), $results); + $this->assertCount(4, $results); } // ========== SPATIAL + RANGE ($gte + $lte) ========== @@ -293,6 +356,8 @@ public function testWithinDistanceWithRangeFilter(): void $this->assertGreaterThanOrEqual(2, $record['price_range']); $this->assertLessThanOrEqual(3, $record['price_range']); } + $this->assertNames(array_keys($this->expected(2000, function ($p) { return $p['price_range'] >= 2 && $p['price_range'] <= 3; })), $results); + $this->assertCount(3, $results); } /** @@ -310,6 +375,8 @@ public function testWithinDistanceWithRatingRange(): void $this->assertGreaterThanOrEqual(4.0, $record['rating']); $this->assertLessThan(4.6, $record['rating']); } + $this->assertNames(array_keys($this->expected(5000, function ($p) { return $p['rating'] >= 4.0 && $p['rating'] < 4.6; })), $results); + $this->assertNotEmpty($results); } // ========== SPATIAL + MIXED OPERATORS ========== @@ -334,6 +401,10 @@ public function testWithinDistanceWithMixedOperators(): void $this->assertLessThanOrEqual(3, $record['price_range']); $this->assertTrue($record['delivery']); } + $this->assertNames(array_keys($this->expected(3000, function ($p) { + return $p['rating'] >= 4.0 && $p['price_range'] <= 3 && $p['delivery'] === true; + })), $results); + $this->assertNames(['Ottoman Kitchen', 'Sultan Kebab'], $results); } /** @@ -359,6 +430,12 @@ public function testNearestWithMultipleOperators(): void $this->assertGreaterThan(4.0, $record['rating']); $this->assertGreaterThanOrEqual(50, $record['review_count']); } + // docs: nearest()'s k after the other filters, then limit(3): the 3 nearest matching ones, nearest first + $expected = $this->expected(INF, function ($p) { + return $p['category'] === 'restaurant' && $p['open_now'] === true && $p['rating'] > 4.0 && $p['review_count'] >= 50; + }); + $this->assertSame(array_slice(array_keys($expected), 0, 3), array_column($results, 'name')); + $this->assertCount(3, $results); } // ========== SPATIAL + $like COMBINATIONS ========== @@ -377,6 +454,8 @@ public function testWithinDistanceWithLikeName(): void foreach ($results as $record) { $this->assertStringContainsStringIgnoringCase('Sultan', $record['name']); } + $this->assertNames(array_keys($this->expected(5000, function ($p) { return stripos($p['name'], 'Sultan') !== false; })), $results); + $this->assertNames(['Sultan Kebab'], $results); } /** @@ -393,6 +472,8 @@ public function testWithinDistanceWithLikeStartsWith(): void foreach ($results as $record) { $this->assertMatchesRegularExpression('/^B/i', $record['name']); } + $this->assertNames(array_keys($this->expected(10000, function ($p) { return stripos($p['name'], 'B') === 0; })), $results); + $this->assertNames(['Blue Cafe', 'Budget Bites'], $results); } // ========== SPATIAL + $exists COMBINATIONS ========== @@ -424,6 +505,8 @@ public function testWithinDistanceWithExists(): void foreach ($results as $record) { $this->assertArrayHasKey('delivery', $record); } + $this->assertNames(array_keys($this->expected(2000, function ($p) { return true; })), $results, 'every place within 2 km but the new one'); + $this->assertCount(3, $results); // Find places without delivery option specified $results = $this->noneDB->query($this->testDbName) @@ -434,6 +517,7 @@ public function testWithinDistanceWithExists(): void foreach ($results as $record) { $this->assertArrayNotHasKey('delivery', $record); } + $this->assertNames(['Mystery Place'], $results); } // ========== SPATIAL + SORT + OPERATORS ========== @@ -456,6 +540,20 @@ public function testSpatialWithOperatorsAndSort(): void $this->assertLessThanOrEqual($prevRating, $record['rating']); $prevRating = $record['rating']; } + $expected = $this->expected(5000, function ($p) { return $p['price_range'] <= 3; }); + uksort($expected, function ($a, $b) { return $this->rating($b) <=> $this->rating($a); }); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + $this->assertSame(['Sultan Kebab', 'Ottoman Kitchen', 'Blue Cafe', 'Street Food Corner', 'Budget Bites'], array_column($results, 'name')); + } + + private function rating(string $name): float + { + foreach ($this->places as $p) { + if ($p['name'] === $name) { + return $p['rating']; + } + } + throw new \LogicException($name); } /** @@ -477,6 +575,11 @@ public function testSpatialWithOperatorsAndDistanceSort(): void $this->assertGreaterThanOrEqual($prevDistance, $record['_distance']); $prevDistance = $record['_distance']; } + $expected = $this->expected(10000, function ($p) { return $p['rating'] >= 4.0; }); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + foreach ($results as $record) { + $this->assertEqualsWithDelta($expected[$record['name']], $record['_distance'], 0.00006, 'withDistance: 4 decimals'); + } } // ========== REAL WORLD SCENARIOS ========== @@ -511,6 +614,13 @@ public function testFoodDeliveryAppScenario(): void $this->assertLessThanOrEqual(3, $record['price_range']); $this->assertGreaterThanOrEqual(4.0, $record['rating']); } + $expected = $this->expected($maxDeliveryRadius, function ($p) { + return in_array($p['cuisine'], ['turkish', 'international'], true) && $p['open_now'] && $p['delivery'] + && $p['price_range'] <= 3 && $p['rating'] >= 4.0; + }); + $this->assertSame(array_keys($expected), array_column($results, 'name')); + $this->assertSame(['Ottoman Kitchen'], array_column($results, 'name')); + $this->assertEqualsWithDelta($expected['Ottoman Kitchen'], $results[0]['_distance'], 0.00006); } /** @@ -535,6 +645,10 @@ public function testTouristAppScenario(): void $this->assertGreaterThanOrEqual(100, $record['review_count']); $this->assertGreaterThanOrEqual(4.5, $record['rating']); } + $this->assertSame(['Sultan Kebab', 'Ottoman Kitchen'], array_column($results, 'name'), 'sorted by review_count desc'); + $this->assertNames(array_keys($this->expected(2000, function ($p) { + return $p['category'] === 'restaurant' && $p['review_count'] >= 100 && $p['rating'] >= 4.5; + })), $results); } /** @@ -555,6 +669,10 @@ public function testExcludeCategoriesScenario(): void $this->assertNotContains($record['category'], ['fast_food', 'cafe']); $this->assertTrue($record['open_now']); } + $this->assertNames(array_keys($this->expected(10000, function ($p) { + return !in_array($p['category'], ['fast_food', 'cafe'], true) && $p['open_now'] === true; + })), $results); + $this->assertNames(['Modern Bistro', 'Ottoman Kitchen', 'Premium Steak House', 'Seafood Paradise'], $results); } /** @@ -576,6 +694,8 @@ public function testBudgetTravelerScenario(): void $this->assertLessThanOrEqual(2, $record['price_range']); $this->assertGreaterThanOrEqual(3.5, $record['rating']); } + $this->assertNames(array_keys($this->expected(5000, function ($p) { return $p['price_range'] <= 2 && $p['rating'] >= 3.5; })), $results); + $this->assertSame([1, 1, 2, 2], array_column($results, 'price_range'), 'sorted by price_range asc'); } // ========== EDGE CASES ========== diff --git a/tests/Feature/SpatialRealWorldTest.php b/tests/Integration/SpatialRealWorldTest.php similarity index 69% rename from tests/Feature/SpatialRealWorldTest.php rename to tests/Integration/SpatialRealWorldTest.php index e5a3966..2850fe9 100644 --- a/tests/Feature/SpatialRealWorldTest.php +++ b/tests/Integration/SpatialRealWorldTest.php @@ -2,15 +2,47 @@ /** * Real-World Spatial Index Tests * Tests that simulate actual use cases and edge cases - * @version 3.1.0 */ -namespace noneDB\Tests\Feature; +namespace noneDB\Tests\Integration; use noneDB\Tests\noneDBTestCase; +use noneDB\Tests\Integration\SpecSpatialTest as Model; class SpatialRealWorldTest extends noneDBTestCase { + /** Keys of the stored records whose $field lies within $meters (independent model), nearest first */ + private function expectedKeys(string $field, float $lon, float $lat, float $meters): array + { + $out = []; + foreach ($this->noneDB->find($this->testDbName, 0) as $record) { + if (isset($record[$field])) { + $d = Model::distance($lon, $lat, $record[$field]); + if ($d <= $meters) { + $out[$record['key']] = $d; + } + } + } + asort($out); + return $out; + } + + /** The expected records (key => model distance), in an order of non-decreasing distance (ties in any order) */ + private function assertSortedByDistance(array $expected, array $rows, string $message = ''): void + { + $keys = array_column($rows, 'key'); + $sorted = $keys; + sort($sorted); + $expectedKeys = array_keys($expected); + sort($expectedKeys); + $this->assertSame($expectedKeys, $sorted, $message); + $previous = -INF; + foreach ($keys as $key) { + $this->assertGreaterThanOrEqual($previous - 0.01, $expected[$key], "$message: sorted by distance"); + $previous = $expected[$key]; + } + } + // ========== BULK OPERATIONS ========== /** @@ -22,6 +54,7 @@ public function testBulkInsertWithSpatialIndex(): void $this->noneDB->createSpatialIndex($this->testDbName, 'location'); // Generate 100 random locations in Istanbul + mt_srand(42); $data = []; for ($i = 0; $i < 100; $i++) { $data[] = [ @@ -43,6 +76,13 @@ public function testBulkInsertWithSpatialIndex(): void // Verify spatial queries work (50km = 50000m) $nearby = $this->noneDB->withinDistance($this->testDbName, 'location', 29.0, 41.0, 50000); $this->assertGreaterThan(0, count($nearby)); + // every point is within 50 km (the area is about 34 x 22 km), nearest first + $this->assertCount(100, $nearby); + $this->assertSortedByDistance($this->expectedKeys('location', 29.0, 41.0, 50000), $nearby); + foreach ([2000, 5000, 10000] as $radius) { + $this->assertSortedByDistance($this->expectedKeys('location', 29.05, 41.02, $radius), + $this->noneDB->withinDistance($this->testDbName, 'location', 29.05, 41.02, $radius), "radius $radius"); + } } /** @@ -78,6 +118,10 @@ public function testBulkDeleteWithSpatialIndex(): void foreach ($all as $record) { $this->assertTrue($record['active']); } + $keys = array_column($all, 'key'); + sort($keys); + $this->assertSame(range(0, 24), $keys); + $this->assertSame([], $this->noneDB->withinBBox($this->testDbName, 'location', 28.925, 40.9, 29.0, 41.1), 'the deleted ones are gone from the index'); } /** @@ -117,6 +161,9 @@ public function testBulkUpdateWithSpatialIndex(): void $this->assertEquals(19, count($inIstanbul)); // 19 left in Istanbul $this->assertEquals(1, count($inAnkara)); // 1 moved to Ankara + $this->assertSame(['Business 10'], array_column($inAnkara, 'name')); + $this->assertNotContains('Business 10', array_column($inIstanbul, 'name')); + $this->assertSortedByDistance($this->expectedKeys('location', 29.0, 41.0, 50000), $inIstanbul); } // ========== EDGE CASES ========== @@ -201,10 +248,18 @@ public function testVeryLargeDistances(): void // 10000km should find Istanbul and NY from Istanbul (10000000m) $results = $this->noneDB->withinDistance($this->testDbName, 'location', 28.9, 41.0, 10000000); $this->assertGreaterThanOrEqual(2, count($results)); + // New York is about 8,060 km and Tokyo about 8,970 km from Istanbul: all three, nearest first + $this->assertSame(['Istanbul', 'New York', 'Tokyo'], array_column($results, 'name')); + $this->assertSame(array_keys($this->expectedKeys('location', 28.9, 41.0, 10000000)), array_column($results, 'key')); + $this->assertSame(['Istanbul', 'New York'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', 28.9, 41.0, 8500000), 'name')); // 20000km should find all (20000000m) $results = $this->noneDB->withinDistance($this->testDbName, 'location', 28.9, 41.0, 20000000); $this->assertCount(3, $results); + // between New York and Tokyo, about 10,850 km + $ny = Model::hav(-74.0, 40.7, 139.7, 35.7); + $this->assertSame(['New York', 'Istanbul', 'Tokyo'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', -74.0, 40.7, $ny + 1), 'name')); + $this->assertSame(['New York', 'Istanbul'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', -74.0, 40.7, $ny - 1000), 'name')); } /** @@ -229,6 +284,18 @@ public function testInternationalDateLine(): void // Distance query works across date line (1000km = 1000000m) $nearby = $this->noneDB->withinDistance($this->testDbName, 'location', 179.0, -18.0, 1000000); $this->assertGreaterThanOrEqual(1, count($nearby)); + $this->assertSame(['Fiji', 'Tonga'], array_column($nearby, 'name'), 'Tonga is about 530 km away'); + + // docs: searches across the antimeridian work: a box with minLon > maxLon, a circle around 180 + $this->noneDB->insert($this->testDbName, ['name' => 'Samoa', 'location' => ['type' => 'Point', 'coordinates' => [-172.0, -13.8]]]); + $this->assertSame(['Fiji', 'Samoa'], array_column($this->noneDB->withinBBox($this->testDbName, 'location', 178, -20, -170, -10), 'name')); + $fromDateLine = $this->noneDB->withinDistance($this->testDbName, 'location', -179.5, -16.0, 1200000, ['includeDistance' => true]); + $expected = $this->expectedKeys('location', -179.5, -16.0, 1200000); + $this->assertSame(array_keys($expected), array_column($fromDateLine, 'key')); + $this->assertSame(['Fiji', 'Tonga', 'Samoa'], array_column($fromDateLine, 'name')); + foreach ($fromDateLine as $row) { + $this->assertEqualsWithDelta($expected[$row['key']], $row['_distance'], 0.006); + } } /** @@ -246,6 +313,12 @@ public function testPolesProximity(): void // Query near pole (100km = 100000m) $results = $this->noneDB->withinDistance($this->testDbName, 'location', 0, 89.5, 100000); $this->assertCount(1, $results); + $this->assertSame(['Arctic Station'], array_column($results, 'name')); + + // across the pole: (180, 89.5) is 1 degree away from (0, 89.5) over the pole, 111.2 km + $this->noneDB->insert($this->testDbName, ['name' => 'Other Side', 'location' => ['type' => 'Point', 'coordinates' => [180, 89.5]]]); + $this->assertSame(['Arctic Station', 'Other Side'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', 0, 89.5, 112000), 'name')); + $this->assertSame(['Arctic Station'], array_column($this->noneDB->withinDistance($this->testDbName, 'location', 0, 89.5, 111000), 'name')); } // ========== COMBINED QUERIES ========== @@ -313,6 +386,10 @@ public function testNearestWithFilters(): void $this->assertTrue($results[0]['open']); // Restaurant B should be first (closest open) $this->assertEquals('Restaurant B', $results[0]['name']); + $this->assertSame(['Restaurant B', 'Restaurant C'], array_column($results, 'name')); + foreach ($results as $row) { + $this->assertEqualsWithDelta(Model::distance(28.980, 41.008, $row['location']), $row['_distance'], 0.00006, 'withDistance replaces nearest\'s _distance'); + } } // ========== GEOMETRY TYPES ========== @@ -351,6 +428,12 @@ public function testMixedGeometryTypes(): void // All should be found within the area $results = $this->noneDB->withinBBox($this->testDbName, 'geometry', 28.96, 40.99, 29.00, 41.02); $this->assertCount(3, $results); + $this->assertSame(['Point Location', 'Road Segment', 'Park Area'], array_column($results, 'name')); + // a box that only the road's and the park's bounding boxes reach + $this->assertSame(['Road Segment', 'Park Area'], array_column($this->noneDB->withinBBox($this->testDbName, 'geometry', 28.9825, 41.0095, 28.984, 41.0098), 'name')); + // a point inside the park is at distance 0 from it; the road passes through the point + $near = $this->noneDB->withinDistance($this->testDbName, 'geometry', 28.980, 41.008, 1, ['includeDistance' => true]); + $this->assertSame([0.0, 0.0, 0.0], array_column($near, '_distance')); } /** @@ -375,6 +458,26 @@ public function testPolygonWithHole(): void // Point in outer ring but outside hole should be found $results = $this->noneDB->withinBBox($this->testDbName, 'boundary', 28.96, 40.99, 29.04, 41.07); $this->assertCount(1, $results); + // docs: withinBBox compares bounding boxes: a box inside the lake still finds the park + $this->assertCount(1, $this->noneDB->withinBBox($this->testDbName, 'boundary', 28.995, 41.025, 29.005, 41.035)); + // a point in the lake is not inside the park: its distance is the one to the lake's shore + $lake = Model::distance(29.0, 41.03, $this->noneDB->find($this->testDbName, 0)[0]['boundary']); + $this->assertGreaterThan(800, $lake, 'model: about 840 m to the nearest shore'); + $this->assertSame([], $this->noneDB->withinDistance($this->testDbName, 'boundary', 29.0, 41.03, $lake - 1)); + $found = $this->noneDB->withinDistance($this->testDbName, 'boundary', 29.0, 41.03, $lake + 1, ['includeDistance' => true]); + $this->assertCount(1, $found); + $this->assertEqualsWithDelta($lake, $found[0]['_distance'], 0.5); + // a point in the park itself is at distance 0 + $this->assertSame(0.0, $this->noneDB->nearest($this->testDbName, 'boundary', 28.98, 41.01, 1)[0]['_distance']); + // withinPolygon: a point record in the lake is not within the park polygon + $park = $this->noneDB->find($this->testDbName, 0)[0]['boundary']; + $this->noneDB->createSpatialIndex($this->testDbName, 'spot'); + $this->noneDB->insert($this->testDbName, [ + ['name' => 'in the lake', 'spot' => ['type' => 'Point', 'coordinates' => [29.0, 41.03]]], + ['name' => 'on the shore', 'spot' => ['type' => 'Point', 'coordinates' => [28.99, 41.03]]], + ['name' => 'in the park', 'spot' => ['type' => 'Point', 'coordinates' => [28.98, 41.01]]], + ]); + $this->assertSame(['on the shore', 'in the park'], array_column($this->noneDB->withinPolygon($this->testDbName, 'spot', $park), 'name')); } // ========== CONCURRENT OPERATIONS ========== @@ -390,6 +493,8 @@ public function testDuplicateSpatialIndexCreate(): void $result2 = $this->noneDB->createSpatialIndex($this->testDbName, 'location'); $this->assertFalse($result2['success']); $this->assertArrayHasKey('error', $result2); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field 'location'"], $result2); + $this->assertSame(['location'], $this->noneDB->getSpatialIndexes($this->testDbName)); } /** @@ -411,11 +516,12 @@ public function testDropAndRecreateSpatialIndex(): void $this->noneDB->dropSpatialIndex($this->testDbName, 'location'); // Recreate - $this->noneDB->createSpatialIndex($this->testDbName, 'location'); + $this->assertSame(['success' => true, 'indexed' => 1, 'error' => null], $this->noneDB->createSpatialIndex($this->testDbName, 'location')); // Should still work (rebuild from data) $results = $this->noneDB->withinDistance($this->testDbName, 'location', 28.9, 41.0, 1000); $this->assertCount(1, $results); + $this->assertSame(['A'], array_column($results, 'name')); } // ========== REAL-WORLD SCENARIOS ========== @@ -462,6 +568,12 @@ public function testFoodDeliveryScenario(): void $this->assertTrue($restaurant['open']); $this->assertLessThanOrEqual($restaurant['delivery_radius_m'], $restaurant['_distance']); } + // both open restaurants are within 10 km and deliver that far; sorted by rating desc + $this->assertSame(['Pizza Place', 'Burger Joint'], array_column($results, 'name')); + $this->assertSame(['Pizza Place', 'Burger Joint'], array_column(array_values($canDeliver), 'name')); + foreach ($results as $row) { + $this->assertEqualsWithDelta(Model::distance(28.982, 41.009, $row['location']), $row['_distance'], 0.00006); + } } /** @@ -507,6 +619,14 @@ public function testRideSharingScenario(): void $nearestDrivers[$i]['_distance'] ); } + // the available drivers, nearest first (D3 ~ 300 m, D1 ~ 200 m, D4 ~ 780 m from the passenger) + $expected = []; + foreach ($this->noneDB->find($this->testDbName, ['status' => 'available']) as $d) { + $expected[$d['driver_id']] = Model::distance($passengerLon, $passengerLat, $d['current_location']); + } + asort($expected); + $this->assertSame(array_keys($expected), array_column($nearestDrivers, 'driver_id')); + $this->assertSame(['D1', 'D3', 'D4'], array_column($nearestDrivers, 'driver_id')); } /** @@ -549,5 +669,8 @@ public function testRealEstateScenario(): void $this->assertEquals('apartment', $property['type']); $this->assertLessThanOrEqual(400000, $property['price']); } + // every property lies in the area; the two apartments under 400k, cheapest first + $this->assertSame(['Budget Studio', 'Modern Condo'], array_column($results, 'title')); + $this->assertCount(4, $this->noneDB->withinPolygon($this->testDbName, 'location', $searchArea)); } } diff --git a/tests/Regression/SpatialRegressionTest.php b/tests/Integration/SpatialRegressionTest.php similarity index 80% rename from tests/Regression/SpatialRegressionTest.php rename to tests/Integration/SpatialRegressionTest.php index e4a50e1..05cf363 100644 --- a/tests/Regression/SpatialRegressionTest.php +++ b/tests/Integration/SpatialRegressionTest.php @@ -1,9 +1,11 @@ assertTrue($r['success']); $this->assertSame(120, $r['indexed']); $this->assertFalse($db->createSpatialIndex('sh', 'loc')['success'], 'second create reports "already exists"'); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field 'loc'"], $db->createSpatialIndex('sh', 'loc')); $more = []; for ($i = 120; $i < 200; $i++) { @@ -45,13 +48,14 @@ public function shardedSpatialIndexWorksAndCoversNewShards(): void } $db->insert('sh', $more); $this->assertCount(200, $db->withinBBox('sh', 'loc', 28.9, 40.9, 29.3, 41.1)); + $this->assertSame(range(150, 179), array_column($db->withinBBox('sh', 'loc', 29.15, 40.9, 29.179, 41.1), 'i'), 'a box inside the new shards'); $this->assertSame(199, $db->nearest('sh', 'loc', 29.199, 41.0, 1)[0]['i']); $this->assertTrue($db->hasSpatialIndex('sh', 'loc')); $this->assertSame(['loc'], $db->getSpatialIndexes('sh')); } /** @test Y-04 */ - public function migrationKeepsTheSpatialIndex(): void + public function theSwitchToShardsKeepsTheSpatialIndex(): void { $db = $this->db(['shardSize' => 30]); $db->insert('mg', ['n' => 'first', 'loc' => $this->point(10, 10)]); @@ -60,12 +64,11 @@ public function migrationKeepsTheSpatialIndex(): void for ($i = 0; $i < 40; $i++) { $rows[] = ['n' => 'p' . $i, 'loc' => $this->point(10 + $i / 100, 10)]; } - $db->insert('mg', $rows); // crosses the threshold: migrates + $db->insert('mg', $rows); // crosses the threshold: switches to shards $this->assertTrue($db->getShardInfo('mg')['sharded']); $this->assertSame(['loc'], $db->getSpatialIndexes('mg')); $this->assertTrue($db->hasSpatialIndex('mg', 'loc')); $this->assertCount(41, $db->withinBBox('mg', 'loc', 9.9, 9.9, 10.5, 10.1)); - $this->assertFileDoesNotExist($this->testDbDir . hash_pbkdf2('sha256', 'mg', self::SECRET, 1000, 20) . '.nonedb.sidx.loc'); $this->assertTrue($db->rebuildSpatialIndex('mg', 'loc')['success']); $this->assertCount(41, $db->withinBBox('mg', 'loc', 9.9, 9.9, 10.5, 10.1)); } @@ -99,7 +102,7 @@ public function randomizedRtreeMatchesBruteForce(): void $radius = mt_rand(1000, 150000); $expected = []; foreach ($all as $r) { - if ($db->haversineDistance($lon, $lat, $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]) <= $radius) { + if (SpecSpatialTest::hav($lon, $lat, $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]) <= $radius) { // independent haversine $expected[] = $r['key']; } } @@ -123,11 +126,14 @@ public function randomizedRtreeMatchesBruteForce(): void $dist = []; foreach ($all as $r) { - $dist[$r['key']] = $db->haversineDistance($lon, $lat, $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]); + $dist[$r['key']] = SpecSpatialTest::hav($lon, $lat, $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]); } asort($dist); $nearest = $db->nearest('rt', 'loc', $lon, $lat, 5); $this->assertEqualsWithDelta(array_slice(array_values($dist), 0, 5), array_column($nearest, '_distance'), 0.01, "nearest query $q"); + foreach ($nearest as $row) { + $this->assertEqualsWithDelta($dist[$row['key']], $row['_distance'], 0.006, "nearest query $q: key {$row['key']}"); + } } } @@ -142,10 +148,17 @@ public function antimeridianAndPolesAreSearched(): void ['n' => 'pole-b', 'loc' => $this->point(180, 89.5)], ]); $db->createSpatialIndex('am', 'loc'); - $this->assertSame(['west', 'east'], array_column($db->withinDistance('am', 'loc', -179.99, 0, 5000, ['includeDistance' => true]), 'n')); + $rows = $db->withinDistance('am', 'loc', -179.99, 0, 5000, ['includeDistance' => true]); + $this->assertSame(['west', 'east'], array_column($rows, 'n')); + $this->assertEqualsWithDelta(SpecSpatialTest::hav(-179.99, 0, -179.99, 0.001), $rows[0]['_distance'], 0.006, 'west: 111.19 m'); + $this->assertEqualsWithDelta(SpecSpatialTest::hav(-179.99, 0, 179.99, 0), $rows[1]['_distance'], 0.006, 'east: 2223.9 m across the antimeridian'); $nearPole = array_column($db->withinDistance('am', 'loc', 0, 88.5, 300000), 'n'); sort($nearPole); $this->assertSame(['pole-a', 'pole-b'], $nearPole); + // pole-b is on the other side of the pole: 1.5 + 0.5 = 2 degrees away over it + $this->assertSame(['pole-a'], array_column($db->withinDistance('am', 'loc', 0, 88.5, 112000), 'n')); + $this->assertEqualsWithDelta(SpecSpatialTest::hav(0, 88.5, 180, 89.5), $db->nearest('am', 'loc', 0, 88.5, 2)[1]['_distance'], 0.006); + $this->assertEqualsWithDelta(2 * 6371000 * M_PI / 180, SpecSpatialTest::hav(0, 88.5, 180, 89.5), 0.001, "model: over the pole"); $this->assertSame(['east', 'west'], array_column($db->withinBBox('am', 'loc', 179.9, -1, -179.9, 1), 'n'), 'box crossing the antimeridian'); } @@ -206,6 +219,14 @@ public function geometrySemantics(): void $this->assertFalse($db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [[[0, 0], [5, 0], [5, 5], [0, 5], [0, 0]], [[6, 6], [7, 6], [7, 7], [6, 6]]]])['valid'], 'hole outside'); $this->assertFalse($db->validateGeoJSON($this->point(NAN, 1))['valid']); $this->assertFalse($db->validateGeoJSON(['type' => 'Point', 'coordinates' => ['1', '2']])['valid'], 'strings are not coordinates'); + // the documented messages (docs/SPATIAL.md, Common Errors) + $this->assertSame('Ring 0 must not intersect itself', $db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [[[0, 0], [10, 10], [10, 0], [0, 10], [0, 0]]]])['error']); + $this->assertSame('Hole 1 must lie inside the outer ring', $db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [[[0, 0], [5, 0], [5, 5], [0, 5], [0, 0]], [[6, 6], [7, 6], [7, 7], [6, 6]]]])['error']); + $this->assertSame('Point coordinates must be finite numbers', $db->validateGeoJSON($this->point(NAN, 1))['error']); + $this->assertSame('Point coordinates must be finite numbers', $db->validateGeoJSON(['type' => 'Point', 'coordinates' => ['1', '2']])['error']); + // the concave polygon's notch: a point there is outside, a chord across it too + $db->insert('gs', [['n' => 'notch-point', 'g' => $this->point(6, 5)], ['n' => 'notch-chord', 'g' => ['type' => 'LineString', 'coordinates' => [[5, 3], [5, 7]]]]]); + $this->assertSame(['inside-line'], array_column($db->withinPolygon('gs', 'g', $c), 'n')); } /** @test O-34 P-05 */ @@ -232,46 +253,13 @@ public function spatialIndexFilesAreEncodedOnce(): void $db = $this->db(); $db->insert('en', ['loc' => $this->point(1, 1)]); $db->createSpatialIndex('en', 'loc'); - $file = $this->testDbDir . hash_pbkdf2('sha256', 'en', self::SECRET, 1000, 20) . '.nonedb.sidx.loc'; - $decoded = json_decode(file_get_contents($file), true); - $this->assertIsArray($decoded); - $this->assertSame(3, $decoded['v']); - } - - /** @test D-08 */ - public function centroidOfPointNeedsNoHashing(): void - { - $method = $this->getPrivateMethod('getGeometryCentroid'); - $this->assertSame([28.9803, 41.0086], $method->invoke($this->noneDB, $this->point(28.9803, 41.0086))); + // A header line and a line of nodes, each JSON encoded once (STORAGE_FORMAT.md §6.5) + [$header, $nodes] = $this->readRTreeFile($this->dbPath('en', 'rx', 'loc')); + $this->assertIsArray($header); + $this->assertSame([1, 'rx', 'loc'], [$header['v'], $header['kind'], $header['field']]); + $this->assertIsArray($nodes); } - /** @test D-08 the memo caches stay bounded and keep caching once full */ - public function memoCachesKeepWorkingWhenFull(): void - { - $db = $this->noneDB; - for ($i = 0; $i < 12000; $i++) { - $db->haversineDistance(0, 0, $i / 1000, 1); - } - $before = $this->getPrivateProperty('distanceCache'); - $this->assertLessThanOrEqual(10000, count($before)); - $d = $db->haversineDistance(10, 10, 20, 20); - $after = $this->getPrivateProperty('distanceCache'); - $this->assertLessThanOrEqual(10000, count($after)); - $this->assertSame([$d], array_values(array_diff_key($after, $before)), 'a new result is still cached'); - $this->assertSame($d, $db->haversineDistance(10, 10, 20, 20)); - - $centroid = $this->getPrivateMethod('getGeometryCentroid'); - for ($i = 0; $i < 6000; $i++) { - $centroid->invoke($db, ['type' => 'LineString', 'coordinates' => [[0, 0], [$i / 100, 1]]]); - } - $before = $this->getPrivateProperty('centroidCache'); - $this->assertLessThanOrEqual(5000, count($before)); - $line = ['type' => 'LineString', 'coordinates' => [[5, 5], [7, 9]]]; - $c = $centroid->invoke($db, $line); - $after = $this->getPrivateProperty('centroidCache'); - $this->assertLessThanOrEqual(5000, count($after)); - $this->assertSame([$c], array_values(array_diff_key($after, $before)), 'a new centroid is still cached'); - } /** * @test Y-07: the expanding radius search must not cut through a rounding tie. A record just outside @@ -330,7 +318,7 @@ public function nearestWithFiltersMatchesBruteForce(): void $expected = []; foreach ($all as $r) { if ($r['t'] <= $tMax) { - $d = round($db->haversineDistance($center[0], $center[1], $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]), 2); + $d = round(SpecSpatialTest::hav($center[0], $center[1], $r['loc']['coordinates'][0], $r['loc']['coordinates'][1]), 2); // independent haversine $expected[] = [$d, $r['key']]; } } diff --git a/tests/Integration/SpecIndexTest.php b/tests/Integration/SpecIndexTest.php new file mode 100644 index 0000000..6376810 --- /dev/null +++ b/tests/Integration/SpecIndexTest.php @@ -0,0 +1,1964 @@ + $y ? 1 : 0); + } + if (is_string($a) && is_string($b)) { + $c = strcmp($a, $b); + return $c < 0 ? -1 : ($c > 0 ? 1 : 0); + } + return null; + } + + /** True when a record's field satisfies one range operator (missing field: never) */ + private static function holds(array $record, string $field, string $op, $operand): bool + { + if (!array_key_exists($field, $record)) { + return false; + } + $c = self::cmp($record[$field], $operand); + if ($c === null) { + return false; + } + switch ($op) { + case '$gt': return $c > 0; + case '$gte': return $c >= 0; + case '$lt': return $c < 0; + case '$lte': return $c <= 0; + } + throw new \LogicException($op); + } + + /** Keys (ascending) of the model rows [key => record] that satisfy $match */ + private static function select(array $rows, callable $match): array + { + $keys = []; + foreach ($rows as $key => $record) { + if ($match($record + ['key' => $key])) { + $keys[] = $key; + } + } + sort($keys); + return $keys; + } + + private static function eq(string $field, $value): callable + { + return function (array $r) use ($field, $value) { return array_key_exists($field, $r) && $r[$field] === $value; }; + } + + private function keys($records): array + { + $this->assertIsArray($records); + return array_column($records, 'key'); + } + + private static function show($value): string + { + if (is_float($value)) { + return is_nan($value) ? 'NAN' : (is_infinite($value) ? ($value > 0 ? 'INF' : '-INF') : var_export($value, true)); + } + return str_replace("\n", '', var_export($value, true)); + } + + /** Values of the equality tests: every type an index bucket can hold, and arrays (never indexed) */ + private static function eqPool(): array + { + return [0, 1, -1, 1.0, -0.0, 0.0, 2.5, '1', '0', '', 'a', 'A', '2.5', '01', true, false, null, [1], ['k' => 'v']]; + } + + // ------------------------------------------------------------------ files and internals + + /** indexes/{segment}.{token}.fx of the active generation (STORAGE_FORMAT.md §5.1) */ + private function fxPath(string $db, string $field, ?int $shard = null): string + { + return $this->dbPath($db, 'gen') . 'indexes/' . ($shard === null ? 'main' : 's' . $shard) . '.' . $this->token($field) . '.fx'; + } + + /** indexes/{token}.gfx of the active generation */ + private function gfxPath(string $db, string $field): string + { + return $this->dbPath($db, 'gen') . 'indexes/' . $this->token($field) . '.gfx'; + } + + private function dataPath(string $db, ?int $shard = null): string + { + return $this->dbPath($db, 'data', $shard); + } + + /** + * The token of a field in index file names (STORAGE_FORMAT.md §5.1): the field when [a-z0-9_]{1,40}, + * else a lower-case prefix (other characters as "_", at most 24 bytes), "-" and 12 hex digits of its + * sha256 + */ + private function token(string $field): string + { + if (preg_match('/^[a-z0-9_]{1,40}$/', $field)) { + return $field; + } + return substr(strtolower(preg_replace('/[^A-Za-z0-9_]/', '_', $field)), 0, 24) . '-' . substr(hash('sha256', $field), 0, 12); + } + + /** + * [header, body, content] of a secondary index file, decoded by the test's own readers: an .fx (§6.5.1): body = + * bucket id => key when there is one, => [keys] (ascending) when there are several; a .gfx (§6.5.2): body = + * shard => its filter + */ + private function secondary(string $path): array + { + clearstatcache(true, $path); + $content = (string)file_get_contents($path); + if (substr($path, -4) === '.gfx') { + [$header, $filters] = $this->readGlobalFieldFile($path); + return [$header, $filters, $content]; + } + if (substr($path, -3) === '.fx') { + [$header, $map] = $this->readFieldIndexFile($path); + return [$header, $map, $content]; + } + $newline = strpos($content, "\n"); + return [json_decode(substr($content, 0, $newline), true), json_decode(substr($content, $newline + 1), true), $content]; + } + + /** Write a field index file with the test's own writers (.fx: the paged format; .gfx: shard filters) */ + private function writeSecondary(string $path, array $header, array $body): void + { + if (substr($path, -4) === '.gfx') { + $this->writeGlobalFieldFile($path, $header, $body); + return; + } + unset($header['pg']); + $this->writeFieldIndexFile($path, $header, $body); + } + + private function size(string $path): int + { + clearstatcache(true, $path); + return (int)filesize($path); + } + + private function setStatic(string $name, $value): void + { + $property = new \noneDB\Tests\InternalValue($name); + $property->setValue(null, $value); + } + + private function getStatic(string $name) + { + $property = new \noneDB\Tests\InternalValue($name); + return $property->getValue(); + } + + private function callPrivate(\noneDB $db, string $method, array $args = []) + { + $m = new \noneDB\Tests\InternalMethod($method); + return $m->invokeArgs($db, $args); + } + + /** Files of the test directory (paths relative to it, every level) whose path contains $part */ + private function filesWith(string $part): array + { + clearstatcache(); + $root = rtrim($this->testDbDir, '/') . '/'; + $files = []; + $walk = function (string $dir) use (&$walk, &$files, $root): void { + foreach (scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($dir . $entry)) { + $walk($dir . $entry . '/'); + } else { + $files[] = substr($dir . $entry, strlen($root)); + } + } + }; + $walk($root); + sort($files); + return array_values(array_filter($files, function ($f) use ($part) { + return strpos($f, $part) !== false; + })); + } + + protected function tearDown(): void + { + $this->setStatic('renameFn', null); + parent::tearDown(); + } + + // ================================================================== equality + + /** + * find(), getKeysFromFieldIndex(), where() (plain / $in), whereIn(), orWhere(), count(), exists(), + * first() on an indexed field give exactly the model's keys, in key order, for every type: int, float, + * -0.0, numeric strings, bools, null, a missing field and arrays; through inserts, updates (also by a + * process with fieldIndexEnabled off), removed fields, deletes, compaction, a cold cache and a + * the switch to shards; unsharded, sharded, buffered. + */ + public function testEqualityLookupsEqualAnIndependentModelThroughTheWholeLifecycle(): void + { + $configs = [ + 'unsharded' => ['shardingEnabled' => false], + 'sharded' => ['shardSize' => 7], + 'converted' => ['shardSize' => 16, 'shardingEnabled' => false], + 'buffered' => ['shardingEnabled' => false, 'bufferEnabled' => true], + ]; + $pool = self::eqPool(); + foreach ($configs as $label => $config) { + $this->cleanTestDirectory(); + $name = 'life'; + $seed = 7; + $rows = []; + $next = 0; + $db = $this->db($config); + $row = function () use (&$seed, $pool) { + $record = ['n' => $seed]; + foreach (['v', 'w'] as $field) { + $i = self::rnd($seed, count($pool) + 1); + if ($i < count($pool)) { + $record[$field] = $pool[$i]; + } + } + return $record; + }; + $insert = function (\noneDB $on, int $count) use (&$rows, &$next, $row, $name) { + $batch = []; + for ($i = 0; $i < $count; $i++) { + $batch[] = $rows[$next++] = $row(); + } + $this->assertSame($count, $on->insert($name, $batch)['n']); + }; + + $insert($db, 40); + $this->assertTrue($db->createFieldIndex($name, 'v')['success']); + $this->assertTrue($db->createFieldIndex($name, 'w')['success']); + $insert($db, 20); + $this->checkEquality($db, $name, $rows, "$label: inserts"); + + // Updates, some through a process that doesn't use field indexes (indexes follow the data anyway) + $off = $this->db($config + ['fieldIndexEnabled' => false]); + for ($i = 0; $i < 16; $i++) { + $keys = array_keys($rows); + $key = $keys[self::rnd($seed, count($keys))]; + $writer = $i % 2 ? $off : $db; + $choice = self::rnd($seed, count($pool) + 1); + if ($choice === count($pool)) { + $result = $writer->query($name)->where(['key' => $key])->removeFields(['v']); + $this->assertSame(array_key_exists('v', $rows[$key]) ? 1 : 0, $result['n']); + unset($rows[$key]['v']); + } else { + $this->assertSame(1, $writer->update($name, [['key' => $key], ['set' => ['v' => $pool[$choice]]]])['n']); + $rows[$key]['v'] = $pool[$choice]; + } + } + $this->checkEquality($db, $name, $rows, "$label: updates, warm"); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->checkEquality($db, $name, $rows, "$label: updates, cold"); + + for ($i = 0; $i < 10; $i++) { + $keys = array_keys($rows); + $key = $keys[self::rnd($seed, count($keys))]; + $this->assertSame(1, $db->delete($name, ['key' => $key])['n']); + unset($rows[$key]); + } + $this->checkEquality($db, $name, $rows, "$label: deletes"); + + $this->assertTrue($db->compact($name)['success']); + $this->checkEquality($db, $name, $rows, "$label: compacted, warm"); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->checkEquality($db, $name, $rows, "$label: compacted, cold"); + + if ($label === 'converted') { + $this->assertFalse($db->getShardInfo($name)['sharded'] ?? false, 'shardingEnabled off: still one file'); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded($name)); + $this->assertTrue($db->getShardInfo($name)['sharded']); + $this->assertSame(['v', 'w'], $db->getFieldIndexes($name)['fields']); + $this->checkEquality($db, $name, $rows, "$label: after the switch"); + $insert($db, 10); + \noneDB::clearStaticCache(); + $this->checkEquality($this->db($config), $name, $rows, "$label: switched, more inserts, cold"); + } + $this->assertTrue($db->checkIntegrity($name)['ok'], $label); + } + } + + private function checkEquality(\noneDB $db, string $name, array $rows, string $label): void + { + if (is_file($this->gfxPath($name, 'v'))) { + $this->assertGlobalEntriesFit($name, 'v', "$label (before lookups)"); + } + $values = array_merge(self::eqPool(), ['zzz', 3, 1.5]); + $count = count($values); + foreach ($values as $i => $x) { + $y = $values[($i * 7 + 3) % $count]; + $msg = "$label, v = " . self::show($x) . ', other = ' . self::show($y); + $expected = self::select($rows, self::eq('v', $x)); + $this->assertSame($expected, $this->keys($db->find($name, ['v' => $x])), "find: $msg"); + $this->assertSame(is_array($x) ? null : $expected, $db->getKeysFromFieldIndex($name, 'v', $x), "getKeysFromFieldIndex: $msg"); + $this->assertSame($expected, $this->keys($db->query($name)->where(['v' => $x])->get()), "where: $msg"); + $this->assertSame(count($expected), $db->query($name)->where(['v' => $x])->count(), "query count: $msg"); + $this->assertSame(count($expected), $db->count($name, ['v' => $x]), "count: $msg"); + $this->assertSame($expected !== [], $db->query($name)->where(['v' => $x])->exists(), "exists: $msg"); + $this->assertSame($expected !== [], $db->exists($name, ['v' => $x]), "noneDB exists: $msg"); + $this->assertSame($expected[0] ?? null, $db->query($name)->where(['v' => $x])->first()['key'] ?? null, "first: $msg"); + + $in = self::select($rows, function ($r) use ($x, $y) { return array_key_exists('v', $r) && in_array($r['v'], [$x, $y], true); }); + $this->assertSame($in, $this->keys($db->query($name)->where(['v' => ['$in' => [$x, $y]]])->get()), "\$in: $msg"); + $this->assertSame($in, $this->keys($db->query($name)->whereIn('v', [$x, $y])->get()), "whereIn: $msg"); + $or = self::select($rows, function ($r) use ($x, $y) { + return (array_key_exists('v', $r) && $r['v'] === $x) || (array_key_exists('w', $r) && $r['w'] === $y); + }); + $this->assertSame($or, $this->keys($db->query($name)->where(['v' => $x])->orWhere(['w' => $y])->get()), "orWhere: $msg"); + $both = self::select($rows, function ($r) use ($x, $y) { + return array_key_exists('v', $r) && $r['v'] === $x && array_key_exists('w', $r) && $r['w'] === $y; + }); + $this->assertSame($both, $this->keys($db->find($name, ['v' => $x, 'w' => $y])), "find v and w: $msg"); + + if ($i === $count - 1 && is_file($this->gfxPath($name, 'v'))) { + $this->assertGlobalEntriesFit($name, 'v', "$label (after lookups)"); + } + if (!is_array($x)) { + // An equality on an indexed field reads index candidates: no segment is scanned + $plan = $db->query($name)->where(['v' => $x])->explain(); + $this->assertSame(0, $plan['segments']['scanned'], "explain: $msg " . json_encode($plan['plan'])); + $this->assertSame($plan['segments']['total'], $plan['segments']['indexed'] + $plan['segments']['skipped'], "explain: $msg"); + $this->assertGreaterThanOrEqual(count($expected), $plan['estimatedRead'], "explain: never fewer candidates than matches: $msg"); + } + } + } + + // ================================================================== ranges + + /** + * $gt / $gte / $lt / $lte, two-sided ranges, query()->between() and noneDB::between() on an indexed + * field give exactly the documented comparison over every value type (numbers, numeric strings with + * exponent / leading space / sign, other strings by byte order, null, bools, arrays, missing), for + * operands of every type including NAN and INF; and they use the index (no segment scanned). + */ + public function testRangeLookupsEqualTheDocumentedComparison(): void + { + $pool = [0, 1, -1, 2.5, -2.5, 10, 1.0, -0.0, '5', '10', '9', '2.5', '-1', ' 7', '1e1', '.5', 'abc', 'Abc', 'Z', '', 'a5', '5a', + null, true, false, [3]]; + $operands = [0, 1, 5, -1, 2.5, 9.5, 10, -0.0, '5', '10', '9', ' 7', '1e1', 'abc', 'Z', '', 'a', 'B', null, true, false, NAN, INF, -INF]; + $pairs = [[0, 10], [-1, '9'], ['1', 'abc'], ['', 'Z'], ['a', 'b'], [2.5, 2.5], [10, 0], [-INF, INF], ['abc', 'abc'], [null, 5]]; + foreach (['unsharded' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 6]] as $label => $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $seed = 99; + $rows = []; + $batch = []; + for ($k = 0; $k < 90; $k++) { + $i = self::rnd($seed, count($pool) + 1); + $rows[$k] = $i < count($pool) ? ['v' => $pool[$i]] : ['other' => $k]; + $batch[] = $rows[$k]; + if ($k === 69) { + $db->insert('rg', $batch); + $batch = []; + $this->assertTrue($db->createFieldIndex('rg', 'v')['success']); + } + } + $db->insert('rg', $batch); // a tail after the checkpoint + $phases = ['inserts' => null, 'updates' => function () use (&$rows, &$seed, $pool, $db) { + for ($i = 0; $i < 20; $i++) { + $key = self::rnd($seed, 90); + $value = $pool[self::rnd($seed, count($pool))]; + $this->assertSame(1, $db->update('rg', [['key' => $key], ['set' => ['v' => $value]]])['n']); + $rows[$key]['v'] = $value; + } + }, 'cold' => function () { \noneDB::clearStaticCache(); }]; + foreach ($phases as $phase => $change) { + if ($change !== null) { + $change(); + } + $db = $this->db($config); + foreach ($operands as $operand) { + foreach (['$gt', '$gte', '$lt', '$lte'] as $op) { + $msg = "$label $phase: v $op " . self::show($operand); + $expected = self::select($rows, function ($r) use ($op, $operand) { return self::holds($r, 'v', $op, $operand); }); + $query = $db->query('rg')->where(['v' => [$op => $operand]]); + $this->assertSame($expected, $this->keys($query->get()), $msg); + $this->assertSame(count($expected), $db->query('rg')->where(['v' => [$op => $operand]])->count(), $msg); + if (is_int($operand) || is_float($operand) || is_string($operand)) { + $this->assertSame(0, $db->query('rg')->where(['v' => [$op => $operand]])->explain()['segments']['scanned'], "range index used: $msg"); + } + } + } + foreach ($pairs as [$lo, $hi]) { + $msg = "$label $phase: " . self::show($lo) . ' .. ' . self::show($hi); + $inclusive = self::select($rows, function ($r) use ($lo, $hi) { return self::holds($r, 'v', '$gte', $lo) && self::holds($r, 'v', '$lte', $hi); }); + $halfOpen = self::select($rows, function ($r) use ($lo, $hi) { return self::holds($r, 'v', '$gt', $lo) && self::holds($r, 'v', '$lt', $hi); }); + $this->assertSame($inclusive, $this->keys($db->query('rg')->between('v', $lo, $hi)->get()), "between: $msg"); + $this->assertSame($inclusive, $this->keys($db->between('rg', 'v', $lo, $hi)), "noneDB::between: $msg"); + $this->assertSame($halfOpen, $this->keys($db->query('rg')->where(['v' => ['$gt' => $lo, '$lt' => $hi]])->get()), "\$gt/\$lt: $msg"); + } + } + } + } + + // ================================================================== planner + + /** + * Random combinations of where() groups (plain values, $eq, $in, ranges, $ne, key filters), orWhere() + * (also an empty group), whereIn(), between(), sort + offset + limit: get(), count(), exists(), + * first() and explain(true) agree with the model on sharded and unsharded databases with two indexed + * fields, before and after updates and deletes. The planner may only narrow down what is read. + */ + public function testRandomQueriesNeverDropAMatch(): void + { + $values = [0, 1, 2, 3, '1', 1.0, 2.5, null, true, 'x', 'y', 'b3']; + foreach (['unsharded' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 30]] as $label => $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->createFieldIndex('rq', 'a'); + $db->createFieldIndex('rq', 'b'); + $seed = 2026; + $rows = []; + for ($k = 0; $k < 200; $k++) { + $rows[$k] = ['b' => $values[self::rnd($seed, count($values))], 'n' => $k]; + if (self::rnd($seed, 10) > 0) { + $rows[$k]['a'] = $values[self::rnd($seed, count($values))]; + } + } + $db->insert('rq', array_values($rows)); + foreach (['inserted', 'changed'] as $phase) { + if ($phase === 'changed') { + for ($i = 0; $i < 30; $i++) { + $key = array_keys($rows)[self::rnd($seed, count($rows))]; + if ($i % 3 === 0) { + $db->delete('rq', ['key' => $key]); + unset($rows[$key]); + } else { + $value = $values[self::rnd($seed, count($values))]; + $db->update('rq', [['key' => $key], ['set' => ['a' => $value]]]); + $rows[$key]['a'] = $value; + } + } + } + for ($round = 0; $round < 150; $round++) { + $spec = $this->randomSpec($seed, $values); + $msg = "$label $phase round $round: " . json_encode($spec); + $expected = $this->modelQuery($rows, $spec); + $this->assertSame($expected, $this->keys($this->buildQuery($db, $spec)->get()), $msg); + $this->assertSame(count($expected), $this->buildQuery($db, $spec)->count(), "count: $msg"); + $this->assertSame($expected !== [], $this->buildQuery($db, $spec)->exists(), "exists: $msg"); + $this->assertSame($expected[0] ?? null, $this->buildQuery($db, $spec)->first()['key'] ?? null, "first: $msg"); + $explain = $this->buildQuery($db, $spec)->explain(true); + $this->assertSame(count($expected), $explain['actual']['returned'], "explain: $msg"); + if ($explain['actual']['recordsRead'] !== null) { + $this->assertLessThanOrEqual($explain['estimatedRead'], $explain['actual']['recordsRead'], "never more than the plan's candidates: $msg"); + if (!isset($spec['limit']) || isset($spec['sort'])) { + $this->assertSame($explain['estimatedRead'], $explain['actual']['recordsRead'], "a query that doesn't stop early reads every candidate: $msg"); + } + } + $this->assertGreaterThanOrEqual(count($expected), $explain['estimatedRead'], "candidates are a superset: $msg"); + } + } + } + } + + /** A random query: ['where' => [group], 'or' => [group], 'in' => [field, values], 'between' => [lo, hi], 'sort', 'offset', 'limit'] */ + private function randomSpec(int &$seed, array $values): array + { + $leaf = function () use (&$seed, $values) { + $field = ['a', 'b', 'n', 'key'][self::rnd($seed, 4)]; + $v = $values[self::rnd($seed, count($values))]; + $w = $values[self::rnd($seed, count($values))]; + if ($field === 'key') { + $k = self::rnd($seed, 230); + switch (self::rnd($seed, 4)) { + case 0: return [$field, 'plain', self::rnd($seed, 2) ? $k : (string)$k]; + case 1: return [$field, '$in', [$k, self::rnd($seed, 230), (string)self::rnd($seed, 230)]]; + case 2: return [$field, '$gte', $k]; + default: return [$field, '$ne', $k]; + } + } + switch (self::rnd($seed, 7)) { + case 0: return [$field, 'plain', $v]; + case 1: return [$field, '$eq', $v]; + case 2: return [$field, '$in', [$v, $w]]; + case 3: return [$field, '$gte', $v]; + case 4: return [$field, 'range', ['$gt' => $v, '$lt' => $w]]; + case 5: return [$field, '$lte', $v]; + default: return [$field, '$ne', $v]; + } + }; + $group = function () use ($leaf, &$seed) { + $group = []; + for ($i = self::rnd($seed, 2) + 1; $i > 0; $i--) { + $l = $leaf(); + $group[$l[0]] = $l; // one condition per field (as a PHP filter array has) + } + return array_values($group); + }; + $spec = ['where' => [], 'or' => []]; + for ($i = self::rnd($seed, 3); $i > 0; $i--) { + $spec['where'][] = $group(); + } + for ($i = self::rnd($seed, 3) === 0 ? self::rnd($seed, 2) + 1 : 0; $i > 0; $i--) { + $spec['or'][] = self::rnd($seed, 8) === 0 ? [] : $group(); + } + if (self::rnd($seed, 4) === 0) { + $spec['in'] = [self::rnd($seed, 2) ? 'a' : 'b', [1, 'x', 2.5, $values[self::rnd($seed, count($values))]]]; + } + if (self::rnd($seed, 4) === 0) { + $spec['between'] = [0, self::rnd($seed, 4)]; + } + if (self::rnd($seed, 3) === 0) { + $spec['sort'] = self::rnd($seed, 2) ? 'asc' : 'desc'; + } + if (self::rnd($seed, 3) === 0) { + $spec['limit'] = self::rnd($seed, 20) + 1; + if (self::rnd($seed, 2) === 0) { + $spec['offset'] = self::rnd($seed, 5); + } + } + return $spec; + } + + private function buildQuery(\noneDB $db, array $spec): \noneDBQuery + { + $filters = function (array $group) { + $filter = []; + foreach ($group as [$field, $op, $value]) { + $filter[$field] = $op === 'plain' ? $value : ($op === 'range' ? $value : [$op => $value]); + } + return $filter; + }; + $q = $db->query('rq'); + foreach ($spec['where'] as $group) { + $q->where($filters($group)); + } + foreach ($spec['or'] as $group) { + $q->orWhere($filters($group)); + } + if (isset($spec['in'])) { + $q->whereIn($spec['in'][0], $spec['in'][1]); + } + if (isset($spec['between'])) { + $q->between('a', $spec['between'][0], $spec['between'][1]); + } + if (isset($spec['sort'])) { + $q->sort('n', $spec['sort']); + } + if (isset($spec['offset'])) { + $q->offset($spec['offset']); + } + if (isset($spec['limit'])) { + $q->limit($spec['limit']); + } + return $q; + } + + /** The model of a random query: keys in result order */ + private function modelQuery(array $rows, array $spec): array + { + $leaf = function (array $r, array $leaf) { + [$field, $op, $value] = $leaf; + $has = array_key_exists($field, $r); + switch ($op) { + case 'plain': + if ($field === 'key') { + // find() semantics (API.md): a key filter is an int or a canonical digit string + return (is_int($value) || preg_match('/^(0|[1-9][0-9]*)$/', $value)) && $r['key'] === (int)$value; + } + return $has && $r[$field] === $value; + case '$eq': return $has && $r[$field] === $value; + case '$in': return $has && in_array($r[$field], $value, true); + case '$ne': return !$has || $r[$field] !== $value; + case 'range': + foreach ($value as $o => $operand) { + if (!self::holds($r, $field, $o, $operand)) { + return false; + } + } + return true; + default: return self::holds($r, $field, $op, $value); + } + }; + $group = function (array $r, array $group) use ($leaf) { + foreach ($group as $l) { + if (!$leaf($r, $l)) { + return false; + } + } + return true; + }; + $keys = self::select($rows, function ($r) use ($spec, $group) { + if (!empty($spec['where']) || !empty($spec['or'])) { + $match = !empty($spec['where']); + foreach ($spec['where'] as $g) { + $match = $match && $group($r, $g); + } + if (!$match) { + foreach ($spec['or'] as $g) { + $match = $match || $group($r, $g); + } + } + if (!$match) { + return false; + } + } + if (isset($spec['in']) && !(array_key_exists($spec['in'][0], $r) && in_array($r[$spec['in'][0]], $spec['in'][1], true))) { + return false; + } + if (isset($spec['between']) && !(self::holds($r, 'a', '$gte', $spec['between'][0]) && self::holds($r, 'a', '$lte', $spec['between'][1]))) { + return false; + } + return true; + }); + // "n" equals the key here: sorting by n is sorting by key + if (($spec['sort'] ?? 'asc') === 'desc') { + $keys = array_reverse($keys); + } + return array_slice($keys, $spec['offset'] ?? 0, $spec['limit'] ?? null); + } + + /** + * explain() of an unsharded database, every field derived by hand: 100 records, city cycles through + * Ankara, Bursa, Izmir, Konya (25 each), age = 20 + n % 10, only city is indexed + */ + public function testExplainOfAnUnshardedDatabase(): void + { + $db = $this->db(['shardingEnabled' => false]); + $cities = ['Ankara', 'Bursa', 'Izmir', 'Konya']; + $db->insert('ex', array_map(function ($n) use ($cities) { return ['city' => $cities[$n % 4], 'age' => 20 + $n % 10, 'n' => $n]; }, range(0, 99))); + $db->createFieldIndex('ex', 'city'); + $q = function () use ($db) { return $db->query('ex'); }; + + $explain = $q()->where(['city' => 'Bursa'])->explain(true); + $ms = $explain['actual']['ms']; + $this->assertIsFloat($ms); + $this->assertGreaterThanOrEqual(0.0, $ms); + unset($explain['actual']['ms']); + $this->assertSame([ + 'explain' => 1, 'db' => 'ex', 'sharded' => false, 'access' => 'index', + 'segments' => ['total' => 1, 'skipped' => 0, 'indexed' => 1, 'scanned' => 0], + 'plan' => [['segment' => null, 'access' => 'index', 'candidates' => 25]], 'planTruncated' => false, + 'estimatedRead' => 25, 'filter' => ['eq', 'city', 'Bursa'], 'residual' => true, 'spatial' => null, + 'sort' => 'none', 'earlyExit' => false, 'joins' => [], 'groupBy' => null, + 'actual' => ['recordsRead' => 25, 'segmentsRead' => 1, 'returned' => 25], + ], $explain); + + // An unindexed field next to an indexed one: the index's candidates, every filter checked on them + $explain = $q()->where(['city' => 'Bursa', 'age' => 21])->explain(true); + $this->assertSame(['and', [['eq', 'city', 'Bursa'], ['eq', 'age', 21]]], $explain['filter']); + $this->assertSame([25, 'index'], [$explain['estimatedRead'], $explain['access']]); + $this->assertSame(['recordsRead' => 25, 'segmentsRead' => 1, 'returned' => 5], array_diff_key($explain['actual'], ['ms' => 0]), 'n = 1, 21, 41, 61, 81'); + + // Unindexed only: the segment is scanned + $explain = $q()->where(['age' => 21])->explain(true); + $this->assertSame(['scan', 100, [['segment' => null, 'access' => 'scan', 'candidates' => 100]]], [$explain['access'], $explain['estimatedRead'], $explain['plan']]); + $this->assertSame(['total' => 1, 'skipped' => 0, 'indexed' => 0, 'scanned' => 1], $explain['segments']); + $this->assertSame([100, 10], [$explain['actual']['recordsRead'], $explain['actual']['returned']]); + + // limit() without sort stops early: batches of 32, then 128 (QUERY.md) + $explain = $q()->where(['age' => 21])->limit(1)->explain(true); + $this->assertTrue($explain['earlyExit']); + $this->assertSame([32, 1], [$explain['actual']['recordsRead'], $explain['actual']['returned']], 'n = 1 is in the first batch'); + $explain = $q()->where(['age' => 29])->limit(4)->explain(true); + $this->assertSame([100, 4], [$explain['actual']['recordsRead'], $explain['actual']['returned']], 'the 4th match (n = 39) is in the second batch (32 + 68)'); + $explain = $q()->where(['age' => 29])->limit(3)->explain(true); + $this->assertSame([32, 3], [$explain['actual']['recordsRead'], $explain['actual']['returned']], 'n = 9, 19, 29'); + + // orWhere uses the index when every branch can, otherwise the segment is scanned + $explain = $q()->where(['city' => 'Bursa'])->orWhere(['city' => 'Izmir'])->explain(true); + $this->assertSame(['or', [['eq', 'city', 'Bursa'], ['eq', 'city', 'Izmir']]], $explain['filter']); + $this->assertSame(['index', 50, 50, 50], [$explain['access'], $explain['estimatedRead'], $explain['actual']['recordsRead'], $explain['actual']['returned']]); + $explain = $q()->where(['city' => 'Bursa'])->orWhere(['age' => 21])->explain(true); + $this->assertSame(['scan', 100, 30], [$explain['access'], $explain['estimatedRead'], $explain['actual']['returned']], '25 Bursa + 5 more with age 21'); + + // $in, whereIn(), ranges and between() on the indexed field + $explain = $q()->where(['city' => ['$in' => ['Bursa', 'Konya', 'nowhere']]])->explain(true); + $this->assertSame([['in', 'city', ['Bursa', 'Konya', 'nowhere']], 'index', 50, 50], [$explain['filter'], $explain['access'], $explain['estimatedRead'], $explain['actual']['returned']]); + $explain = $q()->whereIn('city', ['Ankara'])->explain(); + $this->assertSame([['in', 'city', ['Ankara']], 'index', 25], [$explain['filter'], $explain['access'], $explain['estimatedRead']]); + $explain = $q()->where(['city' => ['$gte' => 'Bursa', '$lte' => 'Izmir']])->explain(true); + $this->assertSame([['and', [['range', 'city', '$gte', 'Bursa'], ['range', 'city', '$lte', 'Izmir']]], 'index', 50, 50], + [$explain['filter'], $explain['access'], $explain['estimatedRead'], $explain['actual']['returned']], 'Bursa and Izmir'); + // (strict bounds: testStrictRangeBoundsReadOnlyTheBucketsInRange) + $explain = $q()->between('city', 'B', 'J')->explain(true); + $this->assertSame([['and', [['range', 'city', '$gte', 'B'], ['range', 'city', '$lte', 'J']]], 'index', 50, 50], + [$explain['filter'], $explain['access'], $explain['estimatedRead'], $explain['actual']['returned']]); + $this->assertSame('scan', $q()->where(['age' => ['$gt' => 25]])->explain()['access'], 'a range on an unindexed field'); + + // Negations read every record + $explain = $q()->where(['city' => ['$ne' => 'Bursa']])->explain(true); + $this->assertSame([['any'], 'scan', 100, 75], [$explain['filter'], $explain['access'], $explain['estimatedRead'], $explain['actual']['returned']]); + $this->assertSame('scan', $q()->whereNot(['city' => 'Bursa'])->explain()['access']); + $this->assertSame('scan', $q()->whereNotIn('city', ['Bursa'])->explain()['access']); + + // Key filters read only those keys + $explain = $q()->where(['key' => 5])->explain(true); + $this->assertSame([['keys-in-filter-order'], 'index', 1], [$explain['filter'], $explain['access'], $explain['estimatedRead']]); + $this->assertSame(['recordsRead' => null, 'segmentsRead' => null, 'returned' => 1], array_diff_key($explain['actual'], ['ms' => 0]), 'find() reads it'); + $explain = $q()->where(['key' => ['$in' => [1, 2, 300]]])->explain(true); + $this->assertSame([['keys', [1, 2, 300]], 'index', 2, 2, 2], [$explain['filter'], $explain['access'], $explain['estimatedRead'], $explain['actual']['recordsRead'], $explain['actual']['returned']]); + + // After deletes the candidates follow the data + foreach (range(1, 37, 4) as $key) { + $db->delete('ex', ['key' => $key]); // the first ten Bursa records + } + $this->assertSame(15, $q()->where(['city' => 'Bursa'])->explain()['estimatedRead']); + $this->assertSame(90, $q()->where(['age' => 1])->explain()['estimatedRead']); + $this->assertSame([], $this->filesWith('.age.fx'), 'explain() builds nothing'); + } + + /** + * CHANGES.md: "$gt, $gte, $lt, $lte and between() read only the buckets in range". A strict + * bound is not in range: `$lt "Konya"` must not read the 25 records of "Konya", `$gt 25` not the ten + * records of age 25 (same data as testExplainOfAnUnshardedDatabase, age indexed too) + */ + public function testStrictRangeBoundsReadOnlyTheBucketsInRange(): void + { + $db = $this->db(['shardingEnabled' => false]); + $cities = ['Ankara', 'Bursa', 'Izmir', 'Konya']; + $db->insert('sr', array_map(function ($n) use ($cities) { return ['city' => $cities[$n % 4], 'age' => 20 + $n % 10]; }, range(0, 99))); + $db->createFieldIndex('sr', 'city'); + $db->createFieldIndex('sr', 'age'); + $cases = [ + [['city' => ['$gt' => 'Bursa']], 50, 'Izmir, Konya'], + [['city' => ['$lt' => 'Konya']], 75, 'Ankara, Bursa, Izmir'], + [['city' => ['$gte' => 'Bursa', '$lt' => 'Konya']], 50, 'Bursa, Izmir'], + [['city' => ['$gt' => 'Ankara', '$lt' => 'Konya']], 50, 'Bursa, Izmir'], + [['age' => ['$gt' => 25]], 40, 'ages 26..29, ten records each'], + [['age' => ['$lt' => 25]], 50, 'ages 20..24'], + [['age' => ['$gt' => 21, '$lt' => 24]], 20, 'ages 22, 23'], + ]; + $report = []; + $exact = []; + foreach ($cases as [$filter, $inRange, $what]) { + $explain = $db->query('sr')->where($filter)->explain(true); + $this->assertSame($inRange, $explain['actual']['returned'], $what); + $report[] = json_encode($filter) . ": estimatedRead {$explain['estimatedRead']}, recordsRead {$explain['actual']['recordsRead']}, in range $inRange ($what)"; + $exact[] = [$explain['estimatedRead'], $explain['actual']['recordsRead']] === [$inRange, $inRange]; + } + $this->assertSame(array_fill(0, count($cases), true), $exact, "candidates beyond the range:\n" . implode("\n", $report)); + } + + /** + * explain() of a sharded database (4 shards of 50, g = "g" + shard number, u unique; both indexed): + * the shards the global field index rules out are skipped; an index that a query would build first + * counts as a scan ("mixed" next to indexed shards) and explain() builds nothing; more than 50 + * segments truncate the plan list + */ + public function testExplainOfAShardedDatabase(): void + { + $db = $this->db(['shardSize' => 50]); + $db->insert('es', array_map(function ($n) { return ['g' => 'g' . intdiv($n, 50), 'u' => 'u' . $n, 'v' => $n % 3]; }, range(0, 199))); + $db->createFieldIndex('es', 'g'); + $db->createFieldIndex('es', 'u'); + $q = function () use ($db) { return $db->query('es'); }; + $skip = function ($segment) { return ['segment' => $segment, 'access' => 'skip', 'candidates' => 0]; }; + $index = function ($segment, $n) { return ['segment' => $segment, 'access' => 'index', 'candidates' => $n]; }; + + $explain = $q()->where(['g' => 'g2'])->explain(true); + $this->assertSame([true, 'index', ['total' => 4, 'skipped' => 3, 'indexed' => 1, 'scanned' => 0], 50], + [$explain['sharded'], $explain['access'], $explain['segments'], $explain['estimatedRead']]); + $this->assertSame([$skip(0), $skip(1), $index(2, 50), $skip(3)], $explain['plan']); + $this->assertSame(['recordsRead' => 50, 'segmentsRead' => 1, 'returned' => 50], array_diff_key($explain['actual'], ['ms' => 0])); + + $explain = $q()->whereIn('g', ['g0', 'g3'])->explain(); + $this->assertSame([$index(0, 50), $skip(1), $skip(2), $index(3, 50)], $explain['plan']); + $this->assertSame(100, $explain['estimatedRead']); + + $explain = $q()->where(['g' => 'g9'])->explain(true); + $this->assertSame(['none', ['total' => 4, 'skipped' => 4, 'indexed' => 0, 'scanned' => 0], 0], [$explain['access'], $explain['segments'], $explain['estimatedRead']]); + $this->assertSame(['recordsRead' => 0, 'segmentsRead' => 0, 'returned' => 0], array_diff_key($explain['actual'], ['ms' => 0])); + + $explain = $q()->where(['g' => 'g1'])->orWhere(['u' => 'u170'])->explain(true); + $this->assertSame([$skip(0), $index(1, 50), $skip(2), $index(3, 1)], $explain['plan']); + $this->assertSame([51, 51], [$explain['estimatedRead'], $explain['actual']['returned']]); + $this->assertSame('none', $q()->where(['g' => 'g1', 'u' => 'u170'])->explain()['access'], 'no shard holds both'); + $this->assertSame([], $q()->where(['g' => 'g1', 'u' => 'u170'])->get()); + + $explain = $q()->where(['v' => 1])->explain(); + $this->assertSame(['scan', ['total' => 4, 'skipped' => 0, 'indexed' => 0, 'scanned' => 4], 200], [$explain['access'], $explain['segments'], $explain['estimatedRead']]); + + // Key 200 opens shard 4, which has no field index yet and no global entry: never skipped, + // and explain() counts its index as a scan (one live record) without building it + $db->insert('es', ['g' => 'g2', 'u' => 'new', 'v' => 0]); + $explain = $q()->where(['g' => 'g2'])->explain(); + $this->assertSame(['mixed', ['total' => 5, 'skipped' => 3, 'indexed' => 1, 'scanned' => 1], 51], [$explain['access'], $explain['segments'], $explain['estimatedRead']]); + $this->assertSame(['segment' => 4, 'access' => 'scan', 'candidates' => 1], $explain['plan'][4]); + $this->assertSame('scan', $q()->where(['g' => 'g9'])->explain()['access'], 'shard 4 must be read'); + $this->assertFileDoesNotExist($this->fxPath('es', 'g', 4)); + // The query builds it (and shard 4's global entry): then shard 4 is indexed, and skipped for g9 + $this->assertSame(array_merge(range(100, 149), [200]), $this->keys($q()->where(['g' => 'g2'])->get())); + $this->assertFileExists($this->fxPath('es', 'g', 4)); + $explain = $q()->where(['g' => 'g2'])->explain(); + $this->assertSame(['index', ['total' => 5, 'skipped' => 3, 'indexed' => 2, 'scanned' => 0]], [$explain['access'], $explain['segments']]); + $this->assertSame(['none', 5], [$q()->where(['g' => 'g9'])->explain()['access'], $q()->where(['g' => 'g9'])->explain()['segments']['skipped']]); + + // More than 50 segments: the plan lists the first 50 + $many = $this->db(['shardSize' => 1]); + $many->insert('many', array_map(function ($n) { return ['x' => $n]; }, range(0, 54))); + $explain = $many->query('many')->where(['x' => 3])->explain(true); + $this->assertSame([true, 50, ['total' => 55, 'skipped' => 0, 'indexed' => 0, 'scanned' => 55], 55], + [$explain['planTruncated'], count($explain['plan']), $explain['segments'], $explain['estimatedRead']]); + $this->assertSame(['segment' => 49, 'access' => 'scan', 'candidates' => 1], $explain['plan'][49]); + $this->assertSame(['recordsRead' => 55, 'segmentsRead' => 55, 'returned' => 1], array_diff_key($explain['actual'], ['ms' => 0])); + } + + /** explain(): "sort" (none / top-k up to offset + limit = 1000 / full), "earlyExit", joins, spatial, groupBy */ + public function testExplainOfSortsEarlyExitJoinsAndSpatialQueries(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createSpatialIndex('se', 'loc'); + $db->insert('se', array_map(function ($n) { + return ['city' => 'c' . ($n % 3), 'age' => $n, 'owner' => $n % 4, 'loc' => ['type' => 'Point', 'coordinates' => [$n % 10, 1]]]; + }, range(0, 29))); + $db->createFieldIndex('se', 'city'); + $db->insert('owners', [['name' => 'c0'], ['name' => 'c1']]); + $q = function () use ($db) { return $db->query('se')->where(['city' => 'c1']); }; + $pick = function (array $e) { return [$e['sort'], $e['earlyExit'], $e['groupBy']]; }; + + $this->assertSame(['none', true, null], $pick($q()->limit(10)->explain())); + $this->assertSame(['none', false, null], $pick($q()->explain()), 'no limit: nothing to stop at'); + $this->assertSame(['top-k', false, null], $pick($q()->sort('age')->limit(10)->explain())); + $this->assertSame(['top-k', false, null], $pick($q()->sort('age')->offset(990)->limit(10)->explain()), 'offset + limit = 1000'); + $this->assertSame(['full', false, null], $pick($q()->sort('age')->offset(991)->limit(10)->explain()), 'offset + limit = 1001'); + $this->assertSame(['full', false, null], $pick($q()->sort('age')->explain()), 'no limit'); + $this->assertSame(['full', false, 'city'], $pick($db->query('se')->groupBy('city')->sort('city')->limit(2)->explain())); + $this->assertSame(['full', false, null], $pick($q()->withDistance('loc', 0, 1)->sort('_distance')->limit(3)->explain()), 'withDistance()'); + + // Joins: in-lookup / key-lookup; after the slice unless they must run before the sort + $joined = $q()->join('owners', 'city', 'name', 'o')->limit(5)->explain(); + $this->assertSame([['db' => 'owners', 'on' => 'city = name', 'strategy' => 'in-lookup', 'when' => 'after-slice']], $joined['joins']); + $this->assertSame(['none', true], [$joined['sort'], $joined['earlyExit']]); + $this->assertSame('key-lookup', $q()->join('owners', 'owner', 'key', 'o')->explain()['joins'][0]['strategy']); + $bySort = $q()->join('owners', 'city', 'name', 'o')->sort('o')->limit(5)->explain(); + $this->assertSame(['before-sort', 'full', false], [$bySort['joins'][0]['when'], $bySort['sort'], $bySort['earlyExit']], 'sorted by the join alias'); + $this->assertSame('before-sort', $db->query('se')->join('owners', 'city', 'name', 'o')->groupBy('city')->explain()['joins'][0]['when']); + $late = $q()->join('owners', 'city', 'name', 'o')->sort('age', 'desc')->limit(3)->explain(); + $this->assertSame(['after-slice', 'top-k', false], [$late['joins'][0]['when'], $late['sort'], $late['earlyExit']]); + // and what the explained join returns: the three oldest c1 records with their owner + $rows = $q()->join('owners', 'city', 'name', 'o')->sort('age', 'desc')->limit(3)->get(); + $this->assertSame([28, 25, 22], array_column($rows, 'age')); + $this->assertSame([1, 1, 1], array_map(function ($r) { return $r['o']['key']; }, $rows)); + + // Spatial filters: access "spatial", no early exit + $spatial = $db->query('se')->withinBBox('loc', 0, 0, 3, 2)->limit(2)->explain(); + $this->assertSame(['spatial', ['withinBBox'], false], [$spatial['access'], $spatial['spatial'], $spatial['earlyExit']]); + $this->assertSame(['withinDistance', 'nearest'], $db->query('se')->withinDistance('loc', 0, 1, 1000)->nearest('loc', 0, 1, 2)->explain()['spatial']); + } + + /** explain() and the other reads of a database that doesn't exist create nothing */ + public function testExplainOfAMissingDatabaseCreatesNothing(): void + { + $db = $this->db(['shardingEnabled' => false]); + $explain = $db->query('nothing')->where(['a' => 1])->explain(true); + $this->assertSame(['sharded' => false, 'access' => 'none', 'segments' => ['total' => 0, 'skipped' => 0, 'indexed' => 0, 'scanned' => 0], + 'plan' => [], 'planTruncated' => false, 'estimatedRead' => 0], + array_intersect_key($explain, array_flip(['sharded', 'access', 'segments', 'plan', 'planTruncated', 'estimatedRead']))); + $this->assertSame(0, $explain['actual']['returned']); + $this->assertFalse($db->hasFieldIndex('nothing', 'a')); + $this->assertNull($db->getKeysFromFieldIndex('nothing', 'a', 1)); + $this->assertSame(['fields' => [], 'sharded' => false], $db->getFieldIndexes('nothing')); + $this->assertSame([], $this->dbFiles('nothing'), 'no file of the database'); + $this->assertDirectoryDoesNotExist($this->dbPath('nothing', 'home')); + } + + // ================================================================== global field index: shard skipping + + /** + * A shard is skipped only when its global entry fits its data (epoch, bytes covered, checksum of them), + * the value isn't in the entry and the shard's newer lines don't hold it (STORAGE_FORMAT.md §6.5 + * "Global indexes"). Each case below removes g2 from the global map AND breaks the fit of shard 2's entry + * (under a stale lock epoch, so entries are checked): shard 2 must still be read. + */ + public function testAShardIsSkippedOnlyWhenItsGlobalEntryFits(): void + { + $config = ['shardSize' => 10]; + $db = $this->db($config); + $db->insert('sk', array_map(function ($n) { return ['g' => 'g' . intdiv($n, 10), 'n' => $n]; }, range(0, 39))); + $db->createFieldIndex('sk', 'g'); + $gfx = $this->gfxPath('sk', 'g'); + [$header, $filters, $pristine] = $this->secondary($gfx); + $this->assertSame([64, 8], [$header['bits'], $header['width']]); + $this->assertSame([0 => $this->shardFilter(['g0'], 64), 1 => $this->shardFilter(['g1'], 64), 2 => $this->shardFilter(['g2'], 64), 3 => $this->shardFilter(['g3'], 64)], $filters); + $this->assertSame(['0', '1', '2', '3'], array_map('strval', array_keys($header['sh']))); + foreach ($header['sh'] as $shard => $entry) { + $this->assertSame($this->size($this->dataPath('sk', (int)$shard)), $entry['s'], "entry $shard covers its whole shard"); + } + $plan = function () use (&$db) { return array_column($db->query('sk')->where(['g' => 'g2'])->explain()['plan'], 'access', 'segment'); }; + $this->assertSame([0 => 'skip', 1 => 'skip', 2 => 'index', 3 => 'skip'], $plan(), 'a fitting global index rules shards out'); + + $cases = [ + 'checksum differs' => function (array &$h) { $h['sh']['2']['h'] = $h['sh']['2']['h'] === '00000000' ? '11111111' : '00000000'; }, + 'another epoch' => function (array &$h) { $h['sh']['2']['e'] = 'deadbeef'; }, + 'covers more than the data' => function (array &$h) { $h['sh']['2']['s'] += 100000; }, + 'no entry' => function (array &$h) { unset($h['sh']['2']); }, + 'key index file missing' => function (array &$h) { unlink($this->dbPath('sk', 'kix', 2)); }, + ]; + foreach ($cases as $case => $break) { + file_put_contents($gfx, $pristine); + [$h, $m] = $this->secondary($gfx); + $m[2] = $this->shardFilter([], 64); // shard 2's filter without g2 + $h['le'] = null; // written under an older lock epoch: every entry is checked + $break($h); + $this->writeSecondary($gfx, $h, $m); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertNotSame('skip', $plan()[2], $case); + $this->assertSame(range(20, 29), $this->keys($db->find('sk', ['g' => 'g2'])), $case); + $this->assertSame(range(20, 29), $this->keys($db->query('sk')->where(['g' => 'g2'])->get()), $case); + $this->assertSame(range(20, 29), $db->getKeysFromFieldIndex('sk', 'g', 'g2'), $case); + $this->assertSame([10], $this->keys($db->find('sk', ['g' => 'g1', 'n' => 10])), $case); + } + + // An entry that fits, without the value: skipped (that's what the global index is for) + $db->rebuildFieldIndex('sk', 'g'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + [$h, $m] = $this->secondary($gfx); + $m[2] = $this->shardFilter([], 64); + $h['le'] = null; + $this->writeSecondary($gfx, $h, $m); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame('skip', $plan()[2], 'every entry fits: the filters are trusted'); + + // Newer lines of a shard are checked: shard 1 is read for g2 once it holds g2, not for g3 + $db->rebuildFieldIndex('sk', 'g'); + $this->assertSame(1, $db->update('sk', [['key' => 15], ['set' => ['g' => 'g2']]])['n']); + $this->assertSame(1, $db->update('sk', [['key' => 16], ['set' => ['g' => 'other']]])['n']); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + $this->assertSame([0 => 'skip', 1 => 'index', 2 => 'index', 3 => 'skip'], $plan(), 'g2: shard 1 has it in its newer lines'); + $this->assertSame(array_merge([15], range(20, 29)), $this->keys($db->find('sk', ['g' => 'g2']))); + $this->assertSame([0 => 'skip', 1 => 'skip', 2 => 'skip', 3 => 'index'], array_column($db->query('sk')->where(['g' => 'g3'])->explain()['plan'], 'access', 'segment'), + "g3: shard 1's newer lines don't hold it"); + $this->assertSame([16], $this->keys($db->find('sk', ['g' => 'other'])), 'a value only in newer lines'); + $this->assertSame([10, 11, 12, 13, 14, 17, 18, 19], $this->keys($db->find('sk', ['g' => 'g1'])), 'holders whose newer lines removed some'); + } + } + + /** + * Shard tails in more than 64 shards at once (the parsed tails are cached for 64 data files): every + * lookup stays exact, and only the shard whose newer lines hold the value is read. Four records per + * shard: one update leaves 20% garbage, under the 30% that would compact the shard (and drop its tail). + */ + public function testManyShardsWithNewerLinesStayExact(): void + { + $config = ['shardSize' => 4]; + $db = $this->db($config); + $db->insert('mt', array_map(function ($n) { return ['c' => 'v' . $n]; }, range(0, 279))); // 70 shards + $db->createFieldIndex('mt', 'c'); + for ($shard = 0; $shard < 70; $shard++) { + $this->assertSame(1, $db->update('mt', [['key' => 4 * $shard], ['set' => ['c' => 'w' . $shard]]])['n']); + } + $this->assertSame(70, $db->getShardInfo('mt')['shards']); + $this->assertSame($this->secondary($this->fxPath('mt', 'c', 9))[0]['s'] + 0, $this->secondary($this->gfxPath('mt', 'c'))[0]['sh']['9']['s'], + 'precondition: the global entries are those of the checkpoints'); + $this->assertGreaterThan($this->secondary($this->gfxPath('mt', 'c'))[0]['sh']['9']['s'], $this->size($this->dataPath('mt', 9)), 'precondition: shard 9 has newer lines'); + // The shards whose filter may hold zz (§6.5.2: a shard that doesn't hold a value may answer "may"; a process + // with the exact map reads none of them) + $filters = $this->secondary($this->gfxPath('mt', 'c'))[1]; + $may = array_keys(array_filter($filters, function ($filter) { return $this->filterMayHold($filter, 'zz'); })); + $notSkipped = function (array $explain) { + return array_keys(array_filter(array_column($explain['plan'], 'access', 'segment'), function ($a) { return $a !== 'skip'; })); + }; + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + // explain() lists the first 50 segments of its plan + $plan = $db->query('mt')->where(['c' => 'zz'])->explain(); + $this->assertSame([], array_diff($notSkipped($plan), $may), 'no newer line holds it: only shards whose filter may hold it are read'); + $this->assertGreaterThanOrEqual(70 - count($may), $plan['segments']['skipped']); + foreach ([0, 5, 66, 69] as $shard) { + $this->assertSame([4 * $shard], $this->keys($db->find('mt', ['c' => 'w' . $shard]))); + $this->assertSame([], $db->find('mt', ['c' => 'v' . (4 * $shard)]), 'replaced by a newer line'); + $this->assertSame([4 * $shard + 1], $this->keys($db->find('mt', ['c' => 'v' . (4 * $shard + 1)]))); + // Read: the shard whose newer lines hold it, and at most the shards whose filter may hold it + $may = array_keys(array_filter($filters, function ($filter) use ($shard) { return $this->filterMayHold($filter, 'w' . $shard); })); + $explain = $db->query('mt')->where(['c' => 'w' . $shard])->explain(); + $this->assertSame(['total' => 70, 'scanned' => 0], array_intersect_key($explain['segments'], ['total' => 0, 'scanned' => 0]), "w$shard"); + $this->assertSame($explain['segments']['total'] - $explain['segments']['skipped'], $explain['segments']['indexed'], "w$shard"); + $this->assertSame([], array_diff($notSkipped($explain), array_merge([$shard], $may)), "w$shard: no other shard read"); + $this->assertLessThanOrEqual(1 + count($may), $explain['segments']['indexed'], "w$shard"); + $this->assertGreaterThanOrEqual(1, $explain['segments']['indexed'], "w$shard: its shard read"); + if ($shard < 50) { + $this->assertContains($shard, $notSkipped($explain), "w$shard: the shard with the newer line is read"); + } + } + } + } + + // ================================================================== public API + + /** + * getKeysFromFieldIndex() (API.md): the keys whose value is === the value, sorted; null without a + * database, without an index, for an array or object value; with a shard id only that shard's keys, + * [] for a shard that doesn't exist + */ + public function testGetKeysFromFieldIndexIsStrict(): void + { + $db = $this->db(['shardSize' => 4]); + $values = [1, '1', 1.0, true, null, 0, -0.0, '', false, 'a', [1], 1, 'a', '1']; + $db->insert('gk', array_map(function ($v) { return ['v' => $v, 'u' => 'x']; }, $values)); + $db->insert('gk', ['u' => 'no v']); // key 14: no field + $this->assertNull($db->getKeysFromFieldIndex('gk', 'v', 1), 'no index yet'); + $db->createFieldIndex('gk', 'v'); + $expected = [[1, [0, 11]], ['1', [1, 13]], [1.0, [2]], [true, [3]], [null, [4]], [0, [5]], [-0.0, [6]], [0.0, [6]], ['', [7]], + [false, [8]], ['a', [9, 12]], ['b', []], [2, []], ['0', []]]; + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + } + foreach ($expected as [$value, $keys]) { + $this->assertSame($keys, $db->getKeysFromFieldIndex('gk', 'v', $value), self::show($value)); + } + $this->assertNull($db->getKeysFromFieldIndex('gk', 'v', [1]), 'arrays are not indexed'); + $this->assertNull($db->getKeysFromFieldIndex('gk', 'v', new \stdClass())); + $this->assertNull($db->getKeysFromFieldIndex('gk', 'u', 'x'), 'u has no index'); + $this->assertNull($db->getKeysFromFieldIndex('missing', 'v', 1)); + // Shards: 0..3, 4..7, 8..11, 12..14 + $this->assertSame([0], $db->getKeysFromFieldIndex('gk', 'v', 1, 0)); + $this->assertSame([], $db->getKeysFromFieldIndex('gk', 'v', 1, 1)); + $this->assertSame([11], $db->getKeysFromFieldIndex('gk', 'v', 1, 2)); + $this->assertSame([12], $db->getKeysFromFieldIndex('gk', 'v', 'a', 3)); + $this->assertSame([13], $db->getKeysFromFieldIndex('gk', 'v', '1', '3'), 'a numeric string shard id'); + $this->assertSame([], $db->getKeysFromFieldIndex('gk', 'v', 1, 7), 'no shard 7'); + } + $db->update('gk', [['key' => 0], ['set' => ['v' => 'a']]]); + $db->delete('gk', ['key' => 9]); + $this->assertSame([11], $db->getKeysFromFieldIndex('gk', 'v', 1)); + $this->assertSame([0, 12], $db->getKeysFromFieldIndex('gk', 'v', 'a')); + $this->assertSame([], $this->dbFiles('missing')); + } + + /** hasFieldIndex() (API.md): true exactly for the indexed fields; case-sensitive; false after a drop; nothing created for a missing database */ + public function testHasFieldIndexReportsExactlyTheIndexedFields(): void + { + $db = $this->db(['shardingEnabled' => false]); + $this->assertFalse($db->hasFieldIndex('hf', 'city')); + $this->assertSame([], $this->dbFiles('hf'), 'asking creates nothing'); + $db->insert('hf', ['city' => 'A', 'City' => 'B', '7' => 'seven']); + $this->assertFalse($db->hasFieldIndex('hf', 'city')); + $db->createFieldIndex('hf', 'city'); + $db->createFieldIndex('hf', '7'); + $this->assertTrue($db->hasFieldIndex('hf', 'city')); + $this->assertFalse($db->hasFieldIndex('hf', 'City'), 'names differing in case are different fields'); + $this->assertTrue($db->hasFieldIndex('hf', '7')); + $this->assertTrue($db->hasFieldIndex('hf', 7), 'a numeric field name'); + $this->assertFalse($db->hasFieldIndex('hf', 'key')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertTrue($db->hasFieldIndex('hf', 'city')); + $db->dropFieldIndex('hf', 'city'); + $this->assertFalse($db->hasFieldIndex('hf', 'city')); + $this->assertTrue($db->hasFieldIndex('hf', '7')); + $this->assertSame([0], $this->keys($db->find('hf', ['7' => 'seven'])), 'a numeric field name is looked up through its index'); + $this->assertSame([0], $db->getKeysFromFieldIndex('hf', '7', 'seven')); + $this->assertSame('index', $db->query('hf')->where(['7' => 'seven'])->explain()['access']); + } + + /** + * createFieldIndex() (API.md): "indexed" = records with an indexable value (missing fields and arrays + * are not indexed, null is), values kept by type (-0.0 is 0.0), "shards" when sharded; creating it again + * rebuilds it from the data + */ + public function testCreateFieldIndexCountsEntriesAndKeepsValuesByType(): void + { + $data = [1, 1.0, '1', true, null, 1, 'x', [1], 'MISSING', -0.0, 0.0, false, 'x', ['a' => 1]]; + // indexable: 1, 1.0, '1', true, null, 1, 'x', -0.0, 0.0, false, 'x' = 11 entries + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('cf', array_map(function ($v) { return $v === 'MISSING' ? ['other' => 1] : ['v' => $v]; }, $data)); + $expected = ['success' => true, 'indexed' => 11, 'error' => null]; + if (isset($config['shardSize'])) { + $expected['shards'] = 5; // 14 records in shards of 3 + } + $this->assertSame($expected, $db->createFieldIndex('cf', 'v'), json_encode($config)); + $this->assertSame(['v'], $db->getFieldIndexes('cf')['fields']); + + // A damaged index file (wrong buckets under a header that still fits) is replaced by a second call + $fx = $this->fxPath('cf', 'v', isset($config['shardSize']) ? 0 : null); + [$header, $buckets] = $this->secondary($fx); + $this->writeSecondary($fx, $header, ['sx' => [0, 1, 2]]); + $db->insert('cf', ['v' => 'y']); + $expected['indexed'] = 12; + if (isset($config['shardSize'])) { + $expected['shards'] = 5; + } + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame($expected, $db->createFieldIndex('cf', 'v'), 'again: rebuilt, counted from the data'); + $this->assertSame(['v'], $db->getFieldIndexes('cf')['fields'], 'listed once'); + [, $rebuilt] = $this->secondary($fx); + if (!isset($config['shardSize'])) { + $buckets['sy'] = 14; // the record added since (sharded: key 14 is in shard 4) + } + ksort($buckets); + ksort($rebuilt); + $this->assertSame($buckets, $rebuilt, 'the buckets of the data, the damage is gone'); + $this->assertSame([6, 12], $this->keys($db->find('cf', ['v' => 'x']))); + $this->assertSame([0, 5], $this->keys($db->find('cf', ['v' => 1]))); + $this->assertSame([[1], [2], [3], [4], [9, 10], [11]], array_map(function ($v) use ($db) { return $db->getKeysFromFieldIndex('cf', 'v', $v); }, [1.0, '1', true, null, 0.0, false]), + 'by type: 1.0, "1", true, null, 0.0 (and -0.0), false'); + $this->assertSame([14], $this->keys($db->find('cf', ['v' => 'y']))); + $this->assertSame($expected, $db->rebuildFieldIndex('cf', 'v'), 'rebuildFieldIndex(): same result as createFieldIndex()'); + } + // On a field without an index, rebuildFieldIndex() creates it + $this->assertSame(['success' => true, 'indexed' => 1, 'error' => null, 'shards' => 5], $db->rebuildFieldIndex('cf', 'other')); + $this->assertSame(['v', 'other'], $db->getFieldIndexes('cf')['fields']); + } + + /** createFieldIndex() errors (API.md): "", "key" or a non-string field; a missing database without autoCreateDB */ + public function testCreateFieldIndexRejectsInvalidFieldsAndMissingDatabases(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('iv', ['a' => 1]); + $invalid = ['success' => false, 'indexed' => 0, 'error' => 'Invalid field name']; + foreach (['', 'key', 5, null, ['a'], 1.5, true] as $field) { + $this->assertSame($invalid, $db->createFieldIndex('iv', $field), self::show($field)); + } + $this->assertSame(['fields' => [], 'sharded' => false], $db->getFieldIndexes('iv')); + $this->assertSame([], $this->filesWith('.fx')); + $this->assertSame([], $this->filesWith('indexes.json')); + + $strict = $this->db(['shardingEnabled' => false, 'autoCreateDB' => false]); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Database does not exist'], $strict->createFieldIndex('absent', 'a')); + // A write takes the exclusive lock (its LOCK file may stay: an empty location, STORAGE_FORMAT.md §8.3); + // no database, index or list file + $this->assertSame([], array_values(array_diff($this->dbFiles('absent'), ['LOCK']))); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->createFieldIndex('fresh', 'a'), 'autoCreateDB: created, empty'); + $this->assertTrue($db->hasFieldIndex('fresh', 'a')); + $db->insert('fresh', [['a' => 1], ['a' => 2]]); + $this->assertSame([1], $this->keys($db->find('fresh', ['a' => 2]))); + + $this->expectException(\InvalidArgumentException::class); + $db->createFieldIndex('bad/name', 'a'); + } + + /** + * Index file names hold no database name (STORAGE_FORMAT.md §4: the database directory is its 32 hex + * locator; §5.1: indexes/{segment}.{token}.fx, indexes/{token}.gfx): a database name of any length can + * index any field, and every file name stays within 255 bytes + */ + public function testLongDatabaseNamesCanIndexAnyField(): void + { + $db = $this->db(['shardingEnabled' => false]); + $field40 = str_repeat('f', 40); + $field39 = str_repeat('f', 39); + $hashed = str_repeat('F', 60); // token: 24 + 1 + 12 = 37 bytes + $cases = [[158, $field40], [159, $field40], [159, $field39], [161, $hashed], [162, $hashed], [197, 'a'], [198, 'a'], [200, 'a']]; + foreach ($cases as [$length, $field]) { + $name = str_repeat('n', $length); + $this->assertSame(1, $db->insert($name, [$field => 'x'])['n']); + $result = $db->createFieldIndex($name, $field); + $label = "name of $length bytes, field token of " . strlen($this->token($field)) . ' bytes'; + $this->assertSame(['success' => true, 'indexed' => 1, 'error' => null], $result, $label); + $this->assertTrue($db->hasFieldIndex($name, $field), $label); + $this->assertFileExists($this->fxPath($name, $field), $label); + $this->assertSame([0], $this->keys($db->find($name, [$field => 'x'])), $label); + $this->assertSame(1, $db->insert($name, [$field => 'y'])['n'], "$label: the database keeps working"); + $this->assertSame([1], $db->getKeysFromFieldIndex($name, $field, 'y'), $label); + foreach ($this->dbFiles($name) as $file) { + $this->assertLessThanOrEqual(255, strlen(basename($file)), "$label: $file"); + } + $db->dropDB($name); + } + } + + /** + * Index file names (STORAGE_FORMAT.md §5.1): indexes/{segment}.{token}.fx and indexes/{token}.gfx; the token is + * the field when it is [a-z0-9_]{1,40}, else a lower-case prefix, "-" and 12 hex digits of its sha256, + * at most 40 bytes; names that differ only in case never share a file + */ + public function testIndexFileTokensFollowTheNamingRule(): void + { + $db = $this->db(['shardSize' => 2]); + $fields = ['city', 'City', 'CITY', 'e-mail', 'e_mail', 'first.name', 'şehir', str_repeat('a', 40), str_repeat('a', 41), str_repeat('ü', 110), '_9', '0']; + $db->insert('tk', [array_fill_keys($fields, 'x'), array_fill_keys($fields, 'y'), array_fill_keys($fields, 'z')]); + $tokens = []; + foreach ($fields as $field) { + $this->assertTrue($db->createFieldIndex('tk', $field)['success'], $field); + $token = $this->token($field); + $this->assertLessThanOrEqual(40, strlen($token)); + if (preg_match('/^[a-z0-9_]{1,40}$/', $field)) { + $this->assertSame($field, $token); + } else { + $this->assertMatchesRegularExpression('/^[a-z0-9_]{0,24}-' . substr(hash('sha256', $field), 0, 12) . '$/', $token); + } + $tokens[] = $token; + foreach ([0, 1] as $shard) { + $this->assertFileExists($this->fxPath('tk', $field, $shard), $field); + $this->assertSame((string)$field, $this->secondary($this->fxPath('tk', $field, $shard))[0]['field'], $field); + } + $this->assertFileExists($this->gfxPath('tk', $field), $field); + $this->assertSame((string)$field, $this->secondary($this->gfxPath('tk', $field))[0]['field'], $field); + } + $this->assertSame(count($fields), count(array_unique(array_map('strtolower', $tokens))), 'no two tokens equal, even ignoring case'); + $this->assertCount(count($fields) * 2, $this->filesWith('.fx')); + $this->assertCount(count($fields), $this->filesWith('.gfx')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 2]); + foreach ($fields as $field) { + $this->assertSame([2], $this->keys($db->find('tk', [$field => 'z'])), $field); + $this->assertSame([1], $db->getKeysFromFieldIndex('tk', $field, 'y'), $field); + } + } + + /** + * dropFieldIndex() (API.md): removes the index files of every shard and the global one, the index list + * entry (the list file when no index is left); queries stay exact (scan); a second drop and an unknown + * field: "Index does not exist". + */ + public function testDropFieldIndexRemovesEveryFileAndKeepsQueriesExact(): void + { + $db = $this->db(['shardSize' => 5]); + $db->insert('dr', array_map(function ($n) { return ['a' => $n % 3, 'b' => 'b' . ($n % 2), 'e-mail' => "m$n"]; }, range(0, 11))); + $db->createFieldIndex('dr', 'a'); + $db->createFieldIndex('dr', 'b'); + $db->createFieldIndex('dr', 'e-mail'); + $this->assertCount(3, $this->filesWith('.a.fx')); + + $this->assertSame(['success' => true, 'error' => null], $db->dropFieldIndex('dr', 'a')); + $this->assertSame([], $this->filesWith('.a.fx')); + $this->assertSame([], $this->filesWith('/a.gfx')); + $this->assertCount(3, $this->filesWith('.b.fx'), 'the other indexes stay'); + $this->assertFileExists($this->gfxPath('dr', 'b')); + $this->assertSame(['b', 'e-mail'], $db->getFieldIndexes('dr')['fields']); + $this->assertSame(['b', 'e-mail'], json_decode(file_get_contents($this->dbPath('dr', 'indexes')), true)['fields']); + $this->assertFalse($db->hasFieldIndex('dr', 'a')); + $this->assertNull($db->getKeysFromFieldIndex('dr', 'a', 1)); + $this->assertSame([1, 4, 7, 10], $this->keys($db->find('dr', ['a' => 1]))); + $explain = $db->query('dr')->where(['a' => 1])->explain(); + $this->assertSame(['scan', 0], [$explain['access'], $explain['segments']['skipped']], 'no index, no shard skipped'); + $this->assertSame(['success' => false, 'error' => 'Index does not exist'], $db->dropFieldIndex('dr', 'a')); + $this->assertSame(['success' => false, 'error' => 'Index does not exist'], $db->dropFieldIndex('dr', 'nothing')); + + $this->assertSame(['success' => true, 'error' => null], $db->dropFieldIndex('dr', 'e-mail')); + $this->assertSame(['success' => true, 'error' => null], $db->dropFieldIndex('dr', 'b')); + $this->assertFileDoesNotExist($this->dbPath('dr', 'indexes'), 'no index left: no list'); + $this->assertSame([], $this->filesWith('.fx')); + $this->assertSame([], $this->filesWith('.gfx')); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $this->assertSame(['fields' => [], 'sharded' => true], $db->getFieldIndexes('dr')); + $this->assertSame([0, 2, 4, 6, 8, 10], $this->keys($db->find('dr', ['b' => 'b0']))); + $this->assertSame([], $this->filesWith('.fx'), 'a lookup builds no dropped index'); + } + + // ================================================================== checkpoints + + /** + * Pads a pure writer's records so that the data after $covered is exactly $tail bytes long. Each line + * is the same record shape: its length is measured once (an input, not an expected value). + */ + private function growTail(string $name, array $config, int $covered, int $tail, array $shape): void + { + \noneDB::clearStaticCache(); + $writer = $this->db($config); // loads no field index: never writes a checkpoint of one + $data = $this->dataPath($name); + $start = $this->size($data); + $writer->insert($name, $shape + ['p' => str_repeat('x', 100)]); + $line = $this->size($data) - $start; + $overhead = $line - 100; + $need = $tail - ($this->size($data) - $covered); + $this->assertGreaterThanOrEqual($overhead, $need, 'precondition: room for one more line'); + $writer->insert($name, $shape + ['p' => str_repeat('x', $need - $overhead)]); + $this->assertSame($tail, $this->size($data) - $covered, 'precondition: the tail is exactly ' . $tail . ' bytes'); + \noneDB::clearStaticCache(); + } + + /** + * CHANGES.md: "A new checkpoint is written once at least 64 KiB of data ... is new, by a process + * that uses the index: a reader that loaded it". 65535 new bytes: no checkpoint; 65536: the reader + * writes one, without moving the lock generation (CLAUDE.md: derived write); explain(), + * checkIntegrity() and a join (snapshot) never write one. + */ + public function testReaderCheckpointsStartAt64KiBOfNewData(): void + { + $config = ['shardingEnabled' => false]; + foreach ([65535 => false, 65536 => true] as $tail => $written) { + $name = 'th' . $tail; + $db = $this->db($config); + $db->insert($name, ['c' => 'a']); + $db->createFieldIndex($name, 'c'); + $fx = $this->fxPath($name, 'c'); + $covered = $this->secondary($fx)[0]['s']; + $this->assertSame($this->size($this->dataPath($name)), $covered); + $this->growTail($name, $config, $covered, $tail, ['c' => 'b']); + $before = file_get_contents($fx); + $gen = $this->lockState($name)['gen']; + + $reader = $this->db($config); + $reader->insert('other' . $tail, ['ref' => 'b']); + $this->assertSame(2, $reader->query($name)->where(['c' => 'b'])->explain()['estimatedRead']); + $this->assertTrue($reader->checkIntegrity($name)['ok']); + $joined = $reader->query('other' . $tail)->join($name, 'ref', 'c', 'o')->get(); + $this->assertSame(2, $joined[0]['o']['key'], 'the last record with the value joins'); + clearstatcache(); + $this->assertSame($before, file_get_contents($fx), "explain(), checkIntegrity() and a join write no checkpoint (tail $tail)"); + + \noneDB::clearStaticCache(); + $reader = $this->db($config); + $this->assertSame([1, 2], $this->keys($reader->find($name, ['c' => 'b']))); + $this->assertSame([0], $this->keys($reader->find($name, ['c' => 'a']))); + [$header, $buckets, $content] = $this->secondary($fx); + if ($written) { + $this->assertSame($this->size($this->dataPath($name)), $header['s'], 'tail 65536: the reader wrote a checkpoint'); + $this->assertSame(['sa' => 0, 'sb' => [1, 2]], $buckets); + $this->assertSame([1, 'fx', 'c', 2, 3, 2], [$header['v'], $header['kind'], $header['field'], $header['mk'], $header['n'], $header['c']]); // STORAGE_FORMAT.md §6.5.1 + } else { + $this->assertSame($before, $content, 'tail 65535: no checkpoint yet'); + } + $this->assertSame($gen, $this->lockState($name)['gen'], 'a reader never moves the generation'); + \noneDB::clearStaticCache(); + $this->assertSame([1, 2], $this->keys($this->db($config)->find($name, ['c' => 'b'])), 'the next process uses it'); + } + } + + /** + * The other half of the rule: "and at least a quarter of the index file's size". An index file of + * about 320 KB waits for its quarter (about 80 KB) of new data, not 64 KiB (margins of 1000 bytes + * either way, so the header line doesn't matter) + */ + public function testReaderCheckpointsOfALargeIndexWaitForAQuarterOfItsSize(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('big', array_map(function ($n) { return ['c' => str_repeat('v', 200) . $n]; }, range(0, 1499))); + $db->createFieldIndex('big', 'c'); + $fx = $this->fxPath('big', 'c'); + $covered = $this->secondary($fx)[0]['s']; + $fileSize = $this->size($fx); + $quarter = intdiv($fileSize, 4); + $this->assertGreaterThan(65536 + 3000, $quarter, 'precondition: the quarter rule decides'); + + $this->growTail('big', $config, $covered, $quarter - 1000, ['c' => 'new']); + $before = file_get_contents($fx); + $this->assertSame([1500, 1501], $this->keys($this->db($config)->find('big', ['c' => 'new']))); + clearstatcache(); + $this->assertSame($before, file_get_contents($fx), 'more than 64 KiB, less than a quarter: no checkpoint'); + + $this->growTail('big', $config, $covered, $quarter + 1000, ['c' => 'new']); + $gen = $this->lockState('big')['gen']; + $this->assertSame([1500, 1501, 1502, 1503], $this->keys($this->db($config)->find('big', ['c' => 'new']))); + $header = $this->secondary($fx)[0]; + $this->assertSame($this->size($this->dataPath('big')), $header['s'], 'beyond a quarter: checkpoint'); + $this->assertSame([1501, 1504, 1503], [$header['c'], $header['n'], $header['mk']], '1500 values and "new" (4 keys), highest key 1503'); + $this->assertSame($gen, $this->lockState('big')['gen']); + } + + /** + * "or a writer that has it loaded": a write session that appended 64 KiB+ to a shard whose field index + * this process has loaded writes the shard's checkpoint and its global entry at its end; the next + * process then rules the shard out with the new entry + */ + public function testAWriterThatHasTheIndexLoadedCheckpointsItAndItsGlobalEntry(): void + { + $config = ['shardSize' => 100]; + $db = $this->db($config); + $db->insert('wl', array_map(function ($n) { return ['c' => 'c' . ($n % 5)]; }, range(0, 149))); + $db->createFieldIndex('wl', 'c'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(3, 148, 5), $this->keys($db->find('wl', ['c' => 'c3'])), 'loads both shard indexes'); + $fx0 = file_get_contents($this->fxPath('wl', 'c', 0)); + $db->insert('wl', array_map(function ($n) { return ['c' => 'new' . $n, 'p' => str_repeat('x', 2000)]; }, range(0, 39))); // keys 150..189, shard 1 + $size1 = $this->size($this->dataPath('wl', 1)); + [$header1, $buckets1] = $this->secondary($this->fxPath('wl', 'c', 1)); + $this->assertSame($size1, $header1['s'], 'the shard index covers the session'); + $this->assertSame(150, $buckets1['snew0']); + [$global, $filters] = $this->secondary($this->gfxPath('wl', 'c')); + $this->assertSame($size1, $global['sh']['1']['s'], 'and its global entry'); + // Shard 1 now holds 45 values: more than the 256 bits its five made allow (32 per value), so the file was + // made again with 2048 bits (the smallest power of two of at least 1440) + $this->assertSame(2048, $global['bits']); + $this->assertSame([0 => $this->shardFilter(['c0', 'c1', 'c2', 'c3', 'c4'], 2048), + 1 => $this->shardFilter(array_merge(['c0', 'c1', 'c2', 'c3', 'c4'], array_map(function ($n) { return 'new' . $n; }, range(0, 39))), 2048)], $filters); + $this->assertSame($fx0, file_get_contents($this->fxPath('wl', 'c', 0)), 'shard 0 got nothing new'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([0 => 'skip', 1 => 'index'], array_column($db->query('wl')->where(['c' => 'new7'])->explain()['plan'], 'access', 'segment')); + $this->assertSame([157], $this->keys($db->find('wl', ['c' => 'new7']))); + } + + /** + * Index files are never needed for correctness: a checkpoint a reader can't write, and an index that + * can't be built (its file can't be written), leave the results exact (the segment is scanned) and + * no partial file behind; once writing works again the files appear + */ + public function testAnIndexFileThatCantBeWrittenFallsBackToAScan(): void + { + $config = ['shardingEnabled' => false, 'lockTimeout' => 0.05]; + $db = $this->db($config); + $rows = [['c' => 'a', 'n' => 1], ['c' => 'b', 'n' => 5]]; + $db->insert('ff', $rows); + $db->createFieldIndex('ff', 'c'); + $db->createFieldIndex('ff', 'n'); + \noneDB::clearStaticCache(); + $more = array_map(function ($i) { return ['c' => 'c' . ($i % 3), 'n' => $i, 'p' => str_repeat('x', 1000)]; }, range(0, 69)); + $this->db($config)->insert('ff', $more); // a pure writer: 70 KB the index doesn't cover + $rows = array_merge($rows, $more); + $fx = $this->fxPath('ff', 'c'); + $before = file_get_contents($fx); + $this->setStatic('renameFn', function ($from, $to) { return substr($to, -3) !== '.fx' && rename($from, $to); }); + + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(self::select($rows, self::eq('c', 'c1')), $this->keys($db->find('ff', ['c' => 'c1']))); + clearstatcache(); + $this->assertSame($before, file_get_contents($fx), 'the checkpoint failed: the file is unchanged'); + + unlink($this->fxPath('ff', 'n')); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(self::select($rows, self::eq('n', 5)), $this->keys($db->find('ff', ['n' => 5])), 'equality: scan'); + $this->assertSame([1, 7], $this->keys($db->find('ff', ['n' => 5]))); + $range = self::select($rows, function ($r) { return self::holds($r, 'n', '$gte', 10) && self::holds($r, 'n', '$lt', 20); }); + $this->assertSame($range, $this->keys($db->query('ff')->where(['n' => ['$gte' => 10, '$lt' => 20]])->get()), 'range: scan'); + $this->assertSame($range, $this->keys($db->between('ff', 'n', 10, 19))); + $this->assertSame([1, 7], $this->keys($db->query('ff')->whereIn('n', [5])->get())); + $this->assertFileDoesNotExist($this->fxPath('ff', 'n')); + $this->assertSame([], $this->filesWith('.tmp'), 'no temp file left'); + + // A writer that has the index loaded can't write its checkpoint at the end of the session: the + // write succeeds anyway (checkpoints are best effort) + $this->assertSame(self::select($rows, self::eq('c', 'c2')), $this->keys($db->find('ff', ['c' => 'c2'])), 'loads the index of c'); + $before = file_get_contents($fx); + $batch = array_map(function ($i) { return ['c' => 'late', 'n' => 1000 + $i, 'p' => str_repeat('y', 1000)]; }, range(0, 69)); + $this->assertSame(['n' => 70], $db->insert('ff', $batch)); + $rows = array_merge($rows, $batch); + clearstatcache(); + $this->assertSame($before, file_get_contents($fx)); + $this->assertSame(self::select($rows, self::eq('c', 'late')), $this->keys($db->find('ff', ['c' => 'late']))); + + $this->setStatic('renameFn', null); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([1, 7], $this->keys($db->find('ff', ['n' => 5]))); + $this->assertFileExists($this->fxPath('ff', 'n')); + $this->assertSame(self::select($rows, self::eq('c', 'c1')), $this->keys($db->find('ff', ['c' => 'c1']))); + $this->assertSame($this->size($this->dataPath('ff')), $this->secondary($fx)[0]['s'], 'now the checkpoint is written'); + } + + /** + * API.md: "Waiting longer than lockTimeout seconds throws RuntimeException with code + * noneDB::LOCK_FAILED". A read that must build a missing index needs the exclusive lock; while another + * process holds a shared one, the read fails with LOCK_FAILED (it never continues without a lock) + */ + public function testBuildingAnIndexUnderAnotherReadersLockFailsWithLockFailed(): void + { + $config = ['shardingEnabled' => false, 'lockTimeout' => 0.3]; + $db = $this->db($config); + $db->insert('lc', [['c' => 'a'], ['c' => 'b']]); + $db->createFieldIndex('lc', 'c'); + unlink($this->fxPath('lc', 'c')); + \noneDB::clearStaticCache(); + $lock = $this->dbPath('lc', 'lock'); + $script = $this->childScript('$fp = fopen(' . var_export($lock, true) . ', "rb"); flock($fp, LOCK_SH); echo "held\n"; fflush(STDOUT); fgets(STDIN);'); + $proc = proc_open([PHP_BINARY, $script], [0 => ['pipe', 'r'], 1 => ['pipe', 'w']], $pipes); + $this->assertIsResource($proc); + try { + $this->assertSame("held\n", fgets($pipes[1])); + foreach (['equality' => function (\noneDB $db) { return $db->find('lc', ['c' => 'b']); }, + 'range' => function (\noneDB $db) { return $db->query('lc')->where(['c' => ['$gte' => 'b']])->get(); }] as $what => $read) { + try { + $read($this->db($config)); + $this->fail("$what: no exception"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode(), "$what: " . $e->getMessage()); + } + } + } finally { + fwrite($pipes[0], "go\n"); + fclose($pipes[0]); + fclose($pipes[1]); + proc_close($proc); + } + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([1], $this->keys($db->find('lc', ['c' => 'b']))); + $this->assertFileExists($this->fxPath('lc', 'c')); + } + + /** + * An index file that doesn't describe its data is rebuilt from the data: a body that isn't JSON, a + * checksum that doesn't match (even with plausible buckets), another kind or a header without "s". + * A file whose bucket is a one-element list (not what this version writes, still a valid index) is + * followed through updates. + */ + public function testDamagedIndexFilesAreRebuiltFromTheData(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('dm', [['c' => 'a'], ['c' => 'b'], ['c' => 'b'], ['c' => 'c']]); + $db->createFieldIndex('dm', 'c'); + $fx = $this->fxPath('dm', 'c'); + [$header, $buckets, $pristine] = $this->secondary($fx); + $this->assertSame(['sa' => 0, 'sb' => [1, 2], 'sc' => 3], $buckets); + // Damages in the paged format (§6.5.1); a valid body with other entries where the header is the broken part + $at = strpos($pristine, "\n") + 1; // the body's first byte + $wrong = ['sa' => [1, 2, 3]]; + $damages = [ + 'body cut' => function () use ($fx, $pristine, $at) { file_put_contents($fx, substr($pristine, 0, $at + intdiv(strlen($pristine) - $at, 2))); }, + 'page crc differs' => function () use ($fx, $pristine, $at) { $c = $pristine; $c[$at + 2] = chr(ord($c[$at + 2]) ^ 1); file_put_contents($fx, $c); }, + 'checksum differs' => function () use ($fx, $header, $wrong) { $this->writeSecondary($fx, ['h' => $header['h'] === '00000000' ? '11111111' : '00000000'] + $header, $wrong); }, + 'another kind' => function () use ($fx, $header, $wrong) { $this->writeSecondary($fx, ['kind' => 'rx'] + $header, $wrong); }, + 'no covered bytes' => function () use ($fx, $header, $wrong) { $this->writeSecondary($fx, array_diff_key($header, ['s' => 0]), $wrong); }, + 'unknown version (a JSON body)' => function () use ($fx, $header, $wrong) { file_put_contents($fx, json_encode(['v' => 1] + array_diff_key($header, ['pg' => 0, 'w' => 0])) . "\n" . json_encode($wrong) . "\n"); }, + 'header not JSON' => function () use ($fx, $pristine, $at) { file_put_contents($fx, "garbage\n" . substr($pristine, $at)); }, + ]; + foreach ($damages as $damage => $write) { + $write(); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([0], $this->keys($db->find('dm', ['c' => 'a'])), $damage); + $this->assertSame([1, 2], $db->getKeysFromFieldIndex('dm', 'c', 'b'), $damage); + $this->assertSame([1, 2, 3], $this->keys($db->query('dm')->where(['c' => ['$gte' => 'b']])->get()), $damage); + [$h, $b] = $this->secondary($fx); + $this->assertSame($buckets, $b, "$damage: rebuilt"); + $this->assertSame([$header['s'], $header['h']], [$h['s'], $h['h']], $damage); + } + + // A file written by the test (its own write token): a bucket of one key, the key leaves it, the bucket goes + file_put_contents($fx, $pristine); + $this->writeSecondary($fx, $header, ['sa' => [0], 'sb' => [1, 2], 'sc' => [3]]); + $written = $this->secondary($fx)[2]; + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([0], $db->getKeysFromFieldIndex('dm', 'c', 'a')); + $this->assertSame($written, $this->secondary($fx)[2], 'precondition: the file fits and is used, not rebuilt'); + $db->update('dm', [['key' => 0], ['set' => ['c' => 'z']]]); + // Replayed onto the loaded file (the next update compacts: garbage over 30%, a new data file) + $this->assertSame([], $db->getKeysFromFieldIndex('dm', 'c', 'a'), 'the one-element bucket is gone'); + $this->assertSame([0], $db->getKeysFromFieldIndex('dm', 'c', 'z')); + $this->assertSame([], $this->keys($db->query('dm')->where(['c' => ['$lt' => 'b']])->get())); + $db->update('dm', [['key' => 3], ['set' => ['c' => 'b']]]); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + $this->assertSame([], $db->getKeysFromFieldIndex('dm', 'c', 'a')); + $this->assertSame([], $db->getKeysFromFieldIndex('dm', 'c', 'c')); + $this->assertSame([0], $db->getKeysFromFieldIndex('dm', 'c', 'z')); + $this->assertSame([1, 2, 3], $db->getKeysFromFieldIndex('dm', 'c', 'b')); + $this->assertSame([0, 1, 2, 3], $this->keys($db->query('dm')->where(['c' => ['$gte' => 'a']])->get())); + $this->assertSame([1, 2, 3], $this->keys($db->query('dm')->where(['c' => ['$lt' => 'z']])->get())); + } + $this->assertTrue($db->checkIntegrity('dm')['ok']); + } + + // ================================================================== index list + + /** + * An index list that is empty is derived from the index files (STORAGE_FORMAT.md §5.1 indexes.json: "An + * empty or invalid file is derived again from the headers of the generation's .fx/.rx files"): the + * header names the field, also one whose token is hashed ("e-mail") + */ + public function testAnEmptyIndexListIsDerivedFromTheIndexFiles(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('dv', [['city' => 'A', 'e-mail' => 'a@x', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], + ['city' => 'B', 'e-mail' => 'b@x', 'loc' => ['type' => 'Point', 'coordinates' => [2, 2]]]]); + $db->createFieldIndex('dv', 'city'); + $db->createFieldIndex('dv', 'e-mail'); + $db->createSpatialIndex('dv', 'loc'); + file_put_contents($this->dbPath('dv', 'indexes'), ''); // zero-length after a power loss + \noneDB::clearStaticCache(); + $db = $this->db($config); + $fields = $db->getFieldIndexes('dv')['fields']; + sort($fields); + $this->assertSame(['city', 'e-mail'], $fields); + $this->assertSame(['loc'], $db->getSpatialIndexes('dv')); + $this->assertSame([1], $this->keys($db->find('dv', ['e-mail' => 'b@x']))); + $this->assertSame([0], $this->keys($db->find('dv', ['city' => 'A']))); + $this->assertSame([1], $this->keys($db->withinBBox('dv', 'loc', 1.5, 1.5, 2.5, 2.5))); + } + + // ================================================================== other filters + + /** + * first(), exists() and count() of noneDB with a key filter (find() semantics: a key, a digit string, + * a list in the order asked for), also next to an indexed field + */ + public function testKeyFiltersInFirstExistsAndCount(): void + { + $db = $this->db(['shardSize' => 4]); + $db->insert('kf', array_map(function ($n) { return ['c' => 'c' . ($n % 2)]; }, range(0, 9))); + $db->createFieldIndex('kf', 'c'); + $this->assertTrue($db->exists('kf', ['key' => 3])); + $this->assertTrue($db->exists('kf', ['key' => '3'])); + $this->assertFalse($db->exists('kf', ['key' => 99])); + $this->assertTrue($db->exists('kf', ['key' => 3, 'c' => 'c1'])); + $this->assertFalse($db->exists('kf', ['key' => 3, 'c' => 'c0'])); + $this->assertSame(7, $db->first('kf', ['key' => [7, 3]])['key'], 'the order of the keys asked for'); + $this->assertSame(3, $db->first('kf', ['key' => [8, 3], 'c' => 'c1'])['key']); + $this->assertNull($db->first('kf', ['key' => [8, 6], 'c' => 'c1'])); + $this->assertSame(2, $db->count('kf', ['key' => [1, 2, 999]])); + $this->assertSame(1, $db->count('kf', ['key' => [1, 2, 999], 'c' => 'c0'])); + } + + /** An object as a filter value is identical to no stored value (records hold arrays): no match, no error */ + public function testObjectFilterValuesMatchNothing(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('ob', [['c' => ['a' => 1]], ['c' => []], ['c' => 'x']]); + $db->createFieldIndex('ob', 'c'); + $this->assertSame([], $db->find('ob', ['c' => (object)['a' => 1]])); + $this->assertSame([], $db->query('ob')->where(['c' => (object)['a' => 1]])->get()); + $this->assertSame([], $db->query('ob')->where(['c' => ['$eq' => new \stdClass()]])->get()); + $this->assertSame(0, $db->query('ob')->where(['c' => new \stdClass()])->count()); + $this->assertFalse($db->exists('ob', ['c' => new \stdClass()])); + $this->assertSame([0], $this->keys($db->find('ob', ['c' => ['a' => 1]])), 'the array itself matches'); + } + } + + /** Index file tokens are cached per process; the cache stays bounded and tokens stay the same */ + public function testTheTokenCacheStaysBounded(): void + { + $db = $this->db(['shardingEnabled' => false]); + for ($i = 0; $i < 5000; $i++) { + $field = 'Field ' . $i; + $this->assertSame($this->token($field), $this->callPrivate($db, 'indexToken', [$field])); + } + $this->assertLessThanOrEqual(4097, count($this->getStatic('indexTokens'))); + $this->assertSame($this->token('Field 7'), $this->callPrivate($db, 'indexToken', ['Field 7']), 'the same token after the cache was cleared'); + } + + /** + * CHANGES.md: "createFieldIndex() builds the index before the index list names it, so a creation + * that fails ... leaves no broken entry"; API.md: creating an index is not undone, run it again + */ + public function testAnIndexListThatCantBeWrittenLeavesNoBrokenEntry(): void + { + $config = ['shardingEnabled' => false, 'lockTimeout' => 0.05]; + $db = $this->db($config); + $db->insert('il', [['c' => 'a'], ['c' => 'b']]); + $db->createFieldIndex('il', 'c'); + $list = $this->dbPath('il', 'indexes'); + $before = file_get_contents($list); + $this->setStatic('renameFn', function ($from, $to) { return basename($to) !== 'indexes.json' && rename($from, $to); }); + try { + $db->createFieldIndex('il', 'd'); + $this->fail('no exception'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('failed to write index list', $e->getMessage()); + } + $this->setStatic('renameFn', null); + clearstatcache(); + $this->assertSame($before, file_get_contents($list), 'the list is unchanged'); + $this->assertSame([], $this->filesWith('.tmp')); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + $this->assertSame(['c'], $db->getFieldIndexes('il')['fields']); + $this->assertFalse($db->hasFieldIndex('il', 'd')); + $this->assertSame([1], $this->keys($db->find('il', ['c' => 'b']))); + } + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->createFieldIndex('il', 'd'), 'run again: works'); + $this->assertSame(['c', 'd'], $db->getFieldIndexes('il')['fields']); + } + + /** The documented bucket id of a value (CLAUDE.md "typed buckets"): n, t, f, i, d, s; null for arrays */ + private static function typedId($v): ?string + { + if ($v === null) { + return 'n'; + } + if (is_bool($v)) { + return $v ? 't' : 'f'; + } + if (is_int($v)) { + return 'i' . $v; + } + if (is_float($v)) { + return 'd' . bin2hex(pack('E', $v + 0.0)); + } + return is_string($v) ? 's' . $v : null; + } + + /** + * STORAGE_FORMAT.md §6.5: a global index records the lock epoch it was written under ("le"); while it + * is current every entry fits (no data file replaced). Checked against the files: each entry's epoch is its + * shard's, it covers at most the shard's bytes and "h" is the crc32b of the up to 256 bytes before "s"; + * an entry that covers its whole shard lists exactly the shard's values in the map. + * @return bool false when the recorded epoch isn't current (then nothing is promised) + */ + private function assertGlobalEntriesFit(string $name, string $field, string $label): bool + { + [$header, $filters] = $this->secondary($this->gfxPath($name, $field)); + $epoch = $this->lockState($name)['epoch']; + if ($header['le'] !== $epoch) { + return false; + } + foreach ($header['sh'] as $shard => $entry) { + $data = $this->dataPath($name, (int)$shard); + $this->assertFileExists($data, "$label: shard $shard exists"); + $ckpt = json_decode(strtok((string)file_get_contents($this->dbPath($name, 'kix', (int)$shard)), "\n"), true); + $this->assertSame($ckpt['e'], $entry['e'], "$label: shard $shard epoch"); + $size = $this->size($data); + $this->assertLessThanOrEqual($size, $entry['s'], "$label: shard $shard bytes"); + $fp = fopen($data, 'rb'); + fseek($fp, max(0, $entry['s'] - 256)); + $bytes = $entry['s'] === 0 ? '' : (string)fread($fp, $entry['s'] - max(0, $entry['s'] - 256)); + fclose($fp); + $this->assertSame(hash('crc32b', $bytes), $entry['h'], "$label: shard $shard checksum"); + if ($entry['s'] === $size) { + $values = []; + foreach (array_filter(explode("\n", (string)file_get_contents($data)), 'strlen') as $line) { + $record = json_decode($line, true); + if (isset($record['__nonedb_deleted'])) { + unset($values[$record['__nonedb_deleted']]); + continue; + } + $id = array_key_exists($field, $record) ? self::typedId($record[$field]) : null; + if ($id === null) { + unset($values[$record['key']]); + } else { + $values[$record['key']] = $id; + } + } + $this->assertSame($this->shardFilterOfIds(array_unique($values), $header['bits']), $filters[$shard], "$label: shard $shard values"); + } + } + return true; + } + + /** + * compact() replaces the data files and drops shards that became empty: lookups stay exact, the global + * index is written again for the shards that are left, every entry fits, shards are skipped again + */ + public function testCompactionThatDropsShardsKeepsTheGlobalIndexTrue(): void + { + $config = ['shardSize' => 10]; + $db = $this->db($config); + $db->insert('ce', array_map(function ($n) { return ['g' => 'g' . intdiv($n, 10), 'n' => $n]; }, range(0, 39))); + $db->createFieldIndex('ce', 'g'); + $this->assertTrue($this->assertGlobalEntriesFit('ce', 'g', 'created'), 'just written: under the current epoch'); + $this->assertSame(10, $db->delete('ce', ['g' => 'g1'])['n']); + $this->assertSame(2, $db->update('ce', [['key' => [20, 21]], ['set' => ['g' => 'moved']]])['n']); + $result = $db->compact('ce'); + $this->assertTrue($result['success']); + $this->assertSame(3, $result['newShardCount'], 'shard 1 is empty: dropped'); + $this->assertFileDoesNotExist($this->dataPath('ce', 1)); + $this->assertFileDoesNotExist($this->fxPath('ce', 'g', 1)); + $this->assertGlobalEntriesFit('ce', 'g', 'compacted'); + foreach ([true, false] as $warm) { + if (!$warm) { + \noneDB::clearStaticCache(); + $db = $this->db($config); + } + $this->assertSame([], $db->find('ce', ['g' => 'g1'])); + $this->assertSame(range(22, 29), $this->keys($db->find('ce', ['g' => 'g2']))); + $this->assertSame([20, 21], $this->keys($db->find('ce', ['g' => 'moved']))); + $this->assertSame(range(30, 39), $this->keys($db->find('ce', ['g' => 'g3']))); + $this->assertSame([0 => 'skip', 2 => 'index', 3 => 'skip'], array_column($db->query('ce')->where(['g' => 'g2'])->explain()['plan'], 'access', 'segment')); + } + $this->assertGlobalEntriesFit('ce', 'g', 'after lookups'); + $this->assertTrue($db->checkIntegrity('ce')['ok']); + } + + /** + * @test A checkpoint that a read-only load put off (explain(), checkIntegrity(), a join: they never + * write) is written by the next plain read of the same process (CHANGES.md "Index formats": a new + * checkpoint is written by a process that uses the index: a reader that loaded it) + */ + public function testACheckpointPutOffByAReadOnlyLoadIsWrittenByTheNextRead(): void + { + $loads = [ + 'explain' => function (\noneDB $db) { $db->query('po')->where(['c' => 'c1'])->explain(); }, + 'checkIntegrity' => function (\noneDB $db) { $db->checkIntegrity('po'); }, + 'join' => function (\noneDB $db) { $db->query('pj')->join('po', 'c', 'c')->get(); }, + ]; + foreach ($loads as $label => $load) { + $this->cleanTestDirectory(); + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('po', [['c' => 'c0']]); + $db->insert('pj', [['c' => 'c1']]); + $this->assertTrue($db->createFieldIndex('po', 'c')['success']); + $before = $this->secondary($this->fxPath('po', 'c'))[0]['s']; + // A writer that never reads the index: more than 64 KiB of data after the checkpoint + \noneDB::clearStaticCache(); + $writer = $this->db($config); + for ($i = 0; $i < 4; $i++) { + $writer->insert('po', array_map(function ($n) { return ['c' => 'c' . ($n % 3), 'pad' => str_repeat('x', 40)]; }, range(0, 499))); + } + $size = $this->size($this->dataPath('po')); + $this->assertGreaterThan($before + 65536, $size, 'precondition: a long tail'); + $this->assertSame($before, $this->secondary($this->fxPath('po', 'c'))[0]['s'], 'the writer wrote no checkpoint'); + + \noneDB::clearStaticCache(); + $db = $this->db($config); + $load($db); + $this->assertSame($before, $this->secondary($this->fxPath('po', 'c'))[0]['s'], $label . ' writes nothing'); + // the same process, a plain read: the checkpoint now covers the data + $this->assertCount(668, $db->find('po', ['c' => 'c1']), $label); // 4 inserts x 167 (n = 1, 4, ..., 499) + $this->assertSame($size, $this->secondary($this->fxPath('po', 'c'))[0]['s'], $label . ': written by the next read'); + } + } + + /** + * @test Building a field or spatial index of a sharded database writes its global file once, not once + * per shard: per shard it was O(shards²) (10M records in 1000 shards: 242 s for createFieldIndex()). + * Counted through the rename hook every atomic write goes through. + */ + public function testIndexBuildsWriteTheGlobalFileOnce(): void + { + $db = $this->db(['shardSize' => 10]); + $rows = []; + for ($n = 0; $n < 200; $n++) { // 20 shards, every value of "c" in every shard + $rows[] = ['c' => 'c' . ($n % 5), 'g' => ['type' => 'Point', 'coordinates' => [$n % 20, intdiv($n, 20)]]]; + } + $db->insert('gw', $rows); + $this->assertSame(20, $db->getShardInfo('gw')['shards']); + $writes = []; + $this->setStatic('renameFn', function ($from, $to) use (&$writes) { + $writes[] = basename($to); + return rename($from, $to); + }); + try { + $count = function (string $kind) use (&$writes) { + return count(array_filter($writes, function ($f) use ($kind) { return substr($f, -strlen($kind) - 1) === '.' . $kind; })); + }; + $this->assertTrue($db->createFieldIndex('gw', 'c')['success']); + $this->assertSame([20, 1], [$count('fx'), $count('gfx')], 'one file per shard, one global file'); + $writes = []; + $this->assertTrue($db->rebuildFieldIndex('gw', 'c')['success']); + $this->assertSame([20, 1], [$count('fx'), $count('gfx')]); + $writes = []; + $this->assertTrue($db->createSpatialIndex('gw', 'g')['success']); + $this->assertSame([20, 1], [$count('rx'), $count('grx')]); + $writes = []; + $this->assertTrue($db->rebuildSpatialIndex('gw', 'g')['success']); + $this->assertSame([20, 1], [$count('rx'), $count('grx')]); + } finally { + $this->setStatic('renameFn', null); + } + // the global files describe every shard: lookups and boxes are right, nothing is skipped wrongly + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 10]); + $this->assertSame(range(3, 198, 5), array_column($db->find('gw', ['c' => 'c3']), 'key')); + $this->assertSame([], $db->find('gw', ['c' => 'c9'])); + $this->assertSame([0, 1, 20, 21], array_column($db->withinBBox('gw', 'g', 0, 0, 1, 1), 'key')); + $header = $this->secondary($this->gfxPath('gw', 'c'))[0]; + $this->assertSame(array_map('strval', range(0, 19)), array_map('strval', array_keys($header['sh']))); + } + + /** + * @test Checkpoints of many shards in one operation (a reader that replays long tails, the end of a + * write session) write each global file once, when the operation ends, not once per shard + */ + public function testCheckpointsOfManyShardsWriteTheGlobalFileOnce(): void + { + // no automatic compaction: it would replace the data files (and invalidate the indexes) instead + $config = ['shardSize' => 10, 'autoCompactThreshold' => 5]; + $db = $this->db($config); + $rows = []; + for ($n = 0; $n < 120; $n++) { // 12 shards + $rows[] = ['c' => 'c' . ($n % 3)]; + } + $db->insert('gc', $rows); + $this->assertSame(12, $db->getShardInfo('gc')['shards']); + $this->assertTrue($db->createFieldIndex('gc', 'c')['success']); + // A writer that never loads the index (no checkpoint): more than 64 KiB of new lines in every shard + \noneDB::clearStaticCache(); + $pad = str_repeat('x', 7000); + $this->assertSame(['n' => 120], $this->db($config)->update('gc', [[], ['set' => ['pad' => $pad]]])); + $writes = []; + $this->setStatic('renameFn', function ($from, $to) use (&$writes) { + $writes[] = basename($to); + return rename($from, $to); + }); + $count = function (string $kind) use (&$writes) { + return count(array_filter($writes, function ($f) use ($kind) { return substr($f, -strlen($kind) - 1) === '.' . $kind; })); + }; + try { + // A reader in a new process state loads every shard's index and replays its long tail + \noneDB::clearStaticCache(); + $reader = $this->db($config); + $this->assertSame(range(1, 118, 3), array_column($reader->find('gc', ['c' => 'c1']), 'key')); + $this->assertSame([12, 1], [$count('fx'), $count('gfx')], 'reader: a checkpoint per shard, the global file once'); + + // A write session that touches every shard of an index this process has loaded + $writes = []; + $this->assertSame(['n' => 120], $reader->update('gc', [[], ['set' => ['pad' => $pad . 'y']]])); + $this->assertSame([12, 1], [$count('fx'), $count('gfx')], 'writer: a checkpoint per shard, the global file once'); + } finally { + $this->setStatic('renameFn', null); + } + // Every shard has its entry, at the bytes its checkpoint covers + $header = $this->secondary($this->gfxPath('gc', 'c'))[0]; + $this->assertCount(12, $header['sh']); + foreach ($header['sh'] as $shard => $entry) { + $this->assertSame($this->secondary($this->fxPath('gc', 'c', (int)$shard))[0]['s'], $entry['s'], 'shard ' . $shard); + } + \noneDB::clearStaticCache(); + $this->assertSame(range(2, 119, 3), array_column($this->db($config)->find('gc', ['c' => 'c2']), 'key')); + $this->assertSame([], $this->db($config)->find('gc', ['c' => 'c9'])); + } + + /** first(), exists() and count() of a database that doesn't exist: nothing, and no file */ + public function testFirstExistsAndCountOfAMissingDatabase(): void + { + foreach ([true, false] as $autoCreate) { + $db = $this->db(['shardingEnabled' => false, 'autoCreateDB' => $autoCreate]); + $this->assertNull($db->first('void', ['c' => 1])); + $this->assertFalse($db->exists('void', ['c' => 1])); + $this->assertSame(0, $db->count('void', ['c' => 1])); + $this->assertSame($autoCreate ? [] : false, $db->find('void', ['c' => 1]), 'API.md: false without autoCreateDB'); + $this->assertSame([], $this->dbFiles('void')); + } + } +} diff --git a/tests/Integration/SpecOperationsTest.php b/tests/Integration/SpecOperationsTest.php new file mode 100644 index 0000000..ac74f8a --- /dev/null +++ b/tests/Integration/SpecOperationsTest.php @@ -0,0 +1,1948 @@ +hadScript = array_key_exists('SCRIPT_FILENAME', $_SERVER); + $this->savedScript = $_SERVER['SCRIPT_FILENAME'] ?? null; + $this->savedCwd = getcwd(); + $this->savedDevEnv = getenv('NONEDB_DEV_MODE'); + } + + protected function tearDown(): void + { + foreach ($this->holders as [$proc, $pipes]) { + foreach ($pipes as $pipe) { + @fclose($pipe); + } + @proc_terminate($proc); + @proc_close($proc); + } + $this->holders = []; + if ($this->hadScript) { + $_SERVER['SCRIPT_FILENAME'] = $this->savedScript; + } else { + unset($_SERVER['SCRIPT_FILENAME']); + } + if ($this->savedCwd !== false && $this->savedCwd !== null) { + chdir($this->savedCwd); + } + putenv($this->savedDevEnv === false ? 'NONEDB_DEV_MODE' : 'NONEDB_DEV_MODE=' . $this->savedDevEnv); + foreach (['syncFn', 'flushFn', 'renameFn'] as $name) { + $this->setStatic($name, null); + } + \noneDB::clearConfigCache(); + parent::tearDown(); + foreach ($this->scratchDirs as $dir) { + $this->removeDirectoryTree($dir); + } + $this->scratchDirs = []; + } + + // ========================================== + // helpers + // ========================================== + + private function staticValue(string $name) + { + $property = new \noneDB\Tests\InternalValue($name); + return $property->getValue(); + } + + private function setStatic(string $name, $value): void + { + $property = new \noneDB\Tests\InternalValue($name); + $property->setValue(null, $value); + } + + private function prop(\noneDB $db, string $name) + { + $property = new \noneDB\Tests\InternalValue($name); + return $property->getValue($db); + } + + private function call(?\noneDB $db, string $method, array $args = []) + { + $reflection = new \noneDB\Tests\InternalMethod($method); + return $reflection->invokeArgs($db, $args); + } + + /** Run $fn inside withLock() of $db (exclusive or shared) */ + private function locked(\noneDB $db, string $name, bool $exclusive, callable $fn) + { + return $this->call($db, 'withLock', [$name, $exclusive, $fn]); + } + + private function scratch(string $name): string + { + $dir = sys_get_temp_dir() . '/nonedb_spec_' . $name . '_' . bin2hex(random_bytes(4)); + mkdir($dir, 0777, true); + $dir = realpath($dir); + $this->scratchDirs[] = $dir; + return $dir; + } + + /** Make $dir the entry script's directory (first place a .nonedb file is looked for) */ + private function entryScriptIn(string $dir, string $file = 'entry.php', bool $create = true): void + { + if ($create) { + file_put_contents($dir . '/' . $file, "dbPath($name, 'lock')); + } + + private function epochOf(string $name): ?string + { + $first = explode(' ', trim($this->lockRecord($name)))[0]; + $parts = explode(':', $first, 2); + return $parts[1] ?? null; + } + + /** + * Start a child process that takes flock($mode) on $file and holds it for $seconds; returns once it + * holds the lock + */ + private function holdLock(string $file, int $mode, float $seconds): void + { + $script = $this->childScript('$fp = fopen(' . var_export($file, true) . ', "c+"); flock($fp, ' . $mode . ');' + . ' echo "locked\n"; fflush(STDOUT); usleep(' . (int)($seconds * 1000000) . '); flock($fp, LOCK_UN); echo "released\n";'); + $proc = proc_open([PHP_BINARY, $script], [1 => ['pipe', 'w']], $pipes); // no shell: proc_terminate() reaches PHP + $this->holders[] = [$proc, $pipes]; + $this->assertSame('locked', trim((string)fgets($pipes[1])), 'the child holds the lock'); + } + + /** Directory of a database in a dbDir under a key: {dbDir}{locator}.ndb4/ (STORAGE_FORMAT.md 1, 4) */ + private function homeOf(string $name, ?string $dir = null, string $key = self::SECRET): string + { + return rtrim($dir ?? $this->testDbDir, '/') . '/' . $this->locatorOf($name, $key) . '.ndb4/'; + } + + /** Data file of a single-layout database in a dbDir under a key (generation from CURRENT) */ + private function mainDataFile(string $name, ?string $dir = null, string $key = self::SECRET): string + { + $home = $this->homeOf($name, $dir, $key); + $current = json_decode((string)@file_get_contents($home . 'CURRENT'), true); + return $home . 'generations/' . (is_int($current['generation'] ?? null) ? $current['generation'] : 1) . '/segments/main.jsonl'; + } + + /** Files of a database (paths relative to its directory), LOCK excluded unless asked for */ + private function filesOfDb(string $name, ?string $dir = null, string $key = self::SECRET, bool $withLock = false): array + { + $home = $this->homeOf($name, $dir, $key); + $files = []; + $walk = function (string $path) use (&$walk, &$files, $home, $withLock) { + foreach (@scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($path . $entry)) { + $walk($path . $entry . '/'); + } elseif ($withLock || $path . $entry !== $home . 'LOCK') { + $files[] = substr($path . $entry, strlen($home)); + } + } + }; + $walk($home); + sort($files); + return $files; + } + + /** + * path => sha1 of every file in the test directory, recursively (store files and LOCK files included: their + * records count too); directories are listed as path/ => 'dir' + */ + private function directorySnapshot(?string $dir = null): array + { + $root = rtrim($dir ?? $this->testDbDir, '/') . '/'; + clearstatcache(); + $files = []; + $walk = function (string $path) use (&$walk, &$files, $root) { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($path . $entry)) { + $files[substr($path . $entry, strlen($root)) . '/'] = 'dir'; + $walk($path . $entry . '/'); + } else { + $files[substr($path . $entry, strlen($root))] = sha1_file($path . $entry); + } + } + }; + $walk($root); + ksort($files); + return $files; + } + + /** chmod every directory of a tree (files keep their mode) */ + private function chmodDirectories(string $dir, int $mode): void + { + $dir = rtrim($dir, '/'); + foreach (scandir($dir) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..' && is_dir($dir . '/' . $entry) && !is_link($dir . '/' . $entry)) { + $this->chmodDirectories($dir . '/' . $entry, $mode); + } + } + chmod($dir, $mode); + } + + /** True when the current user can't write to a directory made read-only (not root) */ + private function permissionsAreEnforced(string $dir): bool + { + chmod($dir, 0555); + $probe = $dir . '/probe-' . bin2hex(random_bytes(3)); + $writable = @file_put_contents($probe, 'x') !== false; + if ($writable) { + @unlink($probe); + } + chmod($dir, 0777); + return !$writable; + } + + // ========================================== + // CONFIGURATION: every key, valid and invalid values + // ========================================== + + /** + * @test docs/CONFIGURATION.md "Configuration Options": boolean settings accept true/false, 1/0, "1"/"0" + * and "true"/"false" in any case; anything else throws InvalidArgumentException + */ + public function booleanSettingsAcceptExactlyTheDocumentedValues(): void + { + $settings = ['autoCreateDB', 'shardingEnabled', 'bufferEnabled', 'fieldIndexEnabled', 'spatialIndexEnabled']; + $valid = [[true, true], [false, false], [1, true], [0, false], ['1', true], ['0', false], + ['true', true], ['false', false], ['TRUE', true], ['False', false], ['tRuE', true], ['FALSE', false]]; + // "anything else ("yes", "off", 2) is an error" + $invalid = ['yes', 'no', 'on', 'off', 2, -1, 1.0, 0.0, '', 'truth', '2', [], [true]]; + foreach ($settings as $setting) { + foreach ($valid as [$value, $expected]) { + $db = $this->db([$setting => $value]); + $this->assertSame($expected, $this->prop($db, $setting), $setting . ' = ' . var_export($value, true)); + } + foreach ($invalid as $value) { + try { + $this->db([$setting => $value]); + $this->fail($setting . ' = ' . var_export($value, true) . ' must be rejected'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'{$setting}'", $e->getMessage()); + $this->assertStringContainsString('true or false', $e->getMessage()); + } + } + } + } + + /** + * @test the documented defaults (docs/CONFIGURATION.md tables), read in a process that has no .nonedb + * file anywhere (a developer's local one would otherwise be merged in) + */ + public function defaultsAreTheDocumentedOnes(): void + { + $dir = $this->scratch('defaults'); + copy(__DIR__ . '/../../noneDB.php', $dir . '/noneDB.php'); + file_put_contents($dir . '/run.php', ' "defaults-key-0123456789", "dbDir" => __DIR__ . "/db/"]); + $out = []; + foreach (["autoCreateDB", "shardingEnabled", "shardSize", "autoCompactThreshold", "lockTimeout", + "lockRetryDelay", "bufferEnabled", "fieldIndexEnabled", "spatialIndexEnabled"] as $name) { + $out[$name] = noneDB\Tests\Support\Testing::getOf($db, $name); + } + $out["durability"] = $db->getDurability()["mode"]; + $out["cacheMemoryLimit"] = noneDB\Tests\Support\Testing::get("cacheMemoryLimit"); + echo json_encode($out);'); + $out = trim((string)shell_exec('cd ' . escapeshellarg($dir) . ' && env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1')); + $this->assertSame([ + 'autoCreateDB' => true, // "autoCreateDB | bool | true" + 'shardingEnabled' => true, // "shardingEnabled | bool | true" + 'shardSize' => 10000, // "shardSize | int | 10000" + 'autoCompactThreshold' => 0.3, // "autoCompactThreshold | float | 0.3" + 'lockTimeout' => 5, // "lockTimeout | float | 5" + 'lockRetryDelay' => 10000, // "lockRetryDelay | int | 10000" + 'bufferEnabled' => false, // "bufferEnabled | bool | false" + 'fieldIndexEnabled' => true, // "fieldIndexEnabled | bool | true" + 'spatialIndexEnabled' => true, // "spatialIndexEnabled | bool | true" + 'durability' => 'performance', // "durability | string | 'performance'" + 'cacheMemoryLimit' => null, // "'auto'" (automatic: null internally, see cacheLimitBytes) + ], json_decode($out, true), $out); + } + + /** + * @test numeric settings: valid values are taken (numeric strings too), out-of-range values throw + * (docs/CONFIGURATION.md: shardSize >= 1, autoCompactThreshold > 0, lockTimeout 0 or positive with + * fractions, NAN/INF/negative throw, lockRetryDelay > 0, cacheMemoryLimit non-negative int or 'auto') + */ + public function numericSettingsAreValidated(): void + { + $cases = [ + // setting => [property, [[value, stored]], [invalid values]] + 'shardSize' => ['shardSize', [[1, 1], [5000, 5000], ['5000', 5000]], [0, -1, '0', 'abc', '']], + 'lockRetryDelay' => ['lockRetryDelay', [[1, 1], [20000, 20000], ['20000', 20000]], [0, -5, 'abc', '']], + 'autoCompactThreshold' => ['autoCompactThreshold', [[0.5, 0.5], ['0.25', 0.25], [1, 1.0], [2.5, 2.5]], [0, 0.0, -0.1, '0', 'abc', '']], + 'lockTimeout' => ['lockTimeout', [[0, 0.0], [0.5, 0.5], ['1.5', 1.5], [10, 10.0]], + [-1, -0.001, NAN, INF, -INF, 'abc', '', [1], true, false]], + ]; + foreach ($cases as $setting => [$property, $valid, $invalid]) { + foreach ($valid as [$value, $stored]) { + $this->assertSame($stored, $this->prop($this->db([$setting => $value]), $property), $setting . ' = ' . var_export($value, true)); + } + foreach ($invalid as $value) { + try { + $this->db([$setting => $value]); + $this->fail($setting . ' = ' . var_export($value, true) . ' must be rejected'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'{$setting}'", $e->getMessage()); + } + } + } + + // cacheMemoryLimit: process-wide, read from the static setting (null = 'auto') + foreach ([[0, 0], [1048576, 1048576], ['2048', 2048], ['auto', null]] as [$value, $stored]) { + $this->db(['cacheMemoryLimit' => $value]); + $this->assertSame($stored, $this->staticValue('cacheMemoryLimit'), var_export($value, true)); + } + foreach ([-1, 1.5, '1.5', 'abc', '', 'AUTO', '-5', true, [1]] as $value) { + try { + $this->db(['cacheMemoryLimit' => $value]); + $this->fail('cacheMemoryLimit = ' . var_export($value, true) . ' must be rejected'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'cacheMemoryLimit'", $e->getMessage()); + } + } + } + + /** + * @test docs/CONFIGURATION.md: "Values are validated when the instance is created: an invalid value throws + * InvalidArgumentException." Non-numbers (arrays, booleans, strings with trailing garbage) and NAN are no + * valid shard size, retry delay or compaction threshold; they must not be cast silently (an array or true + * would mean one record per shard) + */ + public function numericSettingsRejectNonNumericValues(): void + { + $accepted = []; + foreach ([ + ['shardSize', [5]], ['shardSize', true], ['shardSize', '10abc'], + ['lockRetryDelay', [1]], ['lockRetryDelay', true], ['lockRetryDelay', '5abc'], + ['autoCompactThreshold', [0.5]], ['autoCompactThreshold', true], ['autoCompactThreshold', '0.5abc'], ['autoCompactThreshold', NAN], + ] as [$setting, $value]) { + try { + $this->db([$setting => $value]); + $accepted[] = $setting . ' = ' . str_replace("\n", '', var_export($value, true)); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'{$setting}'", $e->getMessage()); + } + } + $this->assertSame([], $accepted, 'invalid values were accepted'); + } + + /** + * @test the key releases before 3.0 shipped with is as public as a placeholder: refused (docs/MIGRATION.md §2: + * their databases are migrated with NONEDB_SOURCE_SECRET=nonedb_123 and a key of one's own) + */ + public function theKeyReleasesBefore30ShippedWithIsRejected(): void + { + try { + new \noneDB(['secretKey' => 'nonedb_123', 'dbDir' => $this->testDbDir]); + $this->fail('nonedb_123 accepted'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString('publicly known', $e->getMessage()); + } + } + + /** + * @test every placeholder key shown in the documentation is rejected (CLAUDE.md "placeholders like + * 'your_key' are rejected"; .nonedb.example, README.md, docs/CONFIGURATION.md lines 33 and 681) + */ + public function documentedPlaceholderKeysAreRejected(): void + { + foreach (['CHANGE_THIS_TO_A_SECURE_RANDOM_STRING', 'YOUR_SECURE_RANDOM_STRING', 'YOUR_SECURE_RANDOM_STRING_HERE', + 'GENERATE_UNIQUE_64_CHAR_STRING', 'your_key'] as $placeholder) { + try { + new \noneDB(['secretKey' => $placeholder, 'dbDir' => $this->testDbDir]); + $this->fail($placeholder . ' accepted'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString('placeholder', $e->getMessage(), $placeholder); + } + // migrateKey() never moves data to a placeholder key either (docs/CONFIGURATION.md "migrateKey") + $result = \noneDB::migrateKey($this->testDbDir, 'some-old-key', $placeholder); + $this->assertFalse($result['success'], $placeholder); + $this->assertNotEmpty($result['error'], $placeholder); + } + // The template's key is one of them: "replace the placeholder secretKey (it is rejected)" + $template = json_decode((string)file_get_contents((string)\noneDB::getConfigTemplate()), true); + $this->expectException(\InvalidArgumentException::class); + new \noneDB(['dbDir' => $this->testDbDir] + $template); + } + + // ========================================== + // CONFIGURATION FILE (.nonedb) in this process: the entry script's directory is looked at first + // ========================================== + + /** + * @test docs/CONFIGURATION.md "Where .nonedb is looked for" (1: the running script's directory), "Relative + * dbDir" (relative to the .nonedb file), "Mixed Configuration" (array merged over the file) and "The first + * file found is used for the whole process" + */ + public function configFileNextToTheEntryScriptIsUsedOncePerProcess(): void + { + $app = $this->scratch('cfg'); + file_put_contents($app . '/.nonedb', json_encode(['secretKey' => 'file-key-0123456789', 'dbDir' => 'data', + 'shardSize' => 7, 'bufferEnabled' => 'true', 'lockTimeout' => 2])); + $this->entryScriptIn($app); + $this->assertTrue(\noneDB::configExists()); + + $db = new \noneDB(); + $this->assertSame($app . '/data/', $db->__debugInfo()['dbDir'], 'relative to the .nonedb file'); + $this->assertSame('***', $db->__debugInfo()['secretKey']); + $this->assertFalse($db->__debugInfo()['devMode']); + $this->assertSame(7, $db->getShardSize()); + $this->assertTrue($db->isBufferingEnabled()); + $this->assertSame(2.0, $this->prop($db, 'lockTimeout')); + $db->insert('fromfile', ['v' => 1]); + // directory = {locator(name, secretKey of the file)}.ndb4 (STORAGE_FORMAT.md 1) + $this->assertFileExists($this->mainDataFile('fromfile', $app . '/data/', 'file-key-0123456789')); + + // The array is merged over the file: given keys win, the rest comes from the file + $merged = new \noneDB(['shardSize' => 9, 'bufferEnabled' => false]); + $this->assertSame(9, $merged->getShardSize()); + $this->assertFalse($merged->isBufferingEnabled()); + $this->assertSame($app . '/data/', $merged->__debugInfo()['dbDir']); + $this->assertSame(2.0, $this->prop($merged, 'lockTimeout')); + $this->assertSame([1], array_column($merged->find('fromfile', 0), 'v'), 'same key and directory as the file'); + + // Loaded once per process: a changed file is seen only after clearConfigCache() + file_put_contents($app . '/.nonedb', json_encode(['secretKey' => 'file-key-0123456789', 'dbDir' => 'other', 'shardSize' => 11])); + $this->assertSame(7, (new \noneDB())->getShardSize()); + \noneDB::clearConfigCache(); + $reloaded = new \noneDB(); + $this->assertSame(11, $reloaded->getShardSize()); + $this->assertSame($app . '/other/', $reloaded->__debugInfo()['dbDir']); + } + + /** @test a .nonedb that isn't a JSON object can't configure anything (unless the array gives secretKey and dbDir) */ + public function invalidConfigFileIsReportedUnlessTheArrayIsComplete(): void + { + foreach (['{"secretKey": "abc", ', '"just a string"', '42', 'null'] as $content) { + $app = $this->scratch('badjson'); + file_put_contents($app . '/.nonedb', $content); + $this->entryScriptIn($app); + try { + new \noneDB(['shardSize' => 3]); + $this->fail('accepted ' . $content); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('Invalid JSON in config file', $e->getMessage(), $content); + $this->assertStringContainsString($app . '/.nonedb', $e->getMessage(), 'names the file'); + } + // Complete array: the broken file is ignored entirely (its values can't be merged) + \noneDB::clearConfigCache(); + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir]); + $this->assertSame(10000, $db->getShardSize()); + $this->assertSame($this->testDbDir, $db->__debugInfo()['dbDir']); + } + } + + /** @test an unreadable .nonedb is an error naming the file, unless the array is complete */ + public function unreadableConfigFileIsReportedUnlessTheArrayIsComplete(): void + { + $app = $this->scratch('unreadable'); + file_put_contents($app . '/.nonedb', json_encode(['secretKey' => 'unreadable-key-123', 'dbDir' => 'data'])); + chmod($app . '/.nonedb', 0000); + if (is_readable($app . '/.nonedb')) { + chmod($app . '/.nonedb', 0600); + $this->markTestSkipped('this user (root) can read a file with mode 0000'); + } + $this->entryScriptIn($app); + try { + new \noneDB(); + $this->fail('an unreadable config file must not be ignored'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('Cannot read config file', $e->getMessage()); + $this->assertStringContainsString($app . '/.nonedb', $e->getMessage()); + } + \noneDB::clearConfigCache(); + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir]); + $this->assertSame($this->testDbDir, $db->__debugInfo()['dbDir']); + chmod($app . '/.nonedb', 0600); + } + + /** @test a config file without dbDir (and no dbDir in the array) can't be used: "dbDir | string | required" */ + public function configFileWithoutDbDirIsRejected(): void + { + $app = $this->scratch('nodir'); + file_put_contents($app . '/.nonedb', json_encode(['secretKey' => 'key-without-dir-0123'])); + $this->entryScriptIn($app); + try { + new \noneDB(); + $this->fail('no dbDir configured'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString("'dbDir' is not configured", $e->getMessage()); + } + // The array can add it + $db = new \noneDB(['dbDir' => $this->testDbDir]); + $this->assertSame($this->testDbDir, $db->__debugInfo()['dbDir']); + $db->insert('nodir', ['v' => 1]); + $this->assertFileExists($this->mainDataFile('nodir', null, 'key-without-dir-0123')); + + // A secretKey given in the array is used as given, never replaced by the file's (S-04): null is rejected + try { + new \noneDB(['secretKey' => null, 'dbDir' => $this->testDbDir]); + $this->fail('a null secretKey in the array must not fall back to the file'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'secretKey' must be a non-empty string", $e->getMessage()); + } + } + + /** + * @test docs/CONFIGURATION.md "Relative dbDir": in a config array, a relative dbDir (with or without ./, + * also ../) is relative to the entry script's directory, used by its canonical path (STORAGE_FORMAT.md §3); + * a trailing slash is added + */ + public function relativeDbDirOfAnArrayFollowsTheEntryScript(): void + { + $root = $this->scratch('rel'); + mkdir($root . '/app'); + mkdir($root . '/cwd'); + $this->entryScriptIn($root . '/app'); + chdir($root . '/cwd'); + foreach (['store' => $root . '/app/store/', './store/' => $root . '/app/store/', '../shared' => $root . '/shared/'] as $dbDir => $expected) { + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dbDir]); + $this->assertSame($expected, $db->__debugInfo()['dbDir'], $dbDir); + } + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => '../shared']); + $db->insert('rel', ['v' => 1]); + $this->assertFileExists($this->mainDataFile('rel', $root . '/shared/')); + $this->assertDirectoryDoesNotExist($root . '/cwd/shared', 'not relative to the working directory'); + + // An absolute directory without trailing slash gets one + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => rtrim($this->testDbDir, '/')]); + $this->assertSame($this->testDbDir, $db->__debugInfo()['dbDir']); + + // Entry script that doesn't exist (any more): its directory still counts; no entry script: the working directory + $this->entryScriptIn($root . '/app', 'gone.php', false); + $this->assertSame($root . '/app/x/', (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); + $_SERVER['SCRIPT_FILENAME'] = ''; + $this->assertSame(getcwd() . '/x/', (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); + + // Many different entry scripts in one process (a long-running worker): each resolves on its own + for ($i = 0; $i < 20; $i++) { + mkdir($root . "/w$i"); + $this->entryScriptIn($root . "/w$i"); + $this->assertSame($root . "/w$i/db/", (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'db']))->__debugInfo()['dbDir']); + } + } + + /** + * @test docs/CONFIGURATION.md "Check Dev Mode Status": isDevMode() is true when NONEDB_DEV_MODE is set + * ("=1", "=true"), "even if a .nonedb file is found and dev mode is therefore not used" + * @runInSeparateProcess + * @preserveGlobalState disabled + */ + public function isDevModeFollowsTheEnvironmentVariable(): void + { + $this->assertFalse(defined('NONEDB_DEV_MODE'), 'a fresh process'); + putenv('NONEDB_DEV_MODE'); + $this->assertFalse(\noneDB::isDevMode()); + foreach (['1' => true, 'true' => true, 'yes' => false, '0' => false, 'false' => false, '' => false] as $value => $expected) { + putenv('NONEDB_DEV_MODE=' . $value); + $this->assertSame($expected, \noneDB::isDevMode(), 'NONEDB_DEV_MODE=' . $value); + } + // A complete array never uses dev mode, whatever the environment says + putenv('NONEDB_DEV_MODE=1'); + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir]); + $this->assertFalse($db->__debugInfo()['devMode']); + putenv('NONEDB_DEV_MODE'); + \noneDB::setDevMode(false); + $this->assertFalse(defined('NONEDB_DEV_MODE'), 'setDevMode(false) defines nothing'); + \noneDB::setDevMode(true); + $this->assertTrue(NONEDB_DEV_MODE); + $this->assertTrue(\noneDB::isDevMode()); + } + + /** + * @test docs/CONFIGURATION.md "Development Mode": used only when no .nonedb is found and the array doesn't + * give both secretKey and dbDir; it fills in what is missing: secretKey "nonedb_dev_mode_key_" . + * md5(), dbDir "db/" next to noneDB.php (no Composer). Enabled by + * NONEDB_DEV_MODE=1/true or the constant (which "must be the boolean true"). + */ + public function devModeFillsInWhatIsMissing(): void + { + $lib = $this->scratch('devlib'); + $cwd = $this->scratch('devcwd'); + copy(__DIR__ . '/../../noneDB.php', $lib . '/noneDB.php'); + $own = $this->scratch('devown'); + $script = ' null, "dir" => ["dbDir" => OWNDIR], "key" => ["secretKey" => "own-dev-key-123"], + "both" => ["secretKey" => "own-dev-key-123", "dbDir" => OWNDIR]] as $label => $config) { + if ($label === "key") { + // The stores "none" and "dir" created belong to the dev key: the own key is registered as a + // second key of both (STORAGE_FORMAT.md 3.2, 9.3) + foreach ([__DIR__ . "/db/", OWNDIR] as $store) { + $r = noneDB::registerKey($store, DEVKEY, "own-dev-key-123"); + if (!$r["success"]) { echo "registerKey: ", $r["error"], "\n"; } + } + } + try { + $db = new noneDB($config); + $db->insert("t" . $label, ["v" => 1]); + $out[$label] = $db->__debugInfo()["devMode"] ? "dev" : "nodev"; + $out[$label] .= " " . $db->__debugInfo()["dbDir"]; + } catch (Throwable $e) { + $out[$label] = get_class($e); + } + } + echo json_encode($out);'; + $devKey = 'nonedb_dev_mode_key_' . md5($lib); + $script = strtr($script, ['DEVKEY' => var_export($devKey, true)]); + $runs = [ + 'constant true' => ['define("NONEDB_DEV_MODE", true);', ''], + 'env 1' => ['', 'NONEDB_DEV_MODE=1'], + 'env true' => ['', 'NONEDB_DEV_MODE=true'], + ]; + foreach ($runs as $label => [$prefix, $env]) { + $this->removeDirectoryTree($lib . '/db'); + $this->removeDirectoryTree($own); + file_put_contents($lib . '/run.php', strtr($script, ['PREFIX' => $prefix, 'OWNDIR' => var_export($own . '/', true)])); + $out = trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($lib . '/run.php') . ' 2>&1')); + $this->assertSame([ + 'none' => 'dev ' . $lib . '/db/', + 'dir' => 'dev ' . $own . '/', + 'key' => 'dev ' . $lib . '/db/', + 'both' => 'nodev ' . $own . '/', + ], json_decode($out, true), $label . ': ' . $out); + // The keys: the predictable dev key where none was given, the given one otherwise + $this->assertFileExists($this->mainDataFile('tnone', $lib . '/db/', $devKey), $label); + $this->assertFileExists($this->mainDataFile('tdir', $own . '/', $devKey), $label); + $this->assertFileExists($this->mainDataFile('tkey', $lib . '/db/', 'own-dev-key-123'), $label); + $this->assertFileExists($this->mainDataFile('tboth', $own . '/', 'own-dev-key-123'), $label); + $this->assertDirectoryDoesNotExist($cwd . '/db', 'not in the working directory'); + } + + // Not dev mode: NONEDB_DEV_MODE=yes, the constant 1 (not the boolean true), setDevMode(false) + foreach (['env yes' => ['', 'NONEDB_DEV_MODE=yes'], 'constant 1' => ['define("NONEDB_DEV_MODE", 1);', ''], + 'setDevMode(false)' => ['', ''], ] as $label => [$prefix, $env]) { + $body = $label === 'setDevMode(false)' ? 'noneDB::setDevMode(false); ' : ''; + file_put_contents($lib . '/run.php', 'getMessage(), "\n"); }'); + $out = trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($lib . '/run.php') . ' 2>&1')); + $this->assertSame('RuntimeException: noneDB: Configuration file not found!', $out, $label); + } + } + + /** + * @test docs/CONFIGURATION.md "Where .nonedb is looked for": the script's directory, noneDB.php's directory, + * the Composer project root (installed under vendor/), the working directory; "The first file found is + * used". Dev mode's dbDir is "db/ in the project root (next to vendor/ when installed with Composer)". + */ + public function configSearchOrderAndComposerProjectRoot(): void + { + $proj = $this->scratch('composer'); + $vendor = $proj . '/vendor/orhanayd/nonedb'; + mkdir($vendor, 0777, true); + mkdir($proj . '/public'); + $cwd = $this->scratch('composercwd'); + copy(__DIR__ . '/../../noneDB.php', $vendor . '/noneDB.php'); + file_put_contents($proj . '/public/index.php', '__debugInfo()["dbDir"], " ", $db->__debugInfo()["devMode"] ? "dev" : "file"; } + catch (Throwable $e) { echo get_class($e); }'); + $run = function (string $env = '') use ($proj, $cwd) { + return trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($proj . '/public/index.php') . ' 2>&1')); + }; + $write = function (string $dir, string $dbDir) { + file_put_contents($dir . '/.nonedb', json_encode(['secretKey' => 'composer-key-0123456789', 'dbDir' => $dbDir])); + }; + + $this->assertSame('RuntimeException', $run(), 'no file anywhere'); + $this->assertSame($proj . '/db/ dev', $run('DEV=1'), 'dev mode: db/ next to vendor/'); + + $write($cwd, 'from-cwd'); // 4. the working directory + $this->assertSame($cwd . '/from-cwd/ file', $run()); + $write($proj, 'from-root'); // 3. the Composer project root + $this->assertSame($proj . '/from-root/ file', $run()); + $write($vendor, 'from-lib'); // 2. noneDB.php's directory + $this->assertSame($vendor . '/from-lib/ file', $run()); + $write($proj . '/public', 'from-script'); // 1. the entry script's directory + $this->assertSame($proj . '/public/from-script/ file', $run()); + $this->assertSame($proj . '/public/from-script/ file', $run('DEV=1'), 'a file found: dev mode is not used'); + } + + /** @test the template is the .nonedb.example next to noneDB.php, null when it isn't there (docs "Configuration Helpers") */ + public function configTemplateIsTheExampleNextToNoneDb(): void + { + $this->assertSame(realpath(dirname(NONEDB_TEST_LIBRARY) . '/.nonedb.example'), \noneDB::getConfigTemplate()); // next to the library that runs + $lib = $this->scratch('notemplate'); + copy(__DIR__ . '/../../noneDB.php', $lib . '/noneDB.php'); + $out = trim((string)shell_exec(escapeshellarg(PHP_BINARY) . ' -r ' . escapeshellarg('require ' . var_export($lib . '/noneDB.php', true) + . '; var_export(noneDB::getConfigTemplate());') . ' 2>&1')); + $this->assertSame('NULL', $out); + } + + /** @test noneDB instances can't be unserialized either (they hold a secret; S-09) */ + public function instancesCannotBeUnserialized(): void + { + $this->expectException(\LogicException::class); + unserialize('O:6:"noneDB":0:{}'); + } + + // ========================================== + // LOCKS: lockTimeout, LOCK_FAILED, lost locks, snapshots + // ========================================== + + /** @test docs/CONFIGURATION.md "lockTimeout": "0 waits forever" */ + public function lockTimeoutZeroWaitsUntilTheLockIsFree(): void + { + $db = $this->db(['lockTimeout' => 0, 'shardingEnabled' => false]); + $db->insert('forever', ['v' => 1]); + $this->holdLock($this->dbPath('forever', 'lock'), LOCK_EX, 0.6); + $start = microtime(true); + $this->assertSame([1], array_column($db->find('forever', 0), 'v')); + $this->assertGreaterThan(0.3, microtime(true) - $start, 'it waited for the other process instead of failing'); + $db->insert('forever', ['v' => 2]); + $this->assertSame([1, 2], array_column($db->find('forever', 0), 'v')); + } + + /** + * @test docs/CONFIGURATION.md "lockTimeout": "Seconds to wait for a database lock before throwing + * RuntimeException (code noneDB::LOCK_FAILED)"; "noneDB never continues an operation without its lock" + */ + public function lockTimeoutThrowsLockFailedForReadsAndWrites(): void + { + $db = $this->db(['lockTimeout' => 0.3, 'shardingEnabled' => false]); + $db->insert('busy', ['v' => 1]); + $before = $this->directorySnapshot(); + $this->holdLock($this->dbPath('busy', 'lock'), LOCK_EX, 5); + foreach (['find' => function () use ($db) { $db->find('busy', 0); }, + 'insert' => function () use ($db) { $db->insert('busy', ['v' => 2]); }, + 'count' => function () use ($db) { $db->count('busy'); }] as $label => $operation) { + $start = microtime(true); + try { + $operation(); + $this->fail($label . ' ran without the lock'); + } catch (\RuntimeException $e) { + $elapsed = microtime(true) - $start; + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode(), $label); + $this->assertStringContainsString('timed out', $e->getMessage(), $label); + $this->assertGreaterThanOrEqual(0.25, $elapsed, $label . ': waited lockTimeout'); + $this->assertLessThan(3.0, $elapsed, $label . ': and not much longer'); + } + } + $this->assertSame($before, $this->directorySnapshot(), 'nothing was written'); + } + + /** + * @test a shared lock is upgraded by releasing it first (CHANGES "Locking"); when the exclusive lock can't + * be taken within lockTimeout the lock is lost: LOCK_FAILED, and every later step of the same operation + * fails too instead of reading without a lock + */ + public function anUpgradeThatTimesOutLosesTheLock(): void + { + $db = $this->db(['lockTimeout' => 0.3, 'shardingEnabled' => false]); + $db->insert('up', ['v' => 1]); + $lock = $this->dbPath('up', 'lock'); + $steps = $this->locked($db, 'up', false, function () use ($db, $lock) { + $this->holdLock($lock, LOCK_SH, 5); // another reader + $steps = []; + foreach (['insert' => function () use ($db) { $db->insert('up', ['v' => 2]); }, + 'find' => function () use ($db) { $db->find('up', 0); }] as $label => $operation) { + try { + $operation(); + $steps[] = $label . ' ran'; + } catch (\RuntimeException $e) { + $steps[] = $label . ' ' . $e->getCode() . (strpos($e->getMessage(), 'lost') !== false ? ' lost' : ''); + } + } + return $steps; + }); + $this->assertSame(['insert ' . \noneDB::LOCK_FAILED, 'find ' . \noneDB::LOCK_FAILED . ' lost'], $steps); + foreach ($this->holders as [$proc, $pipes]) { + proc_terminate($proc); + } + // The operation ended: the database works again, the failed insert wrote nothing + $this->assertSame([1], array_column($db->find('up', 0), 'v')); + $db->insert('up', ['v' => 3]); + $this->assertSame([1, 3], array_column($db->find('up', 0), 'v')); + } + + /** + * @test CHANGES "Locking": "A lock that couldn't be taken throws RuntimeException with code + * noneDB::LOCK_FAILED. Fallback paths never continue without the lock." Here the lock file can't be opened. + */ + public function aLockFileThatCannotBeOpenedFailsWithLockFailed(): void + { + $db = $this->db(); + $db->insert('first', ['v' => 1]); // the store exists (STORAGE_FORMAT.md 3.2) + mkdir($this->dbPath('nolock', 'lock'), 0777, true); // a directory where the lock file should be + try { + $db->insert('nolock', ['v' => 1]); + $this->fail('no lock, no write'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString("'nolock'", $e->getMessage()); + $this->assertSame([], $this->filesOfDb('nolock'), 'nothing written'); + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode(), $e->getMessage()); + } finally { + rmdir($this->dbPath('nolock', 'lock')); + } + } + + /** + * @test CLAUDE.md "Joins: withSnapshot() (shared locks of all databases ...), SNAPSHOT_RETRY instead of + * waiting for an exclusive lock": work that needs an exclusive lock while the snapshot is held makes the + * snapshot start again; the join result is the consistent one; the retries end at lockTimeout (LOCK_FAILED) + */ + public function joinSnapshotsRetryInsteadOfWaitingForAnExclusiveLock(): void + { + $db = $this->db(['shardingEnabled' => false, 'lockTimeout' => 0.5]); + $db->insert('orders', [['user' => 0, 'item' => 'a'], ['user' => 1, 'item' => 'b'], ['user' => 7, 'item' => 'c']]); + $db->insert('users', [['name' => 'ann'], ['name' => 'bob']]); + // Expected: each order with the user whose key is its "user", null when there is none (docs/QUERY.md join()) + $users = []; + foreach ($db->find('users', 0) as $user) { + $users[$user['key']] = $user; + } + $expected = array_map(function ($order) use ($users) { + return $order + ['users' => $users[$order['user']] ?? null]; + }, $db->find('orders', 0)); + + $held = 0; + $this->setFaultHook(function ($point) use ($db, &$held) { + if ($point === 'snapshot:held' && ++$held === 1) { + $db->insert('users', ['name' => 'carl']); // needs the exclusive lock: refused while the snapshot holds locks + } + }); + $this->assertSame($expected, $db->query('orders')->join('users', 'user', 'key')->get()); + $this->assertSame(2, $held, 'started once more'); + $this->setFaultHook(null); + $this->assertSame(['ann', 'bob'], array_column($db->find('users', 0), 'name'), 'the refused write wrote nothing'); + + // Other exceptions are not retried + $held = 0; + $this->setFaultHook(function ($point) use (&$held) { + if ($point === 'snapshot:held') { + $held++; + throw new \RuntimeException('boom', 42); + } + }); + try { + $db->query('orders')->join('users', 'user', 'key')->get(); + $this->fail('must throw'); + } catch (\RuntimeException $e) { + $this->assertSame(['boom', 42, 1], [$e->getMessage(), $e->getCode(), $held]); + } + + // Always refused: retried until lockTimeout, then LOCK_FAILED + $this->setFaultHook(function ($point) use ($db) { + if ($point === 'snapshot:held') { + $db->insert('users', ['name' => 'dan']); + } + }); + $start = microtime(true); + try { + $db->query('orders')->join('users', 'user', 'key')->get(); + $this->fail('must give up'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode()); + $this->assertGreaterThanOrEqual(0.45, microtime(true) - $start); + $this->assertNotNull($e->getPrevious()); + } + $this->setFaultHook(null); + $this->assertSame(['ann', 'bob'], array_column($db->find('users', 0), 'name')); + } + + /** @test a join of a database with itself (one lock) */ + public function selfJoinWorks(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('emp', [['name' => 'boss', 'mgr' => null], ['name' => 'ann', 'mgr' => 0], ['name' => 'bob', 'mgr' => 0]]); + $boss = $db->find('emp', ['key' => 0])[0]; + $rows = $db->query('emp')->join('emp', 'mgr', 'key', 'manager')->get(); + $this->assertSame([null, $boss, $boss], array_column($rows, 'manager')); + $this->assertSame(['boss', 'ann', 'bob'], array_column($rows, 'name')); + } + + // ========================================== + // DATABASE NAMES, DIRECTORIES + // ========================================== + + /** + * @test docs/API.md: "Database names may contain A-Z a-z 0-9, space, ' and -; every method throws + * InvalidArgumentException for anything else"; integers are names ("0" is valid) + */ + public function databaseNamesMustBeStringsOrIntegers(): void + { + $db = $this->db(['shardingEnabled' => false]); + $calls = [ + 'createDB(null)' => function () use ($db) { $db->createDB(null); }, + 'createDB(1.5)' => function () use ($db) { $db->createDB(1.5); }, + 'insert([])' => function () use ($db) { $db->insert(['users'], ['v' => 1]); }, + 'find(true)' => function () use ($db) { $db->find(true, 0); }, + 'dropDB(object)' => function () use ($db) { $db->dropDB(new \stdClass()); }, + 'getDBs([])' => function () use ($db) { $db->getDBs(['users']); }, + 'getDBs(null)' => function () use ($db) { $db->getDBs(null); }, + 'checkIntegrity(2.0)' => function () use ($db) { $db->checkIntegrity(2.0); }, + ]; + foreach ($calls as $label => $call) { + try { + $call(); + $this->fail($label . ' accepted'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString('database name must be a string', $e->getMessage(), $label); + } + } + $this->assertSame([], glob($this->testDbDir . '*'), 'nothing created'); + $this->assertTrue($db->createDB(7)); + $this->assertSame(['7'], $db->getDBs(false)); + $this->assertSame(1, $db->insert(7, ['v' => 1])['n']); + $this->assertSame([['v' => 1, 'key' => 0]], $db->find('7', 0)); + } + + /** @test a dbDir that can't be created fails the write with a RuntimeException naming it; reads still create nothing */ + public function uncreatableDbDirFailsWrites(): void + { + $blocker = $this->scratch('blocker') . '/file'; + file_put_contents($blocker, 'a file, not a directory'); + $dir = $blocker . '/db/'; + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dir]); + $this->assertSame([], $db->find('x', 0), 'a read of a missing database'); + try { + $db->insert('x', ['v' => 1]); + $this->fail('the directory can not exist'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('cannot create database directory', $e->getMessage()); + $this->assertStringContainsString($dir, $e->getMessage()); + } + $this->assertSame('a file, not a directory', file_get_contents($blocker)); + } + + // ========================================== + // DATABASE OPERATIONS: dropDB, createDB, getDBs, getShardInfo, convertToSharded, compact, buffers + // ========================================== + + /** + * @test docs/API.md dropDB(): "Delete a database and all of its files (data, shards, indexes, buffers, + * metadata, recovery log, quarantine and orphan files). Only LOCK is kept" (STORAGE_FORMAT.md 8.2). + * Returns the number of deleted files. + */ + public function dropDbRemovesEveryFileButTheLock(): void + { + $config = ['shardSize' => 3, 'bufferEnabled' => false]; + $db = $this->db($config); + $db->createFieldIndex('all', 'c'); + $db->createSpatialIndex('all', 'loc'); + $db->insert('all', array_map(function ($n) { + return ['c' => 'c' . ($n % 2), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n]]]; + }, range(0, 7))); + // A torn tail: the next write quarantines it (quarantine file + recovery log) + file_put_contents($this->dbPath('all', 'data', 2), '{"torn', FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $db->insert('all', ['c' => 'c0']); + // Buffered records, a rotated recovery log, an orphan shard file + $buffered = $this->db(['shardSize' => 3, 'bufferEnabled' => true]); + $buffered->insert('all', ['c' => 'buffered']); + file_put_contents($this->dbPath('all', 'recovery1'), json_encode(['t' => 1, 'action' => 'old']) . "\n"); + $gen = substr($this->dbPath('all', 'gen'), strlen($this->dbPath('all', 'home'))); // generations/{N}/ + file_put_contents($this->dbPath('all', 'gen') . 'segments/s9.jsonl.orphan.20260101000000', "{}\n"); + + $files = $this->filesOfDb('all'); + foreach (['FORMAT', 'CURRENT', $gen . 'metadata', $gen . 'shards.json', $gen . 'indexes.json', $gen . 'buffer/pending.jsonl', + 'recovery/log.jsonl', 'recovery/log.1.jsonl', $gen . 'indexes/c.gfx', $gen . 'spatial/loc.grx', $gen . 'segments/s0.jsonl', + $gen . 'segments/s0.kix', $gen . 'indexes/s0.c.fx', $gen . 'spatial/s0.loc.rx', + $gen . 'segments/s9.jsonl.orphan.20260101000000'] as $file) { + $this->assertContains($file, $files, 'set up: ' . $file); + } + $this->assertCount(1, preg_grep('#^recovery/q\.s2\.\d+\.6\.[0-9a-f]{64}\.bin$#', $files), 'a quarantine file'); + + $this->assertSame(['success' => true, 'files' => count($files)], $db->dropDB('all')); + $this->assertSame([], $this->filesOfDb('all'), 'every file of the database is gone'); + $this->assertSame(['LOCK'], $this->filesOfDb('all', null, self::SECRET, true), 'but LOCK'); + $this->assertSame([], $this->lockState('all')['flags'], 'nothing pending'); + $this->assertSame([], $db->getDBs(false)); + $this->assertSame([], $db->find('all', 0)); + $this->assertSame(['success' => false, 'files' => 0, 'error' => 'Database does not exist'], $db->dropDB('all')); + } + + /** @test docs/API.md dropDB() of a database that doesn't exist; it creates nothing (no LOCK either) */ + public function dropDbOfAMissingDatabase(): void + { + $db = $this->db(); + $this->assertSame(['success' => false, 'files' => 0, 'error' => 'Database does not exist'], $db->dropDB('never')); + $this->assertSame([], glob($this->testDbDir . '*')); + } + + /** @test files that can't be deleted are reported (success false), the database stays usable and the drop can be repeated */ + public function dropDbThatCannotDeleteReportsIt(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ro', [['v' => 1], ['v' => 2]]); + if (!$this->permissionsAreEnforced(rtrim($this->testDbDir, '/'))) { + $this->markTestSkipped('this user (root) can write to read-only directories'); + } + $files = $this->filesOfDb('ro'); + // Every directory of the database read-only: none of its files can be deleted + $this->chmodDirectories($this->dbPath('ro', 'home'), 0555); + try { + $result = $db->dropDB('ro'); + } finally { + $this->chmodDirectories($this->dbPath('ro', 'home'), 0777); + } + $this->assertFalse($result['success']); + $this->assertSame(0, $result['files']); + foreach ($files as $file) { + $this->assertStringContainsString($file, $result['error']); + } + $this->assertSame($files, $this->filesOfDb('ro'), 'nothing deleted'); + $this->assertSame([1, 2], array_column($db->find('ro', 0), 'v')); + $this->assertSame(['success' => true, 'files' => count($files)], $db->dropDB('ro')); + $this->assertSame([], $this->filesOfDb('ro')); + } + + /** @test createDB() that can't create the data file throws and leaves nothing behind */ + public function createDbThatCannotWriteThrows(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->createDB('rw'); + $db->dropDB('rw'); // LOCK and the database directory stay (STORAGE_FORMAT.md 8.2) + if (!$this->permissionsAreEnforced(rtrim($this->testDbDir, '/'))) { + $this->markTestSkipped('this user (root) can write to read-only directories'); + } + $before = $this->filesOfDb('rw'); + chmod($this->dbPath('rw', 'home'), 0555); + try { + $db->createDB('rw'); + $this->fail('the directory is read-only'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString("'rw'", $e->getMessage()); + } finally { + chmod($this->dbPath('rw', 'home'), 0777); + } + $this->assertSame($before, $this->filesOfDb('rw')); + $this->assertSame([], $db->getDBs(false)); + $this->assertTrue($db->createDB('rw')); + $this->assertSame(['rw'], $db->getDBs(false)); + } + + /** + * @test docs/API.md getDBs(): "Only databases of the current secretKey are listed; sharded databases appear + * once, with the total size of their shards. Sizes use a comma as decimal separator" + */ + public function getDbsListsOnlyThisKeysDatabasesOnce(): void + { + $mine = $this->db(['shardSize' => 4]); + $other = new \noneDB(['secretKey' => 'another-key-0123456789', 'dbDir' => $this->testDbDir, 'shardSize' => 4]); + $mine->insert('plain', ['v' => str_repeat('x', 2000)]); + $mine->insert('sharded', array_map(function ($n) { return ['n' => $n, 'pad' => str_repeat('y', 300)]; }, range(0, 9))); + // A second key of the same store is registered first (STORAGE_FORMAT.md 3.2, 9.3) + $this->assertTrue(\noneDB::registerKey($this->testDbDir, self::SECRET, 'another-key-0123456789')['success']); + $other->insert('theirs', ['v' => 1]); + $other->insert('plain', ['v' => 2]); // same name, other key + + $names = $mine->getDBs(false); + sort($names); + $this->assertSame(['plain', 'sharded'], $names); + $this->assertSame(['plain', 'theirs'], (function () use ($other) { $n = $other->getDBs(false); sort($n); return $n; })()); + + // Size of the sharded database: every shard file, converted like docs/API.md says ("2,5 KB") + $bytes = 0; + foreach (glob($this->dbPath('sharded', 'gen') . 'segments/s*.jsonl') as $shard) { + $bytes += filesize($shard); + } + $this->assertGreaterThan(1024, $bytes); + $expectedSize = str_replace('.', ',', (string)round($bytes / 1024, 2)) . ' KB'; + $info = $mine->getDBs('sharded'); + $this->assertSame(['name', 'createdTime', 'size'], array_keys($info)); + $this->assertSame($expectedSize, $info['size']); + $withInfo = $mine->getDBs(true); + usort($withInfo, function ($a, $b) { return strcmp($a['name'], $b['name']); }); + $this->assertSame(['plain', 'sharded'], array_column($withInfo, 'name')); + $this->assertSame($expectedSize, $withInfo[1]['size']); + $this->assertSame(str_replace('.', ',', (string)round(filesize($this->dbPath('plain', 'data')) / 1024, 2)) . ' KB', $withInfo[0]['size']); + + // A directory that doesn't exist lists nothing and is not created + $missing = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir . 'nothing-here/']); + $this->assertSame([], $missing->getDBs(false)); + $this->assertSame([], $missing->getDBs(true)); + $this->assertDirectoryDoesNotExist($this->testDbDir . 'nothing-here'); + } + + /** @test docs/API.md getShardInfo(): exact fields for both forms, false for a missing database */ + public function getShardInfoHasTheDocumentedFields(): void + { + $db = $this->db(['shardSize' => 4]); + $this->assertFalse($db->getShardInfo('none')); + $db->insert('small', [['n' => 1], ['n' => 2], ['n' => 3]]); + $db->delete('small', ['n' => 2]); + // Not sharded: ["sharded" => false, "shards" => 0, "totalRecords" => n, "shardSize" => the size fixed at creation, + // "nextKey" => the key the next insert gets]; keys are never handed out again, a deleted highest key included + $this->assertSame(['sharded' => false, 'shards' => 0, 'totalRecords' => 2, 'shardSize' => 4, 'nextKey' => 3], $db->getShardInfo('small')); + $db->insert('top', [['n' => 1], ['n' => 2], ['n' => 3]]); + $db->delete('top', ['key' => 2]); + $this->assertSame(['sharded' => false, 'shards' => 0, 'totalRecords' => 2, 'shardSize' => 4, 'nextKey' => 3], $db->getShardInfo('top')); + + $db->insert('big', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); // keys 0-9, shards 0-2 + $db->delete('big', ['n' => 5]); + $db->delete('big', ['n' => 6]); + $this->assertSame(['sharded' => true, 'shards' => 3, 'totalRecords' => 8, 'deletedCount' => 2, 'shardSize' => 4, 'nextKey' => 10], + $db->getShardInfo('big')); + // Every database keeps the size it was created with when the setting changes (STORAGE_FORMAT.md §4.1) + $this->assertSame(4, $this->db(['shardSize' => 100])->getShardInfo('big')['shardSize']); + $this->assertSame(4, $this->db(['shardSize' => 100])->getShardInfo('small')['shardSize']); + $this->db(['shardSize' => 100])->insert('small', [['n' => 4], ['n' => 5]]); // key 4: the fifth key reaches 4 + $this->assertSame(['sharded' => true, 'shards' => 2, 'totalRecords' => 4, 'deletedCount' => 0, 'shardSize' => 4, 'nextKey' => 5], + $this->db(['shardSize' => 100])->getShardInfo('small'), 'sharded at its own size, not the setting'); + $this->db(['shardSize' => 100])->insert('fresh', ['n' => 1]); + $this->assertSame(100, $this->db(['shardSize' => 4])->getShardInfo('fresh')['shardSize'], 'a new database takes the setting of its creation'); + } + + /** @test docs/API.md convertToSharded(): "converted", "already_sharded" (success true), "database_not_found" */ + public function convertToShardedReportsEachStatus(): void + { + $db = $this->db(['shardSize' => 3, 'shardingEnabled' => false]); + $this->assertSame(['success' => false, 'status' => 'database_not_found'], $db->convertToSharded('none')); + $db->insert('mg', array_map(function ($n) { return ['n' => $n]; }, range(0, 6))); + $this->assertFalse($db->getShardInfo('mg')['sharded'], 'shardingEnabled off'); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('mg')); + $this->assertSame(['sharded' => true, 'shards' => 3, 'totalRecords' => 7, 'deletedCount' => 0, 'shardSize' => 3, 'nextKey' => 7], + $db->getShardInfo('mg')); + $this->assertSame(range(0, 6), array_column($db->find('mg', 0), 'n')); + $this->assertSame(range(0, 6), array_column($db->find('mg', 0), 'key'), 'keys preserved'); + $this->assertSame(['success' => true, 'status' => 'already_sharded'], $db->convertToSharded('mg')); + + // Records waiting in the buffer are flushed first and switched with the rest + $buffered = $this->db(['shardSize' => 3, 'bufferEnabled' => true]); // the flush itself reaches shardSize + $buffered->insert('mb', array_map(function ($n) { return ['n' => $n]; }, range(0, 4))); + $this->assertSame(['success' => true, 'status' => 'converted'], $buffered->convertToSharded('mb')); + $this->assertSame(range(0, 4), array_column($buffered->find('mb', 0), 'n')); + $this->assertSame(2, $buffered->getShardInfo('mb')['shards']); + } + + /** + * @test docs/API.md compact(): exact result fields; "Keys never change; shards that became empty are removed, + * except the shard of the newest key" + */ + public function compactResultAndShardRemoval(): void + { + $db = $this->db(['shardSize' => 3]); + $this->assertSame(['success' => false, 'freedSlots' => 0, 'status' => 'database_not_found'], $db->compact('none')); + + // Non-sharded (automatic compaction off): 4 records, one updated twice and one deleted. CLAUDE.md "d": + // garbage lines = superseded and deleted record lines = 2 + 1, which compaction removes + $plain = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); + $plain->insert('cp', [['n' => 1], ['n' => 2], ['n' => 3], ['n' => 4]]); + $plain->update('cp', [['n' => 1], ['set' => ['m' => 1]]]); + $plain->update('cp', [['n' => 1], ['set' => ['m' => 2]]]); + $plain->delete('cp', ['n' => 4]); + $this->assertCount(7, $this->dataLines($this->dbPath('cp', 'data')), '4 inserted lines + 2 updated versions + 1 tombstone'); + $this->assertSame(['success' => true, 'freedSlots' => 3, 'totalRecords' => 3, 'sharded' => false], $plain->compact('cp')); + // The live records, then a tombstone for the highest key ever used (STORAGE_FORMAT.md 6.2: "It writes a + // top tombstone when the highest live key is below the segment's highest assigned key") + $lines = $this->dataLines($this->dbPath('cp', 'data')); + $this->assertCount(4, $lines); + $this->assertSame('{"__nonedb_deleted":3}', $lines[3]); + $this->assertSame([0, 1, 2], array_column($plain->find('cp', 0), 'key')); + $this->assertEquals([['n' => 1, 'm' => 2, 'key' => 0], ['n' => 2, 'key' => 1], ['n' => 3, 'key' => 2]], $plain->find('cp', 0)); + + // Sharded (deletes only: freedSlots = deleted records, docs/API.md example with getShardInfo's deletedCount) + $db = $this->db(['shardSize' => 3, 'autoCompactThreshold' => 1000]); + // One shard only: nothing to remove + $db->insert('one', [['n' => 0], ['n' => 1], ['n' => 2]]); + $db->delete('one', ['n' => 1]); + $this->assertTrue($db->getShardInfo('one')['sharded']); + $this->assertSame(['success' => true, 'freedSlots' => 1, 'totalRecords' => 2, 'sharded' => true, 'newShardCount' => 1], $db->compact('one')); + $this->assertSame([0, 2], array_column($db->find('one', 0), 'key')); + + // Sharded: shard 1 (keys 3-5) emptied and removed, shard 2 (newest key) kept although empty too + $db->insert('many', array_map(function ($n) { return ['n' => $n]; }, range(0, 8))); + foreach (range(3, 8) as $n) { + $db->delete('many', ['n' => $n]); + } + $result = $db->compact('many'); + $this->assertSame(['success' => true, 'freedSlots' => 6, 'totalRecords' => 3, 'sharded' => true, 'newShardCount' => 2], $result); + $this->assertFileDoesNotExist($this->dbPath('many', 'data', 1)); + $this->assertFileExists($this->dbPath('many', 'data', 2)); + $db->insert('many', ['n' => 'new']); + $this->assertSame(9, $db->find('many', ['n' => 'new'])[0]['key'], 'keys are never reused'); + $this->assertSame([0, 1, 2, 9], array_column($db->find('many', 0), 'key')); + } + + /** @test docs/API.md getBufferInfo(): limits and the buffer file */ + public function bufferInfoListsEveryBuffer(): void + { + $db = $this->db(['shardSize' => 4, 'bufferEnabled' => true]); + $this->assertSame(['enabled' => true, 'sizeLimit' => 1048576, 'countLimit' => 10000, 'flushInterval' => 30, + 'buffers' => ['main' => ['exists' => false, 'size' => 0, 'records' => 0]]], $db->getBufferInfo('none')); + $this->assertSame([], glob($this->testDbDir . '*'), 'reading buffer info of a missing database creates nothing'); + + $db->insert('bi', array_map(function ($n) { return ['n' => $n]; }, range(0, 5))); + $db->flush('bi'); + $db->insert('bi', [['n' => 6], ['n' => 7]]); + clearstatcache(); + $info = $db->getBufferInfo('bi'); + $this->assertSame(['exists' => true, 'size' => filesize($this->dbPath('bi', 'buffer')), 'records' => 2], $info['buffers']['main']); + $this->assertSame(['main'], array_keys($info['buffers'])); + $this->assertFileExists($this->dbPath('bi', 'buffer'), 'getBufferInfo() flushes nothing'); + } + + /** + * @test flushAllBuffers() called from inside a database operation (e.g. a shutdown function after a fatal + * error) does nothing instead of waiting for a lock this process holds + */ + public function flushAllBuffersInsideAnOperationDoesNothing(): void + { + $db = $this->db(['bufferEnabled' => true]); + $db->insert('fa', ['v' => 1]); + $result = $this->locked($db, 'fa', false, function () use ($db) { + return $db->flushAllBuffers(); + }); + $this->assertSame(['databases' => 0, 'flushed' => 0], $result); + $this->assertSame(1, $db->getBufferInfo('fa')['buffers']['main']['records'], 'still buffered'); + $this->assertSame(['databases' => 1, 'flushed' => 1], $db->flushAllBuffers()); + $this->assertSame(0, $db->getBufferInfo('fa')['buffers']['main']['records']); + } + + // ========================================== + // checkIntegrity(): read-only, one report entry per kind of problem + // ========================================== + + /** docs/API.md checkIntegrity(): the report of a database without problems */ + private function cleanReport(string $name, int $segments, int $lines, int $records): array + { + return ['ok' => true, 'database' => $name, 'exists' => true, 'segments' => $segments, 'lines' => $lines, 'records' => $records, + 'badLines' => [], 'tornTail' => [], 'indexMismatches' => [], 'bufferedRecords' => 0, 'pendingRecovery' => false, + 'quarantineFiles' => [], 'recoveryLog' => []]; + } + + /** checkIntegrity() through a fresh instance with no caches, asserting that no file changed (lock files included) */ + private function integrity(string $name, array $config = ['shardingEnabled' => false]): array + { + \noneDB::clearStaticCache(); + $before = $this->directorySnapshot(); + $report = $this->db($config)->checkIntegrity($name); + $this->assertSame($before, $this->directorySnapshot(), 'checkIntegrity() writes nothing, not even the lock state'); + return $report; + } + + /** Complete lines of a data file with their offsets: [[offset, length without "\n", json], ...] */ + private function lineOffsets(string $file): array + { + $lines = []; + $offset = 0; + foreach (explode("\n", (string)file_get_contents($file)) as $i => $json) { + if ($json === '' && $offset >= filesize($file)) { + break; + } + $lines[] = [$offset, strlen($json), $json]; + $offset += strlen($json) + 1; + } + return $lines; + } + + /** @test the report of a sound database and of one that doesn't exist (which isn't created) */ + public function integrityReportOfASoundAndAMissingDatabase(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1000]); // no automatic compaction + $db->insert('ic', [['c' => 'x'], ['c' => 'y'], ['c' => 'z']]); + $db->update('ic', [['c' => 'y'], ['set' => ['d' => 1]]]); + $db->delete('ic', ['c' => 'z']); + // 3 records + 1 updated version + 1 tombstone = 5 lines, 2 live records + $this->assertSame($this->cleanReport('ic', 1, 5, 2), $this->integrity('ic')); + + $before = glob($this->testDbDir . '*'); + $this->assertSame(['ok' => true, 'database' => 'missing', 'exists' => false, 'segments' => 0, 'lines' => 0, 'records' => 0, + 'badLines' => [], 'tornTail' => [], 'indexMismatches' => [], 'bufferedRecords' => 0, 'pendingRecovery' => false, + 'quarantineFiles' => [], 'recoveryLog' => []], $db->checkIntegrity('missing')); + $this->assertSame($before, glob($this->testDbDir . '*'), 'no lock file, nothing'); + } + + /** + * @test each kind of key index problem is reported as a "key" mismatch of its segment, read-only, and + * rebuildIndex() fixes it (docs/CONFIGURATION.md "Recovery log and quarantine"; docs/API.md) + */ + public function integrityReportsKeyIndexProblems(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ki', [['c' => 'x'], ['c' => 'y'], ['c' => 'z']]); + $db->rebuildIndex('ki'); // a checkpoint that covers the whole data file + $data = $this->dbPath('ki', 'data'); + $ckpt = $this->dbPath('ki', 'kix'); + $checkpoint = $this->readKeyIndexFile($ckpt); + $this->assertSame(filesize($data), $checkpoint['header']['s'], 'set up: the checkpoint covers the file'); + $size = filesize($data); + $lines = $this->lineOffsets($data); + $this->assertCount(3, $lines); + $edited = $checkpoint['entries']; + unset($edited[1]); + $edited[9] = [0, $lines[0][1]]; + $cases = [ + 'unreadable' => [function () use ($ckpt) { file_put_contents($ckpt, '{"kind":"kix","v":1,"lay'); }, null], + 'beyond the data' => [function () use ($ckpt, $checkpoint, $size) { + $this->writeKeyIndexFile($ckpt, ['s' => $size + 10] + $checkpoint['header'], $checkpoint['entries']); + }, null], + 'not an index' => [function () use ($ckpt) { file_put_contents($ckpt, '[1,2]'); }, null], + 'other entries' => [function () use ($ckpt, $checkpoint, $edited) { + $this->writeKeyIndexFile($ckpt, $checkpoint['header'], $edited); + }, ['1 record', '1 indexed record']], + ]; + foreach ($cases as $label => [$write, $numbers]) { + $write(); + $report = $this->integrity('ki'); + $this->assertFalse($report['ok'], $label); + $this->assertCount(1, $report['indexMismatches'], $label . ': ' . json_encode($report['indexMismatches'])); + $mismatch = $report['indexMismatches'][0]; + $this->assertSame(['main', 'key'], [$mismatch['segment'], $mismatch['index']], $label); + $this->assertNotSame('', (string)$mismatch['detail'], $label); + foreach ($numbers ?? [] as $number) { + $this->assertStringContainsString($number, $mismatch['detail'], $label); + } + $this->assertSame(3, $report['records'], $label . ': records come from the data'); + $this->assertSame(3, $report['lines'], $label); + + $this->assertTrue($this->db(['shardingEnabled' => false])->rebuildIndex('ki')['success'], $label); + $this->assertSame($this->cleanReport('ki', 1, 3, 3), $this->integrity('ki'), $label . ': repaired'); + } + } + + /** @test a spatial index whose entries differ from the records is reported; rebuildIndex() fixes it */ + public function integrityReportsSpatialIndexProblems(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('isp', [['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['loc' => ['type' => 'Point', 'coordinates' => [2, 2]]]]); + $db->createSpatialIndex('isp', 'loc'); + $this->assertSame($this->cleanReport('isp', 1, 2, 2), $this->integrity('isp')); + $rx = $this->dbPath('isp', 'rx', 'loc'); + [$header, $nodes] = $this->readRTreeFile($rx); + $removed = 0; + foreach ($nodes as &$node) { + foreach ($node as $i => $part) { + if (is_array($part) && isset($part[0]) && is_array($part[0]) && count($part[0]) === 5) { + $before = count($part); + $node[$i] = array_values(array_filter($part, function ($entry) { return $entry[0] !== 1; })); + $removed += $before - count($node[$i]); + } + } + } + unset($node); + $this->assertSame(1, $removed, 'set up: the leaf entry of key 1 removed'); + $this->writeRTreeFile($rx, $header, $nodes); + $report = $this->integrity('isp'); + $this->assertFalse($report['ok']); + $this->assertSame([['main', 'spatial loc']], array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches'])); + $this->db(['shardingEnabled' => false])->rebuildIndex('isp'); + $this->assertSame($this->cleanReport('isp', 1, 2, 2), $this->integrity('isp')); + $this->assertSame([0, 1], array_column($this->db(['shardingEnabled' => false])->withinBBox('isp', 'loc', 0, 0, 3, 3), 'key')); + } + + /** @test shard meta disagreeing with the shard files: each case is one "meta" mismatch (segment null) */ + public function integrityReportsShardMetaProblems(): void + { + $config = ['shardSize' => 3]; + $setup = function () use ($config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('ish', array_map(function ($n) { return ['n' => $n]; }, range(0, 7))); // keys 0-7: shards 0, 1, 2 + $this->assertTrue($db->getShardInfo('ish')['sharded']); + return $this->dbPath('ish', 'shards'); + }; + $lineCount = function () { + $n = 0; + foreach (glob($this->dbPath('ish', 'gen') . 'segments/s*.jsonl') as $shard) { + $n += count($this->dataLines($shard)); + } + return $n; + }; + $metaPath = $setup(); + $this->assertSame($this->cleanReport('ish', 3, $lineCount(), 8), $this->integrity('ish', $config)); + + $cases = [ + 'a shard the files lack' => [function (array $m) { $m['shards'][] = ['id' => 5, 'count' => 0, 'deleted' => 0]; return json_encode($m); }, []], + 'wrong totalRecords' => [function (array $m) { $m['totalRecords'] = 99; return json_encode($m); }, ['99', '8']], + 'nextKey behind the data' => [function (array $m) { $m['nextKey'] = 3; return json_encode($m); }, ['3', '7']], + ]; + foreach ($cases as $label => [$edit, $numbers]) { + $metaPath = $setup(); + $lines = $lineCount(); + file_put_contents($metaPath, $edit(json_decode((string)file_get_contents($metaPath), true))); + $report = $this->integrity('ish', $config); + $this->assertFalse($report['ok'], $label); + $this->assertSame([[null, 'meta']], array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches']), + $label . ': ' . json_encode($report['indexMismatches'])); + foreach ($numbers as $number) { + $this->assertStringContainsString($number, $report['indexMismatches'][0]['detail'], $label); + } + $this->assertSame([3, $lines, 8], [$report['segments'], $report['lines'], $report['records']], $label . ': counted from the shard files'); + } + + // A shard file that is gone: the meta lists it, its records are missing + $setup(); + unlink($this->dbPath('ish', 'data', 1)); + @unlink($this->dbPath('ish', 'kix', 1)); + $report = $this->integrity('ish', $config); + $this->assertSame([[null, 'meta'], [null, 'meta']], array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches'])); + $this->assertSame([2, 5], [$report['segments'], $report['records']]); + \noneDB::clearStaticCache(); + $this->assertSame(5, $this->db($config)->getIndexInfo('ish')['totalRecords'], 'the key indexes that exist'); + } + + /** + * @test an unreadable shard meta is reported ("meta file unreadable") and, like everything else, left as it + * is: docs/API.md "Read-only: it creates, changes, cuts and flushes nothing"; CHANGES "writes nothing, not + * even a lock-state change" + */ + public function integrityOfAnUnreadableShardMetaWritesNothing(): void + { + $config = ['shardSize' => 3]; + $db = $this->db($config); + $db->insert('um', array_map(function ($n) { return ['n' => $n]; }, range(0, 7))); + $metaPath = $this->dbPath('um', 'shards'); + file_put_contents($metaPath, 'garbage{'); + $report = $this->integrity('um', $config); // asserts that no file changed + $this->assertSame([[null, 'meta']], array_map(function ($m) { return [$m['segment'], $m['index']]; }, $report['indexMismatches'])); + $this->assertSame([3, 8], [$report['segments'], $report['records']]); + $this->assertSame('garbage{', file_get_contents($metaPath)); + } + + /** + * @test pendingRecovery: "the lock file says a recovery is due" (journal j/J, STORAGE_FORMAT.md 7.3); + * checkIntegrity() reports it without doing it; the next access does it + */ + public function integrityReportsPendingRecoveryWithoutRecovering(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('pr', [['v' => 1], ['v' => 2]]); + $lock = $this->dbPath('pr', 'lock'); + $record = trim($this->lockRecord('pr')); + $this->assertMatchesRegularExpression('/^\d+:[0-9a-f]{8}$/', $record); + [$state] = explode(' ', $record); + foreach (['journal' => $state . ' J', 'segment journal' => $state . ' j=main'] as $label => $content) { + \noneDB::clearStaticCache(); + file_put_contents($lock, str_pad($content, 128)); + $report = $this->integrity('pr'); + $expected = $this->cleanReport('pr', 1, 2, 2); + $expected['ok'] = false; + $expected['pendingRecovery'] = true; + $this->assertSame($expected, $report, $label); + $this->assertSame(str_pad($content, 128), file_get_contents($lock), $label . ': still pending'); + // A normal access recovers + $this->assertSame([1, 2], array_column($this->db(['shardingEnabled' => false])->find('pr', 0), 'v'), $label); + $this->assertSame([], $this->lockState('pr')['flags'], $label); + $this->assertSame($this->cleanReport('pr', 1, 2, 2), $this->integrity('pr'), $label); + } + } + + /** @test buffered records (buffer and interrupted flush files), quarantine files and the recovery log are listed */ + public function integrityListsBuffersQuarantineAndRecoveryLog(): void + { + $db = $this->db(['shardingEnabled' => false, 'bufferEnabled' => true]); + $db->insert('bq', ['v' => 1]); + $db->flush('bq'); + $db->insert('bq', [['v' => 2], ['v' => 3]]); + file_put_contents($this->dbPath('bq', 'flushing', 7), json_encode(['v' => 4]) . "\n"); + $report = $this->integrity('bq', ['shardingEnabled' => false, 'bufferEnabled' => true]); + $this->assertSame(3, $report['bufferedRecords'], 'two in the buffer, one in an interrupted flush'); + $this->assertTrue($report['ok'], 'buffered records are no problem'); + unlink($this->dbPath('bq', 'flushing', 7)); + + // A torn tail cut by the next writer: quarantined under the documented name, logged + $data = $this->dbPath('bq', 'data'); + $db->flush('bq'); + clearstatcache(); + $offset = filesize($data); + file_put_contents($data, '{"torn', FILE_APPEND); + \noneDB::clearStaticCache(); + $torn = $this->integrity('bq'); + $this->assertSame([['segment' => 'main', 'offset' => $offset, 'length' => 6]], $torn['tornTail']); + $this->assertFalse($torn['ok']); + $this->db(['shardingEnabled' => false])->insert('bq', ['v' => 5]); + // STORAGE_FORMAT.md 7.2: recovery/q.{segment}.{offset}.{length}.{sha256}.bin, segment "main" for the single layout + $name = 'q.main.' . $offset . '.6.' . hash('sha256', '{"torn') . '.bin'; + $this->assertSame('{"torn', file_get_contents($this->dbPath('bq', 'quarantine') . $name)); + $report = $this->integrity('bq'); + $this->assertTrue($report['ok']); + $this->assertSame([], $report['tornTail']); + $this->assertSame([$name], $report['quarantineFiles']); + $this->assertCount(1, $report['recoveryLog']); + $this->assertSame(['cut-tail', $name, true], [$report['recoveryLog'][0]['action'], $report['recoveryLog'][0]['quarantine'], $report['recoveryLog'][0]['bytesVerified']]); + $this->assertSame([1, 2, 3, 5], array_column($this->db(['shardingEnabled' => false])->find('bq', 0), 'v')); + } + + // ========================================== + // getRecoveryLog(): actions, pending entries, rotation, limit + // ========================================== + + /** + * @test docs/API.md getRecoveryLog(): "Reads never write the log. A damaged line a read found is listed with + * "persisted" => false until a write session of the same process logs it (once: entries carry an id); + * after the process ends it is found again by the next read" + */ + public function damagedLinesFoundByReadsArePendingUntilAWriteLogsThemOnce(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('pend', [['v' => 1], ['v' => 2]]); + $data = $this->dbPath('pend', 'data'); + $log = $this->dbPath('pend', 'recovery'); + clearstatcache(); + $offset = filesize($data); + file_put_contents($data, "not json\n", FILE_APPEND); + // Every pending entry describes the same damaged line: off/len are its bytes, id names them + // ({segment}.{offset}.{length}.{sha256}, STORAGE_FORMAT.md 7.2) + $id = 'main.' . $offset . '.9.' . hash('sha256', "not json\n"); + + for ($round = 1; $round <= 2; $round++) { + \noneDB::clearStaticCache(); // a new process + $db = $this->db($config); + $this->assertSame(range(1, $round + 1), array_column($db->find('pend', 0), 'v'), "round $round: the damaged line is skipped"); + $entries = $db->getRecoveryLog('pend'); + $pending = end($entries); + $this->assertSame(['skip-line', 'main', $offset, 9, $id, false], + [$pending['action'], $pending['seg'], $pending['off'], $pending['len'], $pending['id'], $pending['persisted']], "round $round"); + $this->assertCount($round, $entries, "round $round: the persisted entries of earlier rounds and this pending one"); + if ($round === 1) { + $this->assertFileDoesNotExist($log, 'reads never write the log'); + } + + $db->insert('pend', ['v' => 2 + $round]); // a write session of this process logs it (once) + $entries = $db->getRecoveryLog('pend'); + $this->assertCount(1, $entries, "round $round: logged once, deduplicated by id"); + $this->assertSame([$id, 'skip-line'], [$entries[0]['id'], $entries[0]['action']]); + $this->assertArrayNotHasKey('persisted', $entries[0]); + $this->assertCount(1, $this->dataLines($log)); + } + $this->assertSame([1, 2, 3, 4], array_column($db->find('pend', 0), 'v')); + } + + /** + * @test STORAGE_FORMAT.md 7.1: the log is recovery/log.jsonl; past 1 MiB it moves to recovery/log.1.jsonl, + * replacing the older one; getRecoveryLog() returns the newest $limit entries, oldest first + */ + public function recoveryLogRotatesPastOneMebibyte(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 0.1]); + $db->insert('rot', array_map(function ($n) { return ['n' => $n]; }, range(1, 10))); + $log = $this->dbPath('rot', 'recovery'); + $log1 = $this->dbPath('rot', 'recovery1'); + @mkdir(dirname($log), 0777, true); + file_put_contents($log1, json_encode(['t' => 1, 'action' => 'oldest']) . "\n"); + $old = ''; + for ($i = 0; strlen($old) <= 1048576; $i++) { + $old .= json_encode(['t' => 2, 'action' => 'old', 'i' => $i, 'pad' => str_repeat('p', 200)]) . "\n"; + } + file_put_contents($log, $old); + $oldCount = $i; + + // An event to log: a deferred compaction that fails + $this->setFaultHook(function ($point) { + if ($point === 'job:compact:main') { + throw new \RuntimeException('compaction failed'); + } + }); + $this->assertSame(10, $db->update('rot', [[], ['set' => ['m' => 1]]])['n']); + $this->setFaultHook(null); + + $this->assertSame($old, file_get_contents($log1), 'the full log moved, replacing the older one'); + $current = $this->dataLines($log); + $this->assertCount(1, $current); + $this->assertSame(['deferred-failed', 'compact:main', 'compaction failed'], + array_values(array_intersect_key(json_decode($current[0], true), ['action' => 1, 'job' => 1, 'error' => 1]))); + + $entries = $db->getRecoveryLog('rot', 3); + $this->assertSame([['old', $oldCount - 2], ['old', $oldCount - 1], ['deferred-failed', null]], + array_map(function ($e) { return [$e['action'], $e['i'] ?? null]; }, $entries)); + $this->assertCount(100, $db->getRecoveryLog('rot'), 'default limit 100'); + $this->assertSame('deferred-failed', $db->getRecoveryLog('rot', 1)[0]['action']); + $this->assertCount($oldCount + 1, $db->getRecoveryLog('rot', 1000000)); + } + + /** @test getRecoveryLog($db, 0): "$limit Newest entries to return" -> none */ + public function recoveryLogLimitZeroReturnsNothing(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('lim', ['v' => 1]); + @mkdir(dirname($this->dbPath('lim', 'recovery')), 0777, true); + file_put_contents($this->dbPath('lim', 'recovery'), json_encode(['t' => 1, 'action' => 'cut-tail']) . "\n" + . json_encode(['t' => 2, 'action' => 'drop-line']) . "\n"); + $this->assertSame(['drop-line'], array_column($db->getRecoveryLog('lim', 1), 'action')); + $this->assertSame(['cut-tail', 'drop-line'], array_column($db->getRecoveryLog('lim', 2), 'action')); + $this->assertSame([], $db->getRecoveryLog('lim', 0)); + } + + // ========================================== + // getDurability() + // ========================================== + + /** @test docs/DURABILITY.md §6 getDurability(): the documented fields, and what the report says in performance mode */ + public function durabilityReportFieldsInPerformanceMode(): void + { + $db = $this->db(['shardingEnabled' => false]); + $report = $db->getDurability(); + $this->assertSame(['mode', 'platform', 'lastOperation', 'lastSyncError', 'syncs'], array_keys($report)); + $this->assertSame('performance', $report['mode'], 'the default mode'); + $this->assertSame(['os' => PHP_OS_FAMILY, 'durableSupported' => PHP_OS_FAMILY === 'Linux'], + array_intersect_key($report['platform'], ['os' => 0, 'durableSupported' => 0])); + $this->assertCount(PHP_OS_FAMILY === 'Linux' ? 0 : 1, $report['platform']['notes'], json_encode($report['platform']['notes'])); + $this->assertSame(['file', 'dir', 'failed'], array_keys($report['syncs'])); + + $syncs = $report['syncs']; + $db->insert('dura', ['v' => 1]); + $last = $db->getDurability()['lastOperation']; + $this->assertSame(['op', 'db', 'outcome', 'required', 'completed', 'reason'], array_keys($last)); + $this->assertSame(['insert', 'dura', 'committed', [], [], null], array_values($last), 'performance mode requires no sync'); + $this->assertSame($syncs, $db->getDurability()['syncs'], 'nothing synced'); + + // "the last write operation": reads and writes that change nothing don't replace it + $db->insert('other', ['v' => 1]); + $db->find('dura', 0); + $db->update('other', [['v' => 99], ['set' => ['w' => 1]]]); + $this->assertSame(['insert', 'other'], [$db->getDurability()['lastOperation']['op'], $db->getDurability()['lastOperation']['db']]); + } + + /** + * @test docs/DURABILITY.md §3: in durable mode "Set a state-record flag (b, ..., j) ... sync file. This always + * happens before the data change it protects"; a write that sets no flag doesn't sync the record + */ + public function lockStateRecordIsSyncedWhenItProtectsData(): void + { + $this->setStatic('osFamily', 'Linux'); // durable mode's rules, on every platform + $synced = []; + $this->setStatic('syncFn', function ($handle, string $kind) use (&$synced) { + $stat = fstat($handle); + $synced[] = $kind . ':' . $stat['ino']; + return true; + }); + $lockSynced = function (string $name) use (&$synced) { + clearstatcache(); + return in_array('file:' . fileinode($this->dbPath($name, 'lock')), $synced, true); + }; + $plain = $this->db(['durability' => 'durable', 'shardingEnabled' => false]); + $plain->insert('plain', ['v' => 1]); + $synced = []; + $plain->insert('plain', ['v' => 2]); + $this->assertContains('file:' . fileinode($this->dbPath('plain', 'data')), $synced, 'the data'); + $this->assertFalse($lockSynced('plain'), 'no flag set: the state record is not synced'); + + $buffered = $this->db(['durability' => 'durable', 'shardingEnabled' => false, 'bufferEnabled' => true]); + $buffered->insert('buf', ['v' => 1]); + $synced = []; + $buffered->insert('buf', ['v' => 2]); + $this->assertContains('b', $this->lockState('buf')['flags']); + $this->assertTrue($lockSynced('buf'), 'buffer flag set: synced'); + + $sharded = $this->db(['durability' => 'durable', 'shardSize' => 2]); + $sharded->insert('sh', [['v' => 1], ['v' => 2]]); + $synced = []; + $sharded->insert('sh', ['v' => 3]); // a sharded write sets the journal + $this->assertTrue($lockSynced('sh'), 'journal set: synced'); + } + + // ========================================== + // migrateKey(): refusals, interruption, caches + // ========================================== + + /** @test every refusal returns success false with an error and renames nothing */ + public function migrateKeyRefusals(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('mk', ['v' => 1]); + $new = 'new-secret-key-0123456789abcdef'; + // A second key of the store (STORAGE_FORMAT.md 9.3) with a database "mk": the target of mk under the new key + $this->assertTrue(\noneDB::registerKey($this->testDbDir, self::SECRET, $new)['success']); + $other = new \noneDB(['secretKey' => $new, 'dbDir' => $this->testDbDir]); + $other->insert('mk', ['v' => 'already there']); + \noneDB::clearStaticCache(); + $before = $this->directorySnapshot(); + $cases = [ + 'empty old key' => [$this->testDbDir, ' ', $new, null], + 'empty new key' => [$this->testDbDir, self::SECRET, '', null], + 'same key' => [$this->testDbDir, self::SECRET, self::SECRET, null], + 'missing directory' => [$this->testDbDir . 'nope/', self::SECRET, $new, $this->testDbDir . 'nope/'], + 'target exists' => [$this->testDbDir, self::SECRET, $new, "'mk'"], + ]; + foreach ($cases as $label => [$dir, $old, $newKey, $mentioned]) { + $result = \noneDB::migrateKey($dir, $old, $newKey); + $this->assertSame(['success' => false, 'databases' => [], 'storeManifest' => null], array_slice($result, 0, 3, true), $label); + $this->assertIsString($result['error'], $label); + $this->assertNotSame('', $result['error'], $label); + if ($mentioned !== null) { + $this->assertStringContainsString($mentioned, $result['error'], $label); + } + $this->assertSame($before, $this->directorySnapshot(), $label . ': nothing renamed'); + } + + // A database whose lock can't be taken (its LOCK can't be opened) is not renamed + $other->dropDB('mk'); + \noneDB::clearStaticCache(); + // dropDB() keeps LOCK and the directory (STORAGE_FORMAT.md 8.2): removed, so the target is free + $this->assertTrue(unlink($this->homeOf('mk', null, $new) . 'LOCK')); + $this->assertTrue(rmdir($this->homeOf('mk', null, $new))); + $lock = $this->dbPath('mk', 'lock'); + $content = file_get_contents($lock); + unlink($lock); + mkdir($lock); + try { + $before = $this->directorySnapshot(); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new); + $this->assertFalse($result['success']); + $this->assertSame(\noneDB::MAINTENANCE_BUSY, $result['code'], 'STORAGE_FORMAT.md 9.2: a target lock that fails'); + $this->assertStringContainsString("'mk'", $result['error']); + $this->assertSame($before, $this->directorySnapshot()); + } finally { + rmdir($lock); + file_put_contents($lock, $content); + } + \noneDB::clearStaticCache(); + $this->assertSame([1], array_column($this->db(['shardingEnabled' => false])->find('mk', 0), 'v')); + } + + /** + * @test docs/API.md migrateKey(): "If it is interrupted, run it again with the same arguments: it finishes + * the job". STORAGE_FORMAT.md 9.1-9.3: interrupted after marking its targets, between and after the renames + * of the database directories, after its commit or after the STORE copies, the rotation leaves its journal + * open: the store answers MAINTENANCE_IN_PROGRESS for every key until the next run finishes it + */ + public function migrateKeyInterruptedInThisProcessFinishesOnTheNextRun(): void + { + $setup = function () { + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 3]); + $db->createFieldIndex('k1', 'c'); + $db->insert('k1', array_map(function ($n) { return ['n' => $n, 'c' => 'c' . ($n % 2)]; }, range(0, 6))); + $db->insert('k2', [['v' => 'a'], ['v' => 'b']]); + \noneDB::clearStaticCache(); + return ['k1' => $this->filesOfDb('k1', null, self::SECRET, true), 'k2' => $this->filesOfDb('k2', null, self::SECRET, true)]; + }; + $new = 'brand-new-key-for-migration-0123456789'; + foreach ([['rotate:marked', 1], ['rotate:renamed', 1], ['rotate:renamed', 2], ['rotate:committed', 1], + ['maintenance:store-written', 1]] as [$point, $count]) { + $label = "stop at $point #$count"; + $files = $setup(); + $seen = 0; + $this->setFaultHook(function ($p) use (&$seen, $point, $count) { + if ($p === $point && ++$seen === $count) { + throw new \RuntimeException('interrupted'); + } + }); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new); + $this->setFaultHook(null); + $this->assertFalse($result['success'], $label); + $this->assertStringContainsString('interrupted', (string)$result['error'], $label); + foreach ([self::SECRET, $new] as $key) { + try { + (new \noneDB(['secretKey' => $key, 'dbDir' => $this->testDbDir, 'autoCreateDB' => false]))->find('k2', 0); + $this->fail("$label: the store is closed while the journal is open"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $e->getCode(), "$label: " . $e->getMessage()); + } + } + + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, $new); + $this->assertTrue($result['success'], "$label: " . json_encode($result)); + $databases = $result['databases']; + sort($databases); + $this->assertSame(['k1', 'k2'], $databases, $label); + $this->assertSame([], array_merge($this->filesOfDb('k1', null, self::SECRET, true), $this->filesOfDb('k2', null, self::SECRET, true)), + "$label: nothing left under the old key"); + $this->assertSame($files, ['k1' => $this->filesOfDb('k1', null, $new, true), 'k2' => $this->filesOfDb('k2', null, $new, true)], + "$label: the directories moved with all their files"); + $db = new \noneDB(['secretKey' => $new, 'dbDir' => $this->testDbDir, 'shardSize' => 3, 'autoCreateDB' => false]); + $this->assertSame(range(0, 6), array_column($db->find('k1', 0), 'n'), $label); + $this->assertSame([1, 3, 5], array_column($db->find('k1', ['c' => 'c1']), 'n'), $label); + $this->assertSame(['a', 'b'], array_column($db->find('k2', 0), 'v'), $label); + } + } + + /** + * @test a rename that fails stops migrateKey() with an error; the started rotation keeps the store closed + * (STORAGE_FORMAT.md 9.1: MAINTENANCE_IN_PROGRESS); fixed, the next run finishes + */ + public function migrateKeyThatCannotRenameReportsItAndCanBeRepeated(): void + { + $dir = $this->scratch('mkro') . '/'; + $other = 'other-key-0123456789'; + $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dir]); + $db->insert('ro', [['v' => 1], ['v' => 2]]); + \noneDB::clearStaticCache(); + $files = $this->filesOfDb('ro', $dir, self::SECRET, true); + // The rename of the database directory fails + $this->setStatic('renameFn', function ($from, $to) { + return substr(rtrim($from, '/'), -5) === '.ndb4' ? false : rename($from, $to); + }); + try { + $result = \noneDB::migrateKey($dir, self::SECRET, $other); + } finally { + $this->setStatic('renameFn', null); + } + $this->assertFalse($result['success']); + $this->assertSame([], $result['databases']); + $this->assertStringContainsString('rename', $result['error']); + $this->assertStringContainsString("'ro'", $result['error']); + $this->assertSame($files, $this->filesOfDb('ro', $dir, self::SECRET, true), 'nothing renamed'); + $this->assertDirectoryDoesNotExist($this->homeOf('ro', $dir, $other)); + foreach ([self::SECRET, $other] as $key) { + try { + (new \noneDB(['secretKey' => $key, 'dbDir' => $dir]))->find('ro', 0); + $this->fail('the store is closed while the rotation is unfinished'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $e->getCode(), $e->getMessage()); + } + } + $this->assertTrue(\noneDB::migrateKey($dir, self::SECRET, $other)['success']); + $this->assertSame([1, 2], array_column((new \noneDB(['secretKey' => $other, 'dbDir' => $dir]))->find('ro', 0), 'v')); + } + + // ========================================== + // Interrupted sessions (fatal error, exit()) + // ========================================== + + /** + * @test CHANGES "Crash safety": "A fatal error in the middle of a write: noneDB's shutdown handler releases + * the interrupted session, so later writes in shutdown functions start a fresh session with recovery" + */ + public function shutdownFunctionsAfterAnInterruptedWriteStartAFreshSession(): void + { + $out = $this->runChild(' + $db = ' . $this->childDb(['shardingEnabled' => false]) . '; + $db->insert("sd", ["v" => "first"]); + register_shutdown_function(function () use ($db) { + noneDB\Tests\Support\Testing::setFaultHook(null); + echo "locks held:", count(noneDB\Tests\Support\Testing::get("lockRegistry")), "\n"; + echo "insert:", $db->insert("sd", ["v" => "shutdown"])["n"], "\n"; + }); + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { + if ($point === "append:after") { exit(3); } // like a fatal error: finally blocks are skipped + }); + $db->insert("sd", ["v" => "interrupted"]); + echo "not reached\n"; + '); + $this->assertSame("locks held:0\ninsert:1", $out); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + // The interrupted line was complete: kept (roll forward); the shutdown write got the next key + $this->assertSame([['v' => 'first', 'key' => 0], ['v' => 'interrupted', 'key' => 1], ['v' => 'shutdown', 'key' => 2]], $db->find('sd', 0)); + $this->assertSame([], $this->lockState('sd')['flags']); + } + + // ========================================== + // Recovery details + // ========================================== + + /** + * @test STORAGE_FORMAT.md 5.2: a switch to shards interrupted after its commit point (PUBLISH: CURRENT names + * the new generation) leaves no second copy of the data: the recovery of g=N>X removes the old generation, + * with its index files, spatial ones included + */ + public function aSwitchToShardsInterruptedAfterItsCommitPointWithASpatialIndexIsCleanedUp(): void + { + $config = ['shardSize' => 4]; + $db = $this->db($config); + $db->createSpatialIndex('mgs', 'loc'); + $db->createFieldIndex('mgs', 'c'); + $db->insert('mgs', array_map(function ($n) { + return ['n' => $n, 'c' => 'c' . ($n % 2), 'loc' => ['type' => 'Point', 'coordinates' => [$n, $n]]]; + }, range(0, 2))); + $old = $this->dbPath('mgs', 'gen'); // generation 1 of the new database (STORAGE_FORMAT.md 8.1) + $this->assertFileExists($this->dbPath('mgs', 'rx', 'loc')); + \noneDB::clearStaticCache(); + $out = $this->runChild(' + $db = ' . $this->childDb($config) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) { if ($p === "generation:published") { exit(0); } }); + $db->insert("mgs", ["n" => 3, "c" => "c1", "loc" => ["type" => "Point", "coordinates" => [3, 3]]]); + echo "not reached"; + '); + $this->assertSame('', $out); + $this->assertFileExists($old . 'segments/main.jsonl', 'the old file is still there'); + $this->assertContains('g=1>2', $this->lockState('mgs')['flags'], 'the state record says so'); + + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(0, 3), array_column($db->find('mgs', 0), 'n')); + $this->assertFileDoesNotExist($old . 'segments/main.jsonl'); + $this->assertFileDoesNotExist($old . 'spatial/main.loc.rx', 'the old segment\'s spatial index went with it'); + $this->assertFileDoesNotExist($old . 'indexes/main.c.fx'); + $got = array_column($db->withinBBox('mgs', 'loc', -1, -1, 10, 10), 'n'); + sort($got); + $this->assertSame(range(0, 3), $got); + $this->assertSame([1, 3], array_column($db->find('mgs', ['c' => 'c1']), 'n')); + $this->assertTrue($db->checkIntegrity('mgs')['ok']); + } + + /** @test the recovery log is best effort: failing to write it never fails the operation */ + public function aRecoveryLogThatCannotBeWrittenNeverFailsTheOperation(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 0.1]); + $db->insert('nolog', array_map(function ($n) { return ['n' => $n]; }, range(1, 10))); + $log = $this->dbPath('nolog', 'recovery'); + $this->setStatic('flushFn', function ($handle, string $path) use ($log) { + return $path === $log ? false : fflush($handle); + }); + $this->setFaultHook(function ($point, $context) use ($log) { + if ($point === 'job:compact:main') { + throw new \RuntimeException('compaction broke'); // to be logged as "deferred-failed" + } + if ($point === 'append:undo' && $context['path'] === $log) { + throw new \RuntimeException('disk gone'); + } + }); + $this->assertSame(['n' => 10], $db->update('nolog', [[], ['set' => ['m' => 1]]])); + $this->setFaultHook(null); + $this->setStatic('flushFn', null); + $this->assertSame(array_fill(0, 10, 1), array_column($db->find('nolog', 0), 'm')); + } + + /** + * @test field and spatial index files whose body is damaged (header intact) never change a result: checkIntegrity() + * stays read-only, queries rebuild them from the data + */ + public function damagedSecondaryIndexBodiesNeverChangeResults(): void + { + $config = ['shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('sec', [['c' => 'x', 'loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['c' => 'y', 'loc' => ['type' => 'Point', 'coordinates' => [2, 2]]]]); + $db->createFieldIndex('sec', 'c'); + $db->createSpatialIndex('sec', 'loc'); + foreach ([$this->dbPath('sec', 'fx', 'c'), $this->dbPath('sec', 'rx', 'loc')] as $index) { + [$header] = explode("\n", (string)file_get_contents($index), 2); + file_put_contents($index, $header . "\n" . '{"not": json'); + } + $report = $this->integrity('sec', $config); + $this->assertSame([], $report['badLines']); + $this->assertSame(2, $report['records']); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(['x'], array_column($db->find('sec', ['c' => 'x']), 'c')); + $this->assertSame([0, 1], array_column($db->withinBBox('sec', 'loc', 0, 0, 3, 3), 'key')); + $this->assertSame([1], array_column($db->query('sec')->where(['c' => 'y'])->get(), 'key')); + } + +} diff --git a/tests/Integration/SpecQueryTest.php b/tests/Integration/SpecQueryTest.php new file mode 100644 index 0000000..cebe537 --- /dev/null +++ b/tests/Integration/SpecQueryTest.php @@ -0,0 +1,1764 @@ + self::refToNumber($b); + } + if (is_string($a) && is_string($b)) { + return strcmp($a, $b) <=> 0; + } + return null; + } + + /** Sort kinds: missing/null < booleans < numbers < numeric strings < other strings < arrays */ + private static function refSortKind($v): int + { + if ($v === null) return 0; + if (is_bool($v)) return 1; + if (is_int($v) || is_float($v)) return 2; + if (self::refNumericString($v)) return 3; + if (is_string($v)) return 4; + return 5; + } + + private static function refSortCompare($a, $b): int + { + $ka = self::refSortKind($a); + $kb = self::refSortKind($b); + if ($ka !== $kb) return $ka <=> $kb; + switch ($ka) { + case 0: return 0; + case 1: return (int)$a <=> (int)$b; // false < true + case 2: return $a <=> $b; // int and float together + case 3: return self::refToNumber($a) <=> self::refToNumber($b); + case 4: return strcmp($a, $b) <=> 0; // byte order + default: return strcmp(json_encode($a), json_encode($b)) <=> 0; // arrays by their JSON + } + } + + /** + * Stable sort of records by [[field, 'asc'|'desc'], ...]; ties keep the input (key) order + */ + private static function refSort(array $records, array $keys): array + { + $indexed = []; + foreach (array_values($records) as $pos => $r) { + $indexed[] = [$pos, $r]; + } + usort($indexed, function ($x, $y) use ($keys) { + foreach ($keys as [$field, $order]) { + $c = self::refSortCompare(array_key_exists($field, $x[1]) ? $x[1][$field] : null, + array_key_exists($field, $y[1]) ? $y[1][$field] : null); + if ($c !== 0) { + return $order === 'desc' ? -$c : $c; + } + } + return $x[0] <=> $y[0]; + }); + return array_column($indexed, 1); + } + + /** like(): literal, case-insensitive; a leading ^ anchors the start, a trailing $ the end (ASCII data) */ + private static function refLike(string $subject, string $pattern): bool + { + $start = $pattern !== '' && $pattern[0] === '^'; + $rest = $start ? (string)substr($pattern, 1) : $pattern; + $end = $rest !== '' && substr($rest, -1) === '$'; + $body = strtolower($end ? (string)substr($rest, 0, -1) : $rest); + $s = strtolower($subject); + if ($start && $end) return $s === $body; + if ($start) return strncmp($s, $body, strlen($body)) === 0; + if ($end) return $body === '' || (strlen($s) >= strlen($body) && substr($s, -strlen($body)) === $body); + return $body === '' || strpos($s, $body) !== false; + } + + /** like/$like/$regex only look at strings and numbers */ + private static function refLikeable($v): bool + { + return is_string($v) || is_int($v) || is_float($v); + } + + private static function refOperator(array $record, string $field, string $op, $operand): bool + { + $has = array_key_exists($field, $record); + $v = $has ? $record[$field] : null; + switch ($op) { + case '$gt': return $has && ($c = self::refCompare($v, $operand)) !== null && $c > 0; + case '$gte': return $has && ($c = self::refCompare($v, $operand)) !== null && $c >= 0; + case '$lt': return $has && ($c = self::refCompare($v, $operand)) !== null && $c < 0; + case '$lte': return $has && ($c = self::refCompare($v, $operand)) !== null && $c <= 0; + case '$eq': return $has && $v === $operand; + case '$ne': return !$has || $v !== $operand; + case '$in': return $has && in_array($v, $operand, true); + case '$nin': return !$has || !in_array($v, $operand, true); + case '$exists': return $operand ? $has : !$has; + case '$like': return $has && self::refLikeable($v) && self::refLike((string)$v, (string)$operand); + case '$regex': // the pattern body as given (no delimiters: "~" is a literal character) + return $has && self::refLikeable($v) && preg_match("\x01" . $operand . "\x01iu", (string)$v) === 1; + case '$contains': + if (!$has) return false; + if (is_array($v)) return in_array($operand, $v, true); + if (is_string($v) && is_scalar($operand) && !is_bool($operand)) { + return (string)$operand === '' || strpos($v, (string)$operand) !== false; + } + return false; + } + throw new \LogicException("reference model: $op"); + } + + private static function refIsOperatorSpec($value): bool + { + if (!is_array($value) || $value === []) { + return false; + } + foreach (array_keys($value) as $k) { + if (!is_string($k) || $k === '' || $k[0] !== '$') { + return false; + } + } + return true; + } + + /** Key filter: integers and canonical digit strings, a single one or a list (docs/API.md find()) */ + private static function refKeyList($value): array + { + $keys = []; + foreach (is_array($value) ? $value : [$value] as $v) { + if (is_int($v) && $v >= 0) { + $keys[$v] = $v; + } elseif (is_string($v) && preg_match('/\A(0|[1-9][0-9]*)\z/', $v) === 1) { + $keys[(int)$v] = (int)$v; + } + } + return array_values($keys); + } + + /** One field condition of where()/orWhere()/whereNot() */ + private static function refCondition(array $record, string $field, $value): bool + { + if (self::refIsOperatorSpec($value)) { + foreach ($value as $op => $operand) { + if (!self::refOperator($record, $field, $op, $operand)) { + return false; + } + } + return true; + } + if ($field === 'key') { + return in_array($record['key'], self::refKeyList($value), true); + } + return array_key_exists($field, $record) && $record[$field] === $value; + } + + private static function refGroup(array $record, array $group): bool + { + foreach ($group as $field => $value) { + if (!self::refCondition($record, (string)$field, $value)) { + return false; + } + } + return true; + } + + /** where() groups all match OR any orWhere() group matches; without where() only orWhere() decides */ + private static function refWhere(array $records, array $where, array $orWhere = []): array + { + $out = []; + foreach ($records as $r) { + $all = !empty($where); + foreach ($where as $g) { + if (!self::refGroup($r, $g)) { + $all = false; + break; + } + } + $any = false; + foreach ($orWhere as $g) { + if (self::refGroup($r, $g)) { + $any = true; + break; + } + } + if ($all || $any || (empty($where) && empty($orWhere))) { + $out[] = $r; + } + } + return $out; + } + + /** Records as stored and read back: the given rows with "key" added (keys 0..n-1 in insert order) */ + private static function withKeys(array $rows): array + { + $out = []; + foreach (array_values($rows) as $k => $row) { + $out[] = $row + ['key' => $k]; + } + return $out; + } + + /** noneDB::sum()/avg() return an int or a float (docblock): a number equal to $expected, never a string or null */ + private function assertNumber($expected, $actual, string $message = ''): void + { + $this->assertTrue((is_int($actual) || is_float($actual)) && (float)$actual === (float)$expected, + $message . ': ' . var_export($actual, true) . ' is not the number ' . var_export($expected, true)); + } + + private static function keysOf(array $records): array + { + return array_map(function ($r) { return $r['key']; }, array_values($records)); + } + + // ====================================================================== + // Fixtures + // ====================================================================== + + /** Values of every kind the rules distinguish (trailing-whitespace strings: see their own test) */ + private static function mixedValues(): array + { + return [ + 5, 30, 0, -1, 5.5, 5.0, 0.5, + '5', '30', ' 7', '1e1', '.5', '-0', + 'abc', 'ABC', 'B', 'b', '', '1a', '0x1A', 'a~b', 'x.y', + null, true, false, + [1], ['a', 'abc'], ['a' => 1], [], + self::MISSING, + ]; + } + + private static function mixedRows(): array + { + $rows = []; + foreach (self::mixedValues() as $i => $v) { + $row = ['i' => $i, 'g' => $i % 3]; + if ($v !== self::MISSING) { + $row['v'] = $v; + } + $rows[] = $row; + } + return $rows; + } + + /** + * The same rows in four databases: plain, plain with a field index on $indexField, sharded (shards of + * 4 records), sharded with the index + * @return array name => \noneDB + */ + private function setups(array $rows, string $indexField = 'v', string $prefix = 'm'): array + { + $setups = []; + foreach (['plain' => false, 'plainix' => true, 'shard' => false, 'shardix' => true] as $variant => $indexed) { + $sharded = strpos($variant, 'shard') === 0; + $db = $this->db($sharded ? ['shardSize' => 4] : ['shardingEnabled' => false]); + $name = $prefix . $variant; + $this->assertSame(['n' => count($rows)], $db->insert($name, $rows)); + if ($indexed) { + $this->assertTrue($db->createFieldIndex($name, $indexField)['success']); + } + $this->assertSame($sharded, $db->getShardInfo($name)['sharded'], "$name sharded"); + $setups[$name] = $db; + } + return $setups; + } + + /** Operands for every operator: every kind of value, both sides of each rule */ + private static function operatorCases(): array + { + $cases = []; + foreach (['$gt', '$gte', '$lt', '$lte'] as $op) { + foreach ([5, 5.5, '5', '30', '1e1', 'abc', 'B', '', 0, -1, null, true, false, [1], NAN, '0x1A', ' 7', '-0'] as $operand) { + $cases[] = [$op, $operand]; + } + } + foreach (['$eq', '$ne'] as $op) { + foreach ([5, '5', 5.0, 0, null, true, false, [1], [], 'abc', 'ABC', ['a' => 1], ['a', 'abc']] as $operand) { + $cases[] = [$op, $operand]; + } + } + $lists = [[5, 'abc'], [null], [true, false], [[1]], [], ['5', 5.0], [0.5, '.5'], + [1, 2, 3, 4, 5, 6, 7, 8, 'abc', null], // more than 8 values: a hash set + [5.0, 30, '30', 'b', [], -1, 0.5, ' 7', 'x', 'y', ['a' => 1]]]; + foreach (['$in', '$nin'] as $op) { + foreach ($lists as $operand) { + $cases[] = [$op, $operand]; + } + } + $cases[] = ['$exists', true]; + $cases[] = ['$exists', false]; + foreach (['a', 'A', '^a', 'c$', '^abc$', '5', '^-', '', '^', '$', '^$', 'b', '.', '1e', '~', 'x.y', '^x.y$', '0X', 5, 0.5] as $operand) { + $cases[] = ['$like', $operand]; + } + foreach (['^a', 'B$', '[0-9]', '^$', 'a|5', '\d+', '^-?\d+(\.\d+)?$', 'a~b', '\.', 'A~'] as $operand) { + $cases[] = ['$regex', $operand]; + } + foreach (['a', 'b', '', 1, '1', true, [1], 5, 'abc', 'A', 'y', null] as $operand) { + $cases[] = ['$contains', $operand]; + } + return $cases; + } + + // ====================================================================== + // Operators and filter methods (QUERY.md "Comparison Operators", "Filter Methods") + // ====================================================================== + + /** @test every operator against every kind of value, through where(), orWhere(), whereNot(), count(), exists(), first() */ + public function everyOperatorMatchesTheReferenceModelOnEveryStorageLayout(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + foreach ($this->setups($rows) as $name => $db) { + foreach (self::operatorCases() as [$op, $operand]) { + $spec = ['v' => [$op => $operand]]; + $label = $name . ' ' . $op . ' ' . var_export($operand, true); + $expected = self::keysOf(self::refWhere($records, [$spec])); + $this->assertSame($expected, self::keysOf($db->query($name)->where($spec)->get()), "where $label"); + $this->assertSame($expected, self::keysOf($db->query($name)->orWhere($spec)->get()), "orWhere $label"); + $this->assertSame(count($expected), $db->query($name)->where($spec)->count(), "count $label"); + $this->assertSame($expected !== [], $db->query($name)->where($spec)->exists(), "exists $label"); + $first = $db->query($name)->where($spec)->first(); + $this->assertSame($expected === [] ? null : $expected[0], $first === null ? null : $first['key'], "first $label"); + // whereNot(): a record is excluded when the field's operators all match: the complement + $complement = array_values(array_diff(self::keysOf($records), $expected)); + $this->assertSame($complement, self::keysOf($db->query($name)->whereNot($spec)->get()), "whereNot $label"); + } + } + } + + /** @test plain values compare with ===; arrays as a whole; find() and where() agree */ + public function plainValuesMatchStrictlyInFindAndWhere(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + foreach ($this->setups($rows) as $name => $db) { + foreach ([5, '5', 5.0, 0, '', null, true, false, [1], [], ['a' => 1], ['a', 'abc'], 'abc', 'ABC', 0.5, '.5', 'nothing'] as $value) { + $label = $name . ' ' . var_export($value, true); + $expected = self::keysOf(self::refWhere($records, [['v' => $value]])); + $this->assertSame($expected, self::keysOf($db->query($name)->where(['v' => $value])->get()), "where $label"); + $this->assertSame($expected, self::keysOf($db->find($name, ['v' => $value])), "find $label"); + $this->assertSame(count($expected), $db->count($name, ['v' => $value]), "noneDB::count $label"); + $this->assertSame($expected !== [], $db->exists($name, ['v' => $value]), "noneDB::exists $label"); + $complement = array_values(array_diff(self::keysOf($records), $expected)); + $this->assertSame($complement, self::keysOf($db->query($name)->whereNot(['v' => $value])->get()), "whereNot $label"); + } + // The documented examples: '25' is not 25, null matches only an existing null + $this->assertSame([22], self::keysOf($db->find($name, ['v' => null])), "$name: a missing field is not null"); + } + } + + /** @test whereIn()/whereNotIn() are strict membership: whereIn drops a missing field, whereNotIn keeps it (like $in/$nin) */ + public function whereInAndWhereNotInAreStrictMembership(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + $lists = [[5, 'abc'], [null], [true], [[1]], [], ['5', 5.0], [1, 2, 3, 4, 5, 6, 7, 8, 'abc', null], + [5.0, 30, '30', 'b', [], -1, 0.5, ' 7', 'x', 'y', ['a' => 1]], [5, 5, 5]]; + foreach ($this->setups($rows) as $name => $db) { + foreach ($lists as $list) { + $label = $name . ' ' . json_encode($list); + $in = self::keysOf(self::refWhere($records, [['v' => ['$in' => $list]]])); + $nin = self::keysOf(self::refWhere($records, [['v' => ['$nin' => $list]]])); + $this->assertSame($in, self::keysOf($db->query($name)->whereIn('v', $list)->get()), "whereIn $label"); + $this->assertSame($nin, self::keysOf($db->query($name)->whereNotIn('v', $list)->get()), "whereNotIn $label"); + $this->assertSame(count($in), $db->query($name)->whereIn('v', $list)->count(), "count whereIn $label"); + } + // The record without "v" is in every whereNotIn() result and in no whereIn() result + $this->assertContains(29, self::keysOf($db->query($name)->whereNotIn('v', [5])->get())); + $this->assertNotContains(29, self::keysOf($db->query($name)->whereIn('v', [null])->get())); + } + } + + /** @test like(), notLike(), $like and noneDB::like() agree; notLike() is the exact complement of like() */ + public function likeMethodsAgreeAndNotLikeIsTheComplement(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + foreach ($this->setups($rows) as $name => $db) { + foreach (['a', 'A', '^a', 'c$', '^abc$', '5', '^-', '', '^', '$', '^$', 'b', '.', '1e', '~', 'x.y', '^.', '0X', '^1a$'] as $pattern) { + $label = "$name like " . var_export($pattern, true); + $expected = self::keysOf(self::refWhere($records, [['v' => ['$like' => $pattern]]])); + $this->assertSame($expected, self::keysOf($db->query($name)->like('v', $pattern)->get()), $label); + $this->assertSame($expected, self::keysOf($db->like($name, 'v', $pattern)), "noneDB::$label"); + $this->assertSame(array_values(array_diff(self::keysOf($records), $expected)), + self::keysOf($db->query($name)->notLike('v', $pattern)->get()), "notLike: $label"); + } + // Booleans, null, arrays and missing fields never match, whatever the pattern ("" matches every string and number) + $this->assertSame([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], + self::keysOf($db->query($name)->like('v', '')->get()), "$name: like('') matches strings and numbers only"); + $this->assertSame([22, 23, 24, 25, 26, 27, 28, 29], self::keysOf($db->query($name)->notLike('v', '')->get())); + } + } + + /** @test between() is inclusive and type-aware; notBetween() returns every other record; noneDB::between() agrees */ + public function betweenAndNotBetweenFollowTheComparisonRules(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + $ranges = [[1, 10], ['1', '10'], [5, 5], ['5', 5.0], ['a', 'c'], ['A', 'b'], [0, 'z'], [null, 10], [-1, 5.5], + [10, 1], [NAN, 10], [0, INF], ['', 'zzz'], [true, 100], [-1e9, 1e9], ['0', '9']]; + foreach ($this->setups($rows) as $name => $db) { + foreach ($ranges as [$min, $max]) { + $label = $name . ' ' . var_export($min, true) . '..' . var_export($max, true); + $expected = []; + foreach ($records as $r) { + if (!array_key_exists('v', $r)) continue; + $lo = self::refCompare($r['v'], $min); + $hi = self::refCompare($r['v'], $max); + if ($lo !== null && $hi !== null && $lo >= 0 && $hi <= 0) { + $expected[] = $r['key']; + } + } + $this->assertSame($expected, self::keysOf($db->query($name)->between('v', $min, $max)->get()), "between $label"); + $this->assertSame($expected, self::keysOf($db->between($name, 'v', $min, $max)), "noneDB::between $label"); + $this->assertSame($expected, self::keysOf($db->query($name)->where(['v' => ['$gte' => $min, '$lte' => $max]])->get()), "\$gte+\$lte $label"); + $this->assertSame(array_values(array_diff(self::keysOf($records), $expected)), + self::keysOf($db->query($name)->notBetween('v', $min, $max)->get()), "notBetween $label"); + } + // The extra filter of noneDB::between() is a find() filter; an invalid one returns false (API.md) + // g = 0: keys 0 (5), 3 (-1), 6 (0.5), 9 ('1e1' = 10), 12 ('-0'), 15, 18, 21, 24, 27 (no numbers) + $this->assertSame([0, 9], self::keysOf($db->between($name, 'v', 1, 10, ['g' => 0])), "$name between + filter"); + $this->assertFalse($db->between($name, 'v', 1, 10, 'not a filter'), "$name between with an invalid filter"); + } + } + + // ====================================================================== + // Sorting (QUERY.md "sort()", "Top-k"; API.md sort()) + // ====================================================================== + + /** @test sort() follows the documented total order on every kind of value; ties keep key order, also with desc */ + public function sortFollowsTheTotalOrderWithStableTies(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + $orders = [[['v', 'asc']], [['v', 'desc']], [['g', 'asc'], ['v', 'desc']], [['g', 'desc'], ['v', 'asc']], + [['nothere', 'asc']], [['nothere', 'desc']], [['g', 'desc']]]; + foreach ($this->setups($rows) as $name => $db) { + foreach ($orders as $keys) { + $q = $db->query($name); + foreach ($keys as [$field, $order]) { + $q->sort($field, $order); + } + $this->assertSame(self::keysOf(self::refSort($records, $keys)), self::keysOf($q->get()), $name . ' ' . json_encode($keys)); + } + $q = $db->query($name)->orderBy('v', 'DESC'); + $this->assertSame(self::keysOf(self::refSort($records, [['v', 'desc']])), self::keysOf($q->get()), "$name orderBy() alias, DESC in capitals"); + // noneDB::sort() on find() results + $this->assertSame(self::keysOf(self::refSort($records, [['v', 'desc']])), self::keysOf($db->sort($db->find($name, 0), 'v', 'desc')), "$name noneDB::sort desc"); + $this->assertSame(self::keysOf(self::refSort($records, [['v', 'asc']])), self::keysOf($db->sort($db->find($name, 0), 'v')), "$name noneDB::sort default asc"); + } + } + + /** @test the documented examples of the order, derived by hand (QUERY.md: "9" < "10" < "1e3"; "1a" < "Z" < "a") */ + public function sortOrderExamplesFromTheDocumentation(): void + { + $db = $this->db(); + // Inserted in a scrambled order: keys 0..13 + $values = ['a', [1], '1e3', true, 2.5, null, 'Z', '10', false, -1, '9', '1a', 0, self::MISSING, '9 ']; + $db->insert('ex', array_map(function ($v) { return $v === self::MISSING ? ['x' => 1] : ['v' => $v]; }, $values)); + $asc = [5, 13, 8, 3, 9, 12, 4, 10, 14, 7, 2, 11, 6, 0, 1]; // null, missing, false, true, -1, 0, 2.5, "9", "9 ", "10", "1e3", "1a", "Z", "a", [1] + $this->assertSame($asc, self::keysOf($db->query('ex')->sort('v')->get())); + // desc reverses the order; equal values ("9" and "9 " are both 9; null and missing) keep key order + $desc = [1, 0, 6, 11, 2, 7, 10, 14, 4, 12, 9, 3, 8, 5, 13]; + $this->assertSame($desc, self::keysOf($db->query('ex')->sort('v', 'desc')->get())); + $this->assertSame($desc, self::keysOf($db->sort($db->find('ex', 0), 'v', 'desc'))); + $this->assertSame($asc, self::keysOf($db->sort($db->find('ex', 0), 'v', 'asc'))); + // min()/max() use the same order (null and missing fields are not values) + $this->assertFalse($db->query('ex')->min('v')); + $this->assertSame([1], $db->query('ex')->max('v')); + $this->assertFalse($db->min('ex', 'v')); + $this->assertSame([1], $db->max('ex', 'v')); + // API.md: max([50, "10"]) is "10" because numeric strings sort after numbers + $db->insert('mx', [['v' => 50], ['v' => '10']]); + $this->assertSame('10', $db->max('mx', 'v')); + $this->assertSame(50, $db->min('mx', 'v')); + $this->assertSame('10', $db->query('mx')->max('v')); + // noneDB::sort(): [] for an empty array, false for a non-array (API.md) + $this->assertSame([], $db->sort([], 'v')); + $this->assertFalse($db->sort('x', 'v')); + $this->assertFalse($db->sort(null, 'v')); + } + + /** @test top-k (sort with offset + limit <= 1000) returns exactly the slice of the full sort, ties in key order */ + public function topKReturnsTheSameRowsAsAFullSort(): void + { + mt_srand(4242); + $pool = [null, true, false, 0, 1, -1, 2.5, 10, '10', '9', '1e1', 'abc', 'ABC', 'b', '', [1], ['a' => 1], 3.0, '1a', ' 7']; + $rows = []; + for ($i = 0; $i < 1500; $i++) { + $row = ['i' => $i, 'h' => mt_rand(0, 4)]; + if (mt_rand(0, 9) > 0) { + $row['v'] = $pool[mt_rand(0, count($pool) - 1)]; + } + $rows[] = $row; + } + $records = self::withKeys($rows); + $slices = [[0, 1], [0, 10], [7, 13], [990, 10], [0, 1000], [1, 999], [500, 500], [1, 1000], [0, 1001], [1490, 20], [3000, 5]]; + foreach ([['shardingEnabled' => false], ['shardSize' => 400]] as $n => $config) { + $db = $this->db($config); + $name = 'topk' . $n; + $db->insert($name, $rows); + foreach ([[['v', 'asc']], [['v', 'desc']], [['h', 'desc'], ['v', 'asc']]] as $keys) { + $full = self::keysOf(self::refSort($records, $keys)); + $filtered = self::keysOf(self::refSort(self::refWhere($records, [['h' => ['$gte' => 2]]]), $keys)); + foreach ($slices as [$offset, $limit]) { + $label = "$name " . json_encode($keys) . " offset $offset limit $limit"; + $q = $db->query($name); + $w = $db->query($name)->where(['h' => ['$gte' => 2]]); + foreach ($keys as [$field, $order]) { + $q->sort($field, $order); + $w->sort($field, $order); + } + $q->offset($offset)->limit($limit); + $w->offset($offset)->limit($limit); + $this->assertSame($offset + $limit <= 1000 ? 'top-k' : 'full', $q->explain()['sort'], $label); + $this->assertSame(array_slice($full, $offset, $limit), self::keysOf($q->get()), $label); + $this->assertSame(array_slice($filtered, $offset, $limit), self::keysOf($w->get()), "filtered $label"); + } + } + } + } + + // ====================================================================== + // Aggregates (QUERY.md "Aggregation"; API.md "Aggregation") + // ====================================================================== + + /** @test sum/avg/min/max/distinct/count of the builder and of noneDB, on every kind of value */ + public function aggregatesFollowTheDocumentedRules(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + // Reference: numbers and numeric strings are numbers; booleans, other strings, null, arrays are not + $numbers = function (array $records) { + $out = []; + foreach ($records as $r) { + if (array_key_exists('v', $r) && self::refIsNumber($r['v'])) { + $out[] = self::refToNumber($r['v']); + } + } + return $out; + }; + $present = function (array $records) { + return array_values(array_filter($records, function ($r) { return isset($r['v']); })); + }; + $distinct = function (array $records) { + $out = []; + foreach ($records as $r) { + if (isset($r['v']) && !in_array($r['v'], $out, true)) { + $out[] = $r['v']; + } + } + return $out; + }; + // The reference itself, by hand: 5+30+0-1+5.5+5.0+0.5+5+30+7+10+0.5+0 = 97.5 over 13 numbers + $this->assertSame(97.5, (float)array_sum($numbers($records))); + $this->assertCount(13, $numbers($records)); + + foreach ($this->setups($rows) as $name => $db) { + $this->assertSame(97.5, $db->query($name)->sum('v'), "$name sum"); + $this->assertSame(7.5, $db->query($name)->avg('v'), "$name avg"); + $this->assertFalse($db->query($name)->min('v'), "$name min: booleans are the smallest values (null is no value)"); + $this->assertSame(['a' => 1], $db->query($name)->max('v'), "$name max: arrays are the largest, by their JSON"); + $this->assertSame($distinct($records), $db->query($name)->distinct('v'), "$name distinct"); + $this->assertSame(30, $db->query($name)->count(), "$name count"); + $this->assertSame(97.5, $db->sum($name, 'v'), "$name noneDB::sum"); + $this->assertSame(7.5, $db->avg($name, 'v'), "$name noneDB::avg"); + $this->assertFalse($db->min($name, 'v'), "$name noneDB::min"); + $this->assertSame(['a' => 1], $db->max($name, 'v'), "$name noneDB::max"); + $this->assertSame($distinct($records), $db->distinct($name, 'v'), "$name noneDB::distinct"); + $this->assertSame(30, $db->count($name), "$name noneDB::count"); + + foreach ([0, 1, 2] as $g) { + $subset = self::refWhere($records, [['g' => $g]]); + $nums = $numbers($subset); + $sum = (float)array_sum($nums); + $avg = count($nums) > 0 ? array_sum($nums) / count($nums) : 0.0; + $asc = self::refSort($present($subset), [['v', 'asc']]); + $desc = self::refSort($present($subset), [['v', 'desc']]); + $q = function () use ($db, $name, $g) { return $db->query($name)->where(['g' => $g]); }; + $label = "$name g=$g"; + $this->assertSame($sum, $q()->sum('v'), "sum $label"); + $this->assertSame((float)$avg, $q()->avg('v'), "avg $label"); + $this->assertSame($asc[0]['v'], $q()->min('v'), "min $label"); + $this->assertSame($desc[0]['v'], $q()->max('v'), "max $label"); + $this->assertSame($distinct($subset), $q()->distinct('v'), "distinct $label"); + $this->assertSame(count($subset), $q()->count(), "count $label"); + $this->assertNumber($sum, $db->sum($name, 'v', ['g' => $g]), "noneDB::sum $label"); + $this->assertNumber($avg, $db->avg($name, 'v', ['g' => $g]), "noneDB::avg $label"); + $this->assertSame($asc[0]['v'], $db->min($name, 'v', ['g' => $g]), "noneDB::min $label"); + $this->assertSame($desc[0]['v'], $db->max($name, 'v', ['g' => $g]), "noneDB::max $label"); + $this->assertSame(count($subset), $db->count($name, ['g' => $g]), "noneDB::count $label"); + } + + // Aggregates read the matching records before select()/except() + $this->assertSame(97.5, $db->query($name)->select(['i'])->sum('v'), "$name select + sum"); + $this->assertSame(7.5, $db->query($name)->except(['v'])->avg('v'), "$name except + avg"); + $this->assertFalse($db->query($name)->select(['g'])->min('v'), "$name select + min"); + $this->assertSame(['a' => 1], $db->query($name)->except(['v', 'g'])->max('v'), "$name except + max"); + $this->assertSame($distinct($records), $db->query($name)->select(['i'])->distinct('v'), "$name select + distinct"); + $this->assertSame(30, $db->query($name)->select(['i'])->count(), "$name select + count"); + + // No values at all: sum and avg 0.0, min/max null, distinct [] (no record has "nothere") + $this->assertSame(0.0, $db->query($name)->sum('nothere')); + $this->assertSame(0.0, $db->query($name)->avg('nothere')); + $this->assertNull($db->query($name)->min('nothere')); + $this->assertNull($db->query($name)->max('nothere')); + $this->assertSame([], $db->query($name)->distinct('nothere')); + $this->assertSame([], $db->distinct($name, 'nothere')); + // Only non-numeric values: nothing to add up + $this->assertSame(0.0, $db->query($name)->whereIn('v', ['abc', true, null, [1]])->sum('v')); + $this->assertSame(0.0, $db->query($name)->whereIn('v', ['abc', true, null, [1]])->avg('v')); + $this->assertNumber(0, $db->sum($name, 'v', ['v' => 'abc'])); + $this->assertNumber(0, $db->avg($name, 'v', ['v' => true])); + } + } + + /** @test distinct() keeps first appearance, types (1, 1.0, "1" are three values) and merges -0.0 with 0.0 */ + public function distinctIsTypedAndOrderedByFirstAppearance(): void + { + $db = $this->db(['shardSize' => 3]); + $db->insert('d', [['v' => '1'], ['v' => 1.0], ['v' => 1], ['v' => -0.0], ['v' => 0.0], ['v' => 0], ['v' => null], + ['x' => 1], ['v' => '1'], ['v' => [1, 2]], ['v' => [2, 1]], ['v' => [1, 2]], ['v' => ['a' => 1, 'b' => 2]], ['v' => ['b' => 2, 'a' => 1]]]); + $expected = ['1', 1.0, 1, -0.0, 0, [1, 2], [2, 1], ['a' => 1, 'b' => 2], ['b' => 2, 'a' => 1]]; + $this->assertSame($expected, $db->distinct('d', 'v')); + $this->assertSame($expected, $db->query('d')->distinct('v')); + // whereIn is strict too: 1.0 (key 1), -0.0 (3), 0.0 (4, === -0.0) and 0 (5); "1" and 1 are not 1.0 + $this->assertSame([1, 3, 4, 5], self::keysOf($db->query('d')->whereIn('v', [1.0, -0.0, 0])->get())); + $this->assertSame([1.0, -0.0, 0], $db->query('d')->whereIn('v', [1.0, -0.0, 0])->distinct('v')); + $minusZero = $db->query('d')->distinct('v')[3]; + $this->assertTrue(is_float($minusZero) && $minusZero == 0.0, '-0.0 and 0.0 are one value: the first one seen is kept'); + // Missing database: [] (autoCreateDB) / false (off), nothing is created + $this->assertSame([], $db->distinct('nodb', 'v')); + $this->assertFalse($this->db(['autoCreateDB' => false])->distinct('nodb', 'v')); + $this->assertSame([], glob($this->testDbDir . '*-nodb*')); + } + + // ====================================================================== + // count(), first(), last(), exists(), limit(0) (QUERY.md "count()", "limit() / offset()", Terminal Methods) + // ====================================================================== + + /** @test count() applies limit() and offset(), and counts the groups with groupBy() */ + public function countAppliesLimitOffsetAndGroupBy(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $n => $config) { + $db = $this->db($config); + $name = 'cnt' . $n; + $rows = []; + for ($i = 0; $i < 10; $i++) { + $rows[] = ['i' => $i, 'm' => $i < 6, 'c' => ['x', 'y', 'z'][$i % 3]]; + } + $db->insert($name, $rows); + $q = function () use ($db, $name) { return $db->query($name)->where(['m' => true]); }; // 6 matches + $this->assertSame(6, $q()->count()); + $this->assertSame(4, $q()->offset(2)->count(), 'QUERY.md: offset(2)->count() on 6 matches is 4'); + $this->assertSame(3, $q()->limit(3)->count()); + $this->assertSame(1, $q()->offset(5)->limit(3)->count()); + $this->assertSame(0, $q()->offset(6)->count()); + $this->assertSame(0, $q()->limit(0)->count()); + $this->assertSame(6, $q()->limit(100)->count()); + $this->assertSame(10, $db->query($name)->count()); + $this->assertSame(8, $db->query($name)->skip(2)->count(), 'skip() is offset()'); + $this->assertSame(3, $db->query($name)->limit(3)->count(), 'limit() without filters'); + $this->assertSame(3, $db->query($name)->groupBy('c')->count(), 'three groups'); + $this->assertSame(3, $q()->groupBy('c')->count(), 'matches 0..5: x, y, z'); + $this->assertSame(2, $q()->groupBy('c')->limit(2)->count(), 'limit applies to the groups'); + $this->assertSame(1, $db->query($name)->groupBy('c')->having('count', '>', 3)->count(), 'x has 4 records'); + $this->assertSame(10, $db->query($name)->having('count', '>', 100)->count(), 'having() without groupBy() is ignored'); + $this->assertSame(6, $q()->sort('i', 'desc')->count(), 'sort() changes nothing'); + $this->assertSame(6, $q()->select(['i'])->except(['m'])->count()); + } + } + + /** @test first(), last() and exists() of the builder follow sort(), offset() and limit(); limit(0) returns nothing */ + public function firstLastExistsFollowTheResultWindow(): void + { + $db = $this->db(['shardSize' => 4]); + $rows = []; + foreach ([30, 10, 50, 20, 40, 10, 60, 25, 35] as $i => $score) { + $rows[] = ['i' => $i, 'score' => $score, 'odd' => $i % 2 === 1]; + } + $db->insert('fl', $rows); + $q = function () use ($db) { return $db->query('fl'); }; + $this->assertSame(0, $q()->first()['key'], 'key order without sort()'); + $this->assertSame(8, $q()->last()['key']); + $this->assertSame(1, $q()->sort('score')->first()['key'], 'lowest score, ties (1 and 5) in key order'); + $this->assertSame(5, $q()->sort('score')->offset(1)->first()['key']); + $this->assertSame(6, $q()->sort('score')->last()['key']); + $this->assertSame(6, $q()->sort('score', 'desc')->first()['key']); + $this->assertSame(5, $q()->sort('score', 'desc')->last()['key'], 'desc: the tie 1, 5 keeps key order, so 5 is last'); + $this->assertSame(3, $q()->sort('score')->limit(3)->last()['key'], '10, 10, 20: the third is key 3'); + // odd keys 1 (10), 3 (20), 5 (10), 7 (25): by score 1, 5, 3, 7 + $this->assertSame(['i' => 5, 'score' => 10, 'odd' => true, 'key' => 5], $q()->where(['odd' => true])->sort('score')->offset(1)->first()); + $this->assertSame(['i' => 3, 'score' => 20, 'odd' => true, 'key' => 3], $q()->where(['odd' => true])->sort('score')->limit(3)->last()); + $this->assertNull($q()->where(['score' => 999])->first()); + $this->assertNull($q()->where(['score' => 999])->last()); + $this->assertFalse($q()->where(['score' => 999])->exists()); + $this->assertTrue($q()->where(['score' => 10])->exists()); + $this->assertTrue($q()->where(['score' => 10])->sort('i')->select(['i'])->exists()); + + // limit(0) returns nothing (QUERY.md) + $this->assertSame([], $q()->limit(0)->get()); + $this->assertNull($q()->limit(0)->first()); + $this->assertNull($q()->limit(0)->last()); + $this->assertFalse($q()->limit(0)->exists()); + $this->assertSame(0, $q()->limit(0)->count()); + $this->assertNull($q()->sort('score')->limit(0)->first()); + + // first() doesn't change the builder's own limit + $builder = $q()->limit(3); + $this->assertSame(0, $builder->first()['key']); + $this->assertSame([0, 1, 2], self::keysOf($builder->get())); + // offset past the end: nothing; exists() is about the same window as get() + $this->assertSame([], $q()->offset(9)->get()); + $this->assertNull($q()->offset(9)->first()); + $this->assertFalse($q()->offset(9)->exists()); + $this->assertTrue($q()->offset(8)->exists()); + $this->assertSame([8], self::keysOf($q()->offset(8)->get())); + } + + /** @test noneDB::first/last/exists/count with every kind of filter, key filters included (key-list order) */ + public function noneDbFirstLastExistsCountWithFilters(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $n => $config) { + $db = $this->db($config); + $name = 'fe' . $n; + $db->insert($name, [['n' => 'a', 't' => 1], ['n' => 'b', 't' => 2], ['n' => 'c', 't' => 1], ['n' => 'd', 't' => 2], ['n' => 'e', 't' => 1]]); + $this->assertSame(['n' => 'a', 't' => 1, 'key' => 0], $db->first($name)); + $this->assertSame(['n' => 'e', 't' => 1, 'key' => 4], $db->last($name)); + $this->assertSame(['n' => 'b', 't' => 2, 'key' => 1], $db->first($name, ['t' => 2])); + $this->assertSame(['n' => 'd', 't' => 2, 'key' => 3], $db->last($name, ['t' => 2])); + // A key list keeps its order (API.md find(): "in the order of a key list") + $this->assertSame(['n' => 'd', 't' => 2, 'key' => 3], $db->first($name, ['key' => [3, '1']])); + $this->assertSame(['n' => 'b', 't' => 2, 'key' => 1], $db->last($name, ['key' => [3, '1']])); + $this->assertSame(2, $db->count($name, ['key' => [3, '1', 3, 99]])); + $this->assertSame(1, $db->count($name, ['key' => '4', 't' => 1])); + $this->assertSame(0, $db->count($name, ['key' => '4', 't' => 2])); + $this->assertTrue($db->exists($name, ['key' => 2])); + $this->assertTrue($db->exists($name, ['key' => '2', 'n' => 'c'])); + $this->assertFalse($db->exists($name, ['key' => 2, 'n' => 'x'])); + $this->assertFalse($db->exists($name, ['key' => 99])); + $this->assertNull($db->first($name, ['key' => 99])); + $this->assertNull($db->first($name, ['t' => 3])); + $this->assertNull($db->last($name, ['t' => 3])); + $this->assertFalse($db->exists($name, ['t' => 3])); + $this->assertTrue($db->exists($name, [])); + $this->assertTrue($db->exists($name, 0)); + $this->assertSame(5, $db->count($name, 0)); + $this->assertSame(5, $db->count($name, null)); + $this->assertSame(5, $db->count($name, [])); + // Invalid filters (API.md: count() counts 0 for a filter find() rejects) + foreach (['x', false, 1.5, true] as $invalid) { + $label = $name . ' ' . var_export($invalid, true); + $this->assertFalse($db->find($name, $invalid), $label); + $this->assertSame(0, $db->count($name, $invalid), $label); + $this->assertNull($db->first($name, $invalid), $label); + $this->assertNull($db->last($name, $invalid), $label); + $this->assertFalse($db->exists($name, $invalid), $label); + $this->assertNull($db->min($name, 't', $invalid), $label); + $this->assertNull($db->max($name, 't', $invalid), $label); + $this->assertNumber(0, $db->sum($name, 't', $invalid), $label); + $this->assertNumber(0, $db->avg($name, 't', $invalid), $label); + $this->assertFalse($db->between($name, 't', 0, 5, $invalid), $label); + } + } + } + + /** @test reads of a missing database return the documented empty values and create nothing */ + public function readsOfAMissingDatabaseCreateNothing(): void + { + foreach ([true, false] as $autoCreate) { + $db = $this->db(['autoCreateDB' => $autoCreate]); + $before = glob($this->testDbDir . '*'); + $this->assertSame($autoCreate ? [] : false, $db->find('ghost', 0)); + $this->assertSame($autoCreate ? [] : false, $db->find('ghost', ['a' => 1])); + $this->assertSame($autoCreate ? [] : false, $db->find('ghost', ['key' => 1])); + $this->assertSame(0, $db->count('ghost')); + $this->assertSame(0, $db->count('ghost', ['a' => 1])); + $this->assertSame(0, $db->count('ghost', ['key' => [1, 2]])); + $this->assertNull($db->first('ghost')); + $this->assertNull($db->first('ghost', ['key' => 0])); + $this->assertNull($db->last('ghost')); + $this->assertFalse($db->exists('ghost', ['a' => 1])); + $this->assertFalse($db->exists('ghost', ['key' => 0])); + $this->assertNumber(0, $db->sum('ghost', 'a')); + $this->assertNumber(0, $db->avg('ghost', 'a')); + $this->assertNull($db->min('ghost', 'a')); + $this->assertNull($db->max('ghost', 'a')); + $this->assertSame([], $db->query('ghost')->get()); + $this->assertSame(0, $db->query('ghost')->count()); + $this->assertSame(0, $db->query('ghost')->where(['a' => 1])->count()); + $this->assertNull($db->query('ghost')->first()); + $this->assertFalse($db->query('ghost')->exists()); + $this->assertSame(0.0, $db->query('ghost')->sum('a')); + $this->assertSame([], $db->query('ghost')->distinct('a')); + $this->assertSame([], $db->query('ghost')->groupBy('a')->get()); + $this->assertSame($before, glob($this->testDbDir . '*'), 'autoCreateDB ' . var_export($autoCreate, true) . ': no file was created'); + // Writes that find nothing to write report 0 and don't create the database + $this->assertSame(['n' => 0], $db->query('ghost')->update(['a' => 1])); + $this->assertSame(['n' => 0], $db->query('ghost')->delete()); + $this->assertSame(['n' => 0], $db->update('ghost', [['a' => 1], ['set' => ['b' => 2]]])); + $this->assertSame(['n' => 0], $db->delete('ghost', ['a' => 1])); + $this->assertFalse($db->getDBs('ghost')); + $this->assertNotContains('ghost', $db->getDBs(false)); + // Not created: no FORMAT, no data (an empty location holding only LOCK is absent, STORAGE_FORMAT.md §8.3) + $this->assertFileDoesNotExist($this->dbPath('ghost', 'format')); + $this->assertFileDoesNotExist($this->dbPath('ghost', 'data')); + } + } + + // ====================================================================== + // groupBy() / having() (QUERY.md "Grouping & Having") + // ====================================================================== + + /** Field order inside a record is not part of the contract: compare with string keys sorted */ + private static function canon($v) + { + if (!is_array($v)) { + return $v; + } + $out = []; + foreach ($v as $k => $x) { + $out[$k] = self::canon($x); + } + if ($v !== [] && array_keys($v) !== range(0, count($v) - 1)) { + ksort($out, SORT_STRING); + } + return $out; + } + + private static function groupRows(): array + { + return [ + ['c' => 1, 'p' => 10], // 0 + ['c' => '1', 'p' => 20], // 1 + ['c' => 1.5, 'p' => 'x'], // 2 + ['c' => 1.0, 'p' => 5], // 3 + ['c' => null, 'p' => 1], // 4 + ['p' => 2], // 5 no "c": the null group + ['c' => -0.0, 'p' => 3], // 6 + ['c' => 0.0, 'p' => 4], // 7 same group as -0.0 + ['c' => 0, 'p' => 6], // 8 the integer 0 is another value + ['c' => true, 'p' => 7], // 9 + ['c' => 1, 'p' => 30], // 10 + ['c' => [1], 'p' => '8'], // 11 + ['c' => 'x', 'p' => true], // 12 + ['c' => [1], 'p' => 9.5], // 13 + ]; + } + + /** @test groupBy(): values keep their type, missing = null, -0.0 = 0.0; _items/_count; having() with every aggregate and operator */ + public function groupByAndHavingFollowTheDocumentedRules(): void + { + $rows = self::groupRows(); + $records = self::withKeys($rows); + // By hand: group value => member keys + $groups = [[1, [0, 10]], ['1', [1]], [1.5, [2]], [1.0, [3]], [null, [4, 5]], [-0.0, [6, 7]], [0, [8]], [true, [9]], [[1], [11, 13]], ['x', [12]]]; + $expectedRows = function (array $which) use ($groups, $records) { + $out = []; + foreach ($which as $i) { + [$value, $keys] = $groups[$i]; + $out[] = ['_group' => $value, '_items' => array_map(function ($k) use ($records) { return $records[$k]; }, $keys), '_count' => count($keys)]; + } + return $out; + }; + foreach ([['shardingEnabled' => false], ['shardSize' => 4]] as $n => $config) { + $db = $this->db($config); + $name = 'grp' . $n; + $db->insert($name, $rows); + $q = function () use ($db, $name) { return $db->query($name)->groupBy('c'); }; + $all = $q()->get(); + $this->assertSame(self::canon($expectedRows(range(0, 9))), self::canon($all), "$name groups in order of first appearance"); + $this->assertSame([1, '1', 1.5, 1.0, null, -0.0, 0, true, [1], 'x'], array_column($all, '_group')); + $this->assertTrue(is_float($all[5]['_group']), '-0.0 and 0.0 are one float group'); + + // having(): count, sum/avg/min/max of the numeric values of the group's items + // sums 40, 20, 0, 5, 3, 7, 6, 7, 17.5, 0; mins 10, 20, -, 5, 1, 3, 6, 7, 8, 9.5... (see each case) + $cases = [ + [['count', '>', 1], [0, 4, 5, 8]], + [['count', '!=', 1], [0, 4, 5, 8]], + [['count', '<', 2], [1, 2, 3, 6, 7, 9]], + [['count', '=', 2], [0, 4, 5, 8]], + [['count', '<=', 1], [1, 2, 3, 6, 7, 9]], + [['count', '>=', 3], []], + [['sum:p', '>=', 30], [0]], + [['sum:p', '>', 0], [0, 1, 3, 4, 5, 6, 7, 8]], + [['sum:p', '=', 0], [2, 9]], + [['avg:p', '=', 20], [0, 1]], + [['avg:p', '<', 4], [2, 4, 5, 9]], + [['min:p', '<', 5], [4, 5]], + [['min:p', '>=', 8], [0, 1, 8]], + [['max:p', '<=', 6], [3, 4, 5, 6]], + [['max:p', '=', 9.5], [8]], + ]; + foreach ($cases as [[$aggregate, $operator, $value], $which]) { + $label = "$name having($aggregate $operator " . var_export($value, true) . ')'; + $this->assertSame(array_map(function ($i) use ($groups) { return $groups[$i][0]; }, $which), + array_column($q()->having($aggregate, $operator, $value)->get(), '_group'), $label); + $this->assertSame(count($which), $q()->having($aggregate, $operator, $value)->count(), "count $label"); + $this->assertSame($which !== [], $q()->having($aggregate, $operator, $value)->exists(), "exists $label"); + } + // Several having() calls all apply + $this->assertSame([1, -0.0, [1]], array_column($q()->having('count', '>', 1)->having('sum:p', '>', 5)->get(), '_group')); + // Unknown aggregates and operators are ignored + foreach ([['median:p', '>', 0], ['sum', '>', 1000], ['count', '<>', 1], ['count', '==', 1], ['sum:p', 'gt', 1000]] as [$aggregate, $operator, $value]) { + $this->assertSame(10, count($q()->having($aggregate, $operator, $value)->get()), "$name ignored: $aggregate $operator"); + } + // Without groupBy() having() is ignored: records, not groups + $this->assertSame(self::keysOf($records), self::keysOf($db->query($name)->having('count', '>', 100)->get())); + $this->assertTrue($db->query($name)->having('count', '>', 100)->exists()); + // Groups are rows: sort() sorts them (stable), limit()/offset() cut them, first()/last() pick them + $this->assertSame([1, null, -0.0, [1], '1', 1.5, 1.0, 0, true, 'x'], array_column($q()->sort('_count', 'desc')->get(), '_group')); + $this->assertSame([1.5, 1.0], array_column($q()->offset(2)->limit(2)->get(), '_group')); + $this->assertSame('x', $q()->last()['_group']); + $this->assertSame(1, $q()->first()['_group']); + $this->assertSame(10, $q()->count()); + // Grouping happens after the filters + $this->assertSame([1, 1.0], array_column($q()->where(['p' => ['$gte' => 5, '$lte' => 10]])->whereIn('c', [1, 1.0])->get(), '_group')); + $this->assertSame([], $q()->where(['p' => 'nothing'])->get()); + $this->assertFalse($q()->where(['p' => 'nothing'])->exists()); + $this->assertFalse($q()->having('count', '>', 5)->exists(), 'having() can drop every group'); + } + } + + /** @test having() with an unknown operator is ignored for every group, also groups whose min/max has no value */ + public function havingWithAnUnknownOperatorIsIgnoredForEveryGroup(): void + { + $db = $this->db(); + $db->insert('hu', self::groupRows()); + // Groups 1.5 and "x" have no numeric "p": their min/max is no value, but the condition is ignored as a whole + foreach (['gt', '<>', '=='] as $operator) { + $this->assertSame(10, count($db->query('hu')->groupBy('c')->having('max:p', $operator, 100)->get()), "max:p $operator"); + $this->assertSame(10, count($db->query('hu')->groupBy('c')->having('min:p', $operator, 0)->get()), "min:p $operator"); + } + } + + /** @test select()/except() with groupBy(): _group/_count/_items shape the rows, other fields the items (key kept) */ + public function groupByWithSelectAndExcept(): void + { + $db = $this->db(['shardSize' => 4]); + $db->insert('gp', [['s' => 'pending', 'total' => 12.5, 'x' => 1], ['s' => 'done', 'total' => 3, 'x' => 2], ['s' => 'pending', 'total' => 7, 'x' => 3]]); + $q = function () use ($db) { return $db->query('gp')->groupBy('s'); }; + // The documented example: select(['_group', 'total']) + $this->assertSame(self::canon([ + ['_group' => 'pending', '_items' => [['total' => 12.5, 'key' => 0], ['total' => 7, 'key' => 2]]], + ['_group' => 'done', '_items' => [['total' => 3, 'key' => 1]]], + ]), self::canon($q()->select(['_group', 'total'])->get())); + // Only item fields: the rows keep _group, _count and _items + $this->assertSame(self::canon([ + ['_group' => 'pending', '_count' => 2, '_items' => [['x' => 1, 'key' => 0], ['x' => 3, 'key' => 2]]], + ['_group' => 'done', '_count' => 1, '_items' => [['x' => 2, 'key' => 1]]], + ]), self::canon($q()->select(['x'])->get())); + $this->assertSame([['_count' => 2], ['_count' => 1]], $q()->select(['_count'])->get()); + $this->assertSame(self::canon([['_group' => 'pending', '_count' => 2], ['_group' => 'done', '_count' => 1]]), + self::canon($q()->except(['_items'])->get())); + $this->assertSame(self::canon([ + ['_group' => 'pending', '_items' => [['s' => 'pending', 'x' => 1, 'key' => 0], ['s' => 'pending', 'x' => 3, 'key' => 2]]], + ['_group' => 'done', '_items' => [['s' => 'done', 'x' => 2, 'key' => 1]]], + ]), self::canon($q()->except(['_count', 'total'])->get())); + $this->assertSame(self::canon([ + ['_group' => 'pending', '_count' => 2, '_items' => [['s' => 'pending', 'total' => 12.5, 'key' => 0], ['s' => 'pending', 'total' => 7, 'key' => 2]]], + ['_group' => 'done', '_count' => 1, '_items' => [['s' => 'done', 'total' => 3, 'key' => 1]]], + ]), self::canon($q()->except(['x'])->get())); + // The projection doesn't change what having() and the aggregates see + $this->assertSame(['pending'], array_column($q()->select(['_group'])->having('sum:total', '>', 15)->get(), '_group')); + // Stored records are untouched + $this->assertSame([['s' => 'pending', 'total' => 12.5, 'x' => 1, 'key' => 0]], $db->find('gp', ['key' => 0])); + } + + /** @test the groupBy() identity doesn't depend on serialize_precision or precision */ + public function groupIdentityIgnoresPrecisionSettings(): void + { + $db = $this->db(); + $db->insert('pr', [['v' => 0.1 + 0.2], ['v' => 0.3], ['v' => 1 / 3], ['v' => 0.3333333333333333], ['v' => 1e-20], ['v' => 1.00000000000000001e-20]]); + $saved = [ini_get('serialize_precision'), ini_get('precision')]; + try { + foreach ([['3', '3'], ['17', '17'], ['-1', '14']] as [$serialize, $precision]) { + ini_set('serialize_precision', $serialize); + ini_set('precision', $precision); + $groups = $db->query('pr')->groupBy('v')->get(); + // 0.1 + 0.2 is not 0.3; 1/3 and 0.3333333333333333 are the same double; the two 1e-20 literals are one double + $this->assertSame([[0], [1], [2, 3], [4, 5]], array_map(function ($g) { return self::keysOf($g['_items']); }, $groups), "precision $serialize/$precision"); + $this->assertSame([0.1 + 0.2, 0.3, 1 / 3], $db->query('pr')->whereIn('v', [0.1 + 0.2, 0.3, 1 / 3])->distinct('v')); + } + } finally { + ini_set('serialize_precision', (string)$saved[0]); + ini_set('precision', (string)$saved[1]); + } + } + + // ====================================================================== + // select() / except() (QUERY.md "select() / except()") + // ====================================================================== + + /** @test select() keeps only the given fields plus key; except() drops fields; neither affects filters, sort or aggregates */ + public function selectAndExceptShapeOnlyTheResult(): void + { + $rows = self::mixedRows(); + $records = self::withKeys($rows); + foreach ($this->setups($rows) as $name => $db) { + $expected = array_map(function ($r) { + $out = []; + if (array_key_exists('v', $r)) { + $out['v'] = $r['v']; + } + return $out + ['key' => $r['key']]; + }, $records); + $this->assertSame(self::canon($expected), self::canon($db->query($name)->select(['v'])->get()), "$name select v"); + $this->assertSame(self::canon(array_map(function ($r) { return ['i' => $r['i'], 'key' => $r['key']]; }, $records)), + self::canon($db->query($name)->select(['i', 'nothere'])->get()), "$name select: a missing field is left out"); + $this->assertSame(array_map(function ($r) { return ['key' => $r['key']]; }, $records), + $db->query($name)->select(['key'])->get(), "$name select key"); + $this->assertSame(self::canon(array_map(function ($r) { return ['i' => $r['i'], 'key' => $r['key']]; }, $records)), + self::canon($db->query($name)->except(['v', 'g', 'nothere'])->get()), "$name except"); + // Filters and sort see every field + $subset = self::refSort(self::refWhere($records, [['g' => 1]]), [['v', 'desc']]); + $this->assertSame(array_map(function ($r) { return ['i' => $r['i'], 'key' => $r['key']]; }, $subset), + array_map(function ($r) { ksort($r); return $r; }, $db->query($name)->select(['i'])->where(['g' => 1])->sort('v', 'desc')->get()), "$name select + where + sort"); + $this->assertSame(self::keysOf($subset), self::keysOf($db->query($name)->except(['v', 'g'])->where(['g' => 1])->sort('v', 'desc')->get())); + } + } + + /** @test except(['key']) keeps key: "key is always kept" (QUERY.md select() / except()) */ + public function exceptRemovesKeyOnlyWhenListed(): void + { + // QUERY.md select() / except(): select() always keeps key; except() removes key only when it is + // listed + $db = $this->db(); + $db->insert('ek', [['a' => 1, 'b' => 2], ['a' => 3, 'b' => 4]]); + $this->assertSame([['b' => 2, 'key' => 0], ['b' => 4, 'key' => 1]], $db->query('ek')->except(['a'])->get()); + $this->assertSame([['b' => 2], ['b' => 4]], $db->query('ek')->except(['key', 'a'])->get()); + $this->assertSame([[], []], $db->query('ek')->except(['a', 'b', 'key'])->get()); + $this->assertSame([['a' => 1, 'key' => 0], ['a' => 3, 'key' => 1]], $db->query('ek')->select(['a'])->get()); + $this->assertSame([['key' => 0], ['key' => 1]], $db->query('ek')->select(['key'])->get()); + } + + /** @test _distance: withDistance()'s is kept by select(), withinDistance()/nearest()'s only when selected; except() drops it */ + public function distanceFieldAndProjection(): void + { + $db = $this->db(); + $db->insert('pd', [ + ['n' => 'a', 'loc' => ['type' => 'Point', 'coordinates' => [0, 0]]], + ['n' => 'b', 'loc' => ['type' => 'Point', 'coordinates' => [0.01, 0]]], + ['n' => 'c', 'loc' => ['type' => 'Point', 'coordinates' => [0.02, 0]]], + ['n' => 'd', 'loc' => ['type' => 'Point', 'coordinates' => [1, 0]]], + ]); + // On the equator the distance is R * longitude difference (radians), R = 6371000 m (API.md: 0.01 degrees ~ 839.2 m at 41 N) + $meters = function (float $deg) { return 6371000.0 * deg2rad($deg); }; + $within = function () use ($db) { return $db->query('pd')->withinDistance('loc', 0, 0, 5000); }; + $this->assertSame(self::canon([['n' => 'a', 'key' => 0], ['n' => 'b', 'key' => 1], ['n' => 'c', 'key' => 2]]), + self::canon($within()->select(['n'])->get()), 'select() drops the _distance of withinDistance()'); + $selected = $within()->select(['n', '_distance'])->get(); + $this->assertSame(['a', 'b', 'c'], array_column($selected, 'n')); + $this->assertEqualsWithDelta([0.0, $meters(0.01), $meters(0.02)], array_column($selected, '_distance'), 0.011, 'meters, 2 decimals'); + foreach ($within()->get() as $row) { + $this->assertArrayHasKey('_distance', $row); + } + foreach ($within()->except(['_distance'])->get() as $row) { + $this->assertArrayNotHasKey('_distance', $row); + $this->assertArrayHasKey('loc', $row); + } + $withDistance = $db->query('pd')->withDistance('loc', 0.02, 0)->select(['n'])->get(); + $this->assertSame(['a', 'b', 'c', 'd'], array_column($withDistance, 'n')); + $this->assertEqualsWithDelta([$meters(0.02), $meters(0.01), 0.0, $meters(0.98)], array_column($withDistance, '_distance'), 0.0002, + 'withDistance(): kept by select(), 4 decimals, measured from its own point'); + foreach ($db->query('pd')->nearest('loc', 0, 0, 2)->select(['n'])->get() as $row) { + $this->assertArrayNotHasKey('_distance', $row, 'nearest(): dropped unless selected'); + } + $this->assertSame(['a', 'b'], array_column($db->query('pd')->nearest('loc', 0, 0, 2)->select(['n', '_distance'])->get(), 'n')); + // search() without a field list doesn't look into _distance + $this->assertSame([], $within()->search('1111')->get()); + } + + // ====================================================================== + // join() (QUERY.md "Joins") + // ====================================================================== + + /** @test join on key: whole numbers (5, 5.0, "5") match a record key; "00", 1.5, booleans, null, arrays don't */ + public function joinOnKeyMatchesWholeNumbersOnly(): void + { + $refs = [0, '0', 0.0, -0.0, '00', 1.0, '1', 1.5, true, false, null, [0], 2, 3, '-0', ' 1', '1 ', 'u0', 999.0, self::MISSING]; + // By hand (QUERY.md: integers, integral floats and canonical integer strings match; others don't) + $expected = [0, 0, 0, 0, null, 1, 1, null, null, null, null, null, 2, null, null, null, null, null, null, null]; + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $n => $config) { + $db = $this->db($config); + $users = 'users' . $n; + $orders = 'orders' . $n; + $db->insert($users, [['name' => 'u0'], ['name' => 'u1'], ['name' => 'u2']]); + $db->insert($orders, array_map(function ($r) { return $r === self::MISSING ? ['x' => 1] : ['user_id' => $r]; }, $refs)); + $userRecords = $db->find($users, 0); + $want = array_map(function ($k) use ($userRecords) { return $k === null ? null : $userRecords[$k]; }, $expected); + $joined = $db->query($orders)->join($users, 'user_id', 'key')->get(); + $this->assertSame($want, array_column($joined, $users), "$users: attached under the database name"); + $this->assertSame($want, array_column($db->query($orders)->join($users, 'user_id', 'key', 'buyer')->get(), 'buyer'), 'alias'); + $this->assertSame(array_slice($want, 3, 4), array_column($db->query($orders)->join($users, 'user_id', 'key')->offset(3)->limit(4)->get(), $users), 'joined after the slice'); + } + } + + private static function joinForeignRows(): array + { + return [ + ['id' => 5, 'v' => 'A'], ['id' => '5', 'v' => 'B'], ['id' => 5.0, 'v' => 'C'], ['id' => 1.5, 'v' => 'D'], + ['id' => '1.5', 'v' => 'E'], ['id' => true, 'v' => 'F'], ['id' => 1, 'v' => 'G'], ['id' => '007', 'v' => 'H'], + ['id' => 7, 'v' => 'I'], ['id' => null, 'v' => 'J'], ['id' => [5], 'v' => 'K'], ['id' => false, 'v' => 'L'], + ['id' => 0, 'v' => 'M'], ['id' => '', 'v' => 'N'], ['v' => 'O'], + ]; + } + + /** @test join on a field: 5/"5"/5.0 match each other (last duplicate wins), floats, booleans, "007", "", null, arrays per the rules */ + public function joinOnAFieldFollowsTheMatchingRules(): void + { + $refs = [5, '5', 5.0, 1.5, '1.5', true, 1, '1', 1.0, '007', 7, '7', null, [5], false, 0, '0', -0.0, '', 'nope', self::MISSING, '-0', ' 5']; + // By hand: the foreign key (index in joinForeignRows) each ref joins, null for none + $expected = [2, 2, 2, 3, 4, 5, 6, 6, 6, 7, 8, 8, null, null, 11, 12, 12, 12, 13, null, null, null, null]; + $local = array_map(function ($r) { return $r === self::MISSING ? ['f' => 'stored'] : ['ref' => $r, 'f' => 'stored']; }, $refs); + $variants = [['plain', ['shardingEnabled' => false], false], ['indexed', ['shardingEnabled' => false], true], ['sharded', ['shardSize' => 4], true]]; + foreach ($variants as [$variant, $config, $indexed]) { + $db = $this->db($config); + $foreign = 'fj' . $variant; + $db->insert($foreign, self::joinForeignRows()); + if ($indexed) { + $this->assertTrue($db->createFieldIndex($foreign, 'id')['success']); + } + $db->insert('lj', $local); + $foreignRecords = $db->find($foreign, 0); + $want = array_map(function ($k) use ($foreignRecords) { return $k === null ? null : $foreignRecords[$k]; }, $expected); + $this->assertSame($want, array_column($db->query('lj')->join($foreign, 'ref', 'id', 'f')->get(), 'f'), $variant); + // The attached record replaces the field in the result only + $this->assertSame(array_fill(0, count($refs), 'stored'), array_column($db->find('lj', 0), 'f'), "$variant: storage unchanged"); + // Sorting by the join alias: joins run first, the attached records sort as arrays (by their JSON), null first + $rows = $db->query('lj')->join($foreign, 'ref', 'id', 'f')->get(); + $q = $db->query('lj')->join($foreign, 'ref', 'id', 'f')->sort('f', 'desc')->limit(5); + $plan = $q->explain(); + $this->assertSame('full', $plan['sort']); + $this->assertSame('before-sort', $plan['joins'][0]['when']); + $this->assertSame('in-lookup', $plan['joins'][0]['strategy']); + $this->assertSame(self::keysOf(array_slice(self::refSort($rows, [['f', 'desc']]), 0, 5)), self::keysOf($q->get()), "$variant sort by alias"); + $late = $db->query('lj')->join($foreign, 'ref', 'id', 'f')->sort('ref')->limit(5)->explain(); + $this->assertSame('after-slice', $late['joins'][0]['when']); + $this->assertSame('top-k', $late['sort']); + $db->dropDB('lj'); + } + } + + /** @test joining a database that doesn't exist attaches null and creates nothing, with and without autoCreateDB */ + public function joinWithAMissingDatabaseAttachesNull(): void + { + foreach ([true, false] as $autoCreate) { + $db = $this->db(['autoCreateDB' => $autoCreate]); + $db->createDB('jl'); + $this->assertSame(['n' => 3], $db->insert('jl', [['ref' => 0], ['ref' => '1'], ['ref' => 'x']])); + $before = glob($this->testDbDir . '*'); + $this->assertSame([null, null, null], array_column($db->query('jl')->join('nowhere', 'ref', 'key')->get(), 'nowhere')); + $this->assertSame([null, null, null], array_column($db->query('jl')->join('nowhere', 'ref', 'id', 'w')->get(), 'w')); + $this->assertSame(3, $db->query('jl')->join('nowhere', 'ref', 'key')->count()); + $this->assertSame($before, glob($this->testDbDir . '*'), 'nothing created'); + $this->assertFalse($db->getDBs('nowhere')); + $db->dropDB('jl'); + } + } + + /** @test several joins, a join after where()/sort()/limit(), and count()/exists() with joins */ + public function multipleJoinsAndJoinsWithOtherClauses(): void + { + $db = $this->db(['shardSize' => 3]); + $db->insert('cust', [['name' => 'Ann'], ['name' => 'Bob']]); + $db->insert('prod', [['sku' => 'p1', 'title' => 'Pen'], ['sku' => 'p2', 'title' => 'Ink']]); + $db->insert('items', [ + ['c' => 0, 'p' => 'p2', 'qty' => 3], ['c' => 1, 'p' => 'p1', 'qty' => 1], ['c' => 0, 'p' => 'p1', 'qty' => 5], + ['c' => 2, 'p' => 'p3', 'qty' => 2], ['c' => 1, 'p' => 'p2', 'qty' => 4], + ]); + $rows = $db->query('items')->join('cust', 'c', 'key')->join('prod', 'p', 'sku')->where(['qty' => ['$gte' => 2]])->sort('qty', 'desc')->limit(3)->get(); + $this->assertSame([2, 4, 0], self::keysOf($rows)); + $this->assertSame(['Ann', 'Bob', 'Ann'], array_map(function ($r) { return $r['cust']['name']; }, $rows)); + $this->assertSame(['Pen', 'Ink', 'Ink'], array_map(function ($r) { return $r['prod']['title']; }, $rows)); + $all = $db->query('items')->join('cust', 'c', 'key')->join('prod', 'p', 'sku')->get(); + $this->assertNull($all[3]['cust'], 'no customer 2'); + $this->assertNull($all[3]['prod'], 'no product p3'); + $this->assertSame(5, $db->query('items')->join('cust', 'c', 'key')->count(), 'joins never filter'); + $this->assertSame(1, $db->query('items')->join('cust', 'c', 'key')->offset(4)->count()); + $this->assertTrue($db->query('items')->join('cust', 'c', 'key')->where(['c' => 2])->exists()); + // With groupBy() the joins run before grouping: the items carry the joined record + $groups = $db->query('items')->join('cust', 'c', 'key')->groupBy('c')->get(); + $this->assertSame([0, 1, 2], array_column($groups, '_group')); + $this->assertSame([['Ann', 'Ann'], ['Bob', 'Bob'], [null]], array_map(function ($g) { + return array_map(function ($item) { return $item['cust']['name'] ?? null; }, $g['_items']); + }, $groups)); + } + + // ====================================================================== + // Writes of the builder (QUERY.md update()/delete()/removeFields()) + // ====================================================================== + + /** @test sort(), offset() and limit() choose the records written; select(), except() and join() don't matter */ + public function builderWritesChooseRecordsBySortOffsetAndLimit(): void + { + $rows = [['n' => 'a', 's' => 30, 'grp' => 'x'], ['n' => 'b', 's' => 10, 'grp' => 'y'], ['n' => 'c', 's' => 50, 'grp' => 'x'], + ['n' => 'd', 's' => 20, 'grp' => 'y'], ['n' => 'e', 's' => 50, 'grp' => 'x'], ['n' => 'f', 's' => 40, 'grp' => 'y'], + ['n' => 'g', 's' => 10, 'grp' => 'x'], ['n' => 'h', 's' => 60, 'grp' => 'y']]; + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $i => $config) { + $db = $this->db($config); + $name = 'wr' . $i; + $db->insert($name, $rows); + $db->insert('other', [['n' => 'h', 'extra' => 1]]); + // desc by s: h(7) 60, c(2) 50, e(4) 50, f(5) 40, ...: the first two are 7 and 2 + $this->assertSame(['n' => 2], $db->query($name)->sort('s', 'desc')->limit(2)->select(['n'])->update(['top' => true])); + $this->assertSame([2, 7], self::keysOf($db->find($name, ['top' => true]))); + // grp x by s: g(6) 10, a(0) 30, c(2) 50, e(4) 50; offset 1, limit 2: 0 and 2 + $this->assertSame(['n' => 2], $db->query($name)->where(['grp' => 'x'])->sort('s')->offset(1)->limit(2)->except(['s'])->delete()); + $this->assertSame([1, 3, 4, 5, 6, 7], self::keysOf($db->find($name, 0))); + // n desc among b, d, e, f, g, h: h, g, f + $this->assertSame(['n' => 3, 'fields_removed' => ['s']], + $db->query($name)->sort('n', 'desc')->limit(3)->select(['n'])->join('other', 'n', 'n')->removeFields(['s', 'nothere'])); + $this->assertSame([1, 3, 4], self::keysOf($db->query($name)->where(['s' => ['$exists' => true]])->get())); + $this->assertSame(self::canon([['n' => 'h', 'grp' => 'y', 'top' => true, 'key' => 7]]), self::canon($db->find($name, ['key' => 7]))); + // Nothing selected: nothing written + $this->assertSame(['n' => 0], $db->query($name)->offset(10)->update(['z' => 1])); + $this->assertSame(['n' => 0], $db->query($name)->limit(0)->delete()); + $this->assertSame(['n' => 0], $db->query($name)->where(['n' => 'zz'])->update(['z' => 1])); + $this->assertSame([], $db->find($name, ['z' => 1])); + // An empty set changes nothing and reports the matches (API.md update()) + $this->assertSame(['n' => 3], $db->query($name)->where(['grp' => 'y'])->limit(3)->update([])); + // "key" can't be set + $this->assertSame(['n' => 0, 'error' => 'Please check your update parameters'], $db->query($name)->update(['key' => 5])); + $this->assertSame([1, 3, 4, 5, 6, 7], self::keysOf($db->find($name, 0))); + $db->dropDB('other'); + } + } + + /** @test update(), delete() and removeFields() with groupBy() throw LogicException and write nothing */ + public function builderWritesRejectGroupBy(): void + { + $db = $this->db(); + $db->insert('lg', [['g' => 1, 'a' => 1], ['g' => 2, 'a' => 2]]); + $before = $db->find('lg', 0); + $writes = [ + 'update' => function ($q) { return $q->update(['a' => 9]); }, + 'delete' => function ($q) { return $q->delete(); }, + 'removeFields' => function ($q) { return $q->removeFields(['a']); }, + ]; + foreach ($writes as $label => $write) { + try { + $write($db->query('lg')->groupBy('g')); + $this->fail("$label with groupBy() must throw"); + } catch (\LogicException $e) { + $this->assertSame(\LogicException::class, get_class($e), $label); + $this->assertSame('noneDB: update(), delete() and removeFields() cannot be combined with groupBy()', $e->getMessage()); + } + } + $this->assertSame($before, $db->find('lg', 0)); + } + + // ====================================================================== + // Error results and exceptions (API.md insert/update/delete/find; QUERY.md "How values compare") + // ====================================================================== + + /** @test insert(): the documented error results; nothing of a rejected call is written */ + public function insertErrorResults(): void + { + $db = $this->db(); + $db->insert('ins', ['a' => 1]); + $json = 'insert data cannot be encoded as JSON (invalid UTF-8, NAN/INF or unsupported value)'; + $cases = [ + ['not an array', 'insert data must be array'], + [5, 'insert data must be array'], + [null, 'insert data must be array'], + [[], 'insert data must not be empty'], + [[0 => ['a' => 1], 'name' => 'x'], 'insert data must be either one record or a list of records'], + [['key' => 'value'], 'You cannot set key name to key'], + [[['a' => 2], ['key' => 3]], 'You cannot set key name to key'], + [['v' => NAN], $json], + [['v' => INF], $json], + [['v' => "\xff\xfe"], $json], + [[['a' => 2], ['nested' => ['deep' => -INF]]], $json], + ]; + foreach ($cases as [$data, $error]) { + $this->assertSame(['n' => 0, 'error' => $error], $db->insert('ins', $data), var_export($data, true)); + } + $this->assertSame([['a' => 1, 'key' => 0]], $db->find('ins', 0), 'nothing written'); + // A nested "key" is fine, floats keep their type, keys continue + $this->assertSame(['n' => 2], $db->insert('ins', [['data' => ['key' => 'v']], ['f' => 2.0]])); + $this->assertSame(self::canon([['a' => 1, 'key' => 0], ['data' => ['key' => 'v'], 'key' => 1], ['f' => 2.0, 'key' => 2]]), self::canon($db->find('ins', 0))); + } + + /** @test update()/delete(): the documented error results; nothing is written */ + public function updateAndDeleteErrorResults(): void + { + $db = $this->db(); + $db->insert('ud', [['n' => 'a'], ['n' => 'b']]); + $bad = 'Please check your update parameters'; + foreach ([ + 'x', 5, null, [], [1 => ['set' => ['x' => 1]]], [['n' => 'a']], [['n' => 'a'], 'set'], [['n' => 'a'], ['set' => 'str']], + [['n' => 'a'], ['set' => ['key' => 5]]], ['n', ['set' => ['x' => 1]]], [['n' => 'a'], ['put' => ['x' => 1]]], [['n' => 'a'], ['set' => null]], + ] as $data) { + $this->assertSame(['n' => 0, 'error' => $bad], $db->update('ud', $data), var_export($data, true)); + } + $result = $db->update('ud', [['n' => 'a'], ['set' => ['x' => NAN]]]); + $this->assertSame(0, $result['n']); + $this->assertStringStartsWith('update data cannot be encoded as JSON', $result['error']); + $this->assertSame(['n' => 1], $db->update('ud', [['n' => 'a'], ['set' => []]]), 'empty set: the number of matches'); + foreach (['bad', 0, null, 1.5, true] as $data) { + $this->assertSame(['n' => 0, 'error' => 'Please check your delete parameters'], $db->delete('ud', $data), var_export($data, true)); + } + $this->assertSame([['n' => 'a', 'key' => 0], ['n' => 'b', 'key' => 1]], $db->find('ud', 0), 'nothing written'); + $this->assertSame(['n' => 2], $db->update('ud', [[], ['set' => ['all' => 1]]])); + $this->assertSame(['n' => 2], $db->delete('ud', []), 'an empty filter matches everything'); + $this->assertSame([], $db->find('ud', 0)); + } + + /** @test exceptions: mixed operator specs at where()/orWhere()/whereNot() call, the rest when the query runs (also on an empty or missing database) */ + public function filterExceptionsAreThrownAtTheDocumentedTime(): void + { + $db = $this->db(); + $db->insert('full', [['a' => 1], ['a' => 2]]); + $db->insert('empty', ['a' => 1]); + $db->delete('empty', []); + $mix = 'noneDB: a filter value cannot mix operators ($gt, $in, ...) with plain keys'; + foreach (['where', 'orWhere', 'whereNot'] as $method) { + try { + $db->query('full')->$method(['a' => ['$gt' => 1, 'b' => 2]]); + $this->fail("$method must reject a mixed spec when it is called"); + } catch (\InvalidArgumentException $e) { + $this->assertSame($mix, $e->getMessage(), $method); + } + } + $runners = [ + 'get' => function ($q) { return $q->get(); }, 'count' => function ($q) { return $q->count(); }, + 'first' => function ($q) { return $q->first(); }, 'last' => function ($q) { return $q->last(); }, + 'exists' => function ($q) { return $q->exists(); }, 'sum' => function ($q) { return $q->sum('a'); }, + 'avg' => function ($q) { return $q->avg('a'); }, 'min' => function ($q) { return $q->min('a'); }, + 'max' => function ($q) { return $q->max('a'); }, 'distinct' => function ($q) { return $q->distinct('a'); }, + 'update' => function ($q) { return $q->update(['z' => 1]); }, 'delete' => function ($q) { return $q->delete(); }, + 'removeFields' => function ($q) { return $q->removeFields(['a']); }, + ]; + $specs = [ + [['$gtee' => 1], "noneDB: unknown operator '\$gtee' (supported: \$gt, \$gte, \$lt, \$lte, \$ne, \$eq, \$in, \$nin, \$exists, \$like, \$regex, \$contains)"], + [['$in' => 'notarray'], 'noneDB: $in needs an array'], + [['$nin' => 5], 'noneDB: $nin needs an array'], + [['$regex' => '(unclosed'], "noneDB: invalid \$regex pattern '(unclosed'"], + [['$regex' => ''], 'noneDB: $regex needs a non-empty string pattern'], + [['$regex' => 5], 'noneDB: $regex needs a non-empty string pattern'], + [['$like' => ['x']], 'noneDB: like pattern must be a string'], + ]; + foreach (['full', 'empty', 'missing'] as $name) { + foreach ($specs as [$spec, $message]) { + foreach ($runners as $runner => $run) { + foreach (['where', 'orWhere'] as $method) { + $label = "$name $method " . json_encode($spec) . " $runner"; + $query = $db->query($name)->$method(['a' => $spec]); // no exception yet + try { + $run($query); + $this->fail("$label must throw"); + } catch (\InvalidArgumentException $e) { + $this->assertSame($message, $e->getMessage(), $label); + } + } + } + // whereNot() checks its operators when it is called + try { + $db->query($name)->whereNot(['a' => $spec]); + $this->fail("$name whereNot " . json_encode($spec)); + } catch (\InvalidArgumentException $e) { + $this->assertSame($message, $e->getMessage()); + } + } + } + $this->assertSame([['a' => 1, 'key' => 0], ['a' => 2, 'key' => 1]], $db->find('full', 0), 'failed writes wrote nothing'); + $this->assertFalse($db->getDBs('missing')); + // Invalid database names + foreach (['bad!', 'a/b', '', 'x_y'] as $bad) { + try { + $db->query($bad); + $this->fail("query('$bad')"); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString('invalid database name', $e->getMessage()); + } + } + } + + /** @test a $regex that exceeds PCRE's backtracking limit is reported, never taken as "no match" */ + public function regexRuntimeFailureIsReported(): void + { + $db = $this->db(); + $db->insert('rx', [['s' => str_repeat('a', 40) . 'b'], ['s' => 'aab']]); + $saved = ini_get('pcre.backtrack_limit'); + $jit = ini_get('pcre.jit'); + ini_set('pcre.backtrack_limit', '100'); + ini_set('pcre.jit', '0'); + try { + try { + $db->query('rx')->where(['s' => ['$regex' => '^(a+)+$']])->get(); + $this->fail('a failed regex match must throw'); + } catch (\RuntimeException $e) { + $this->assertNotInstanceOf(\InvalidArgumentException::class, $e); + $this->assertStringStartsWith('noneDB: regular expression failed', $e->getMessage()); + } + } finally { + ini_set('pcre.backtrack_limit', (string)$saved); + ini_set('pcre.jit', (string)$jit); + } + $this->assertSame([1], self::keysOf($db->query('rx')->where(['s' => ['$regex' => '^A{2}B$']])->get()), 'case-insensitive'); + } + + // ====================================================================== + // Key filters (QUERY.md: "A key filter ... means the same in every clause"; API.md find()) + // ====================================================================== + + /** @test key filters: integers and canonical digit strings only, in find(), where(), orWhere(), whereNot() and with spatial filters */ + public function keyFiltersMeanTheSameEverywhere(): void + { + $values = [5, '5', [5, '1'], [3, 3, '3'], [4, '0', 2], '05', ' 5', '5 ', "5\n", -1, '-1', 5.0, true, false, null, '', [], ['x', 2], 99, '+5', '5.0', '1e0']; + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $i => $config) { + $db = $this->db($config); + $name = 'kf' . $i; + $rows = []; + foreach (['a', 'b', 'c', 'd', 'e', 'f'] as $j => $n) { + $rows[] = ['n' => $n, 'loc' => ['type' => 'Point', 'coordinates' => [$j * 0.001, 0]]]; + } + $db->insert($name, $rows); + $all = [0, 1, 2, 3, 4, 5]; + foreach ($values as $value) { + $label = "$name key " . var_export($value, true); + $listed = array_values(array_filter(self::refKeyList($value), function ($k) { return $k <= 5; })); + $sorted = $listed; + sort($sorted); + $this->assertSame($listed, self::keysOf($db->find($name, ['key' => $value])), "find $label (key-list order)"); + $this->assertSame($listed, self::keysOf($db->query($name)->where(['key' => $value])->get()), "where $label"); + $this->assertSame(count($listed), $db->count($name, ['key' => $value]), "count $label"); + $this->assertSame(count($listed), $db->query($name)->where(['key' => $value])->count(), "builder count $label"); + $this->assertSame($sorted, self::keysOf($db->query($name)->where(['key' => $value])->where(['n' => ['$ne' => 'zz']])->get()), "two where() $label"); + $this->assertSame($sorted, self::keysOf($db->query($name)->orWhere(['key' => $value])->get()), "orWhere $label"); + $this->assertSame($sorted, self::keysOf($db->query($name)->where(['n' => 'zz'])->orWhere(['key' => $value])->get()), "where OR key $label"); + $this->assertSame(array_values(array_diff($all, $sorted)), self::keysOf($db->query($name)->whereNot(['key' => $value])->get()), "whereNot $label"); + $spatial = self::keysOf($db->query($name)->withinBBox('loc', -1, -1, 1, 1)->where(['key' => $value])->get()); + sort($spatial); + $this->assertSame($sorted, $spatial, "spatial + where $label"); + } + } + } + + /** @test $eq and $in on key mean what the plain key filter means ("$eq: explicit equality, same as simple value") */ + public function keyOperatorsCompareStrictlyThePlainKeyFilterNormalizes(): void + { + // QUERY.md "How values compare": a plain key filter means what it means in find() ("1" is key 1); + // $eq and $in compare strictly on key as on any field ("1" is not the integer key 1) + $db = $this->db(); + $db->insert('ko', [['n' => 'a'], ['n' => 'b'], ['n' => 'c']]); + $this->assertSame([1], self::keysOf($db->query('ko')->where(['key' => ['$eq' => 1]])->get())); + $this->assertSame([0, 2], self::keysOf($db->query('ko')->where(['key' => ['$in' => [2, 0]]])->get())); + $this->assertSame([1], self::keysOf($db->query('ko')->where(['key' => '1'])->get()), 'plain "1" is key 1'); + $this->assertSame([], self::keysOf($db->query('ko')->where(['key' => ['$eq' => '1']])->get()), '$eq is strict'); + $this->assertSame([2], self::keysOf($db->query('ko')->where(['key' => ['$in' => ['1', 2]]])->get()), '$in is strict'); + $this->assertSame([0, 2], self::keysOf($db->query('ko')->where(['key' => ['$ne' => 1]])->get()), '$ne is strict'); + $this->assertSame([0, 1, 2], self::keysOf($db->query('ko')->where(['key' => ['$ne' => '1']])->get())); + } + + // ====================================================================== + // like() / search() folding (QUERY.md "Pattern Matching", "search()") + // ====================================================================== + + private function setMbstringHook(?callable $hook): void + { + $property = new \noneDB\Tests\InternalValue('mbstringFn'); + $property->setValue(null, $hook); + } + + private function seedFolding(\noneDB $db, string $name): void + { + $db->insert($name, [ + ['t' => 'PHP Tutorial', 'body' => 'learn php', 'num' => 30, 'tags' => ['php', 'web']], // 0 + ['t' => 'tutorial on php', 'body' => 'x'], // 1 + ['t' => 'İzmir guide', 'body' => 'STRASSE'], // 2 + ['t' => 'Ankara', 'flag' => true, 'num' => 1], // 3 + ['t' => 'IZMIR', 'body' => 'straße'], // 4 + ['t' => 'izmir'], // 5 + ['t' => 'nothing', 'meta' => ['note' => 'php inside']], // 6 + ['t' => 'KIRMIZI', 'body' => 'Ğüneş'], // 7 + ]); + } + + /** @test search(): whole term, scalar values only, key only when listed, Unicode case folding (mbstring) */ + public function searchMatchesTheWholeTermWithUnicodeFolding(): void + { + if (!function_exists('mb_convert_case')) { + $this->markTestSkipped('needs mbstring (the no-mbstring rules are tested with the hook below)'); + } + $db = $this->db(['shardSize' => 3]); + $this->seedFolding($db, 'se'); + $s = function (string $term, array $fields = []) use ($db) { return self::keysOf($db->query('se')->search($term, $fields)->get()); }; + $this->assertSame([0], $s('php tutorial'), 'no word splitting: "tutorial on php" doesn\'t match'); + $this->assertSame([0, 1], $s('PHP'), 'nested arrays (tags, meta) are not searched'); + $this->assertSame([0], $s('php', ['body'])); + $this->assertSame([], $s('php', ['tags']), 'array values are not scalar values'); + $this->assertSame([0], $s('30'), 'numbers are searched'); + $this->assertSame([3], $s('1'), 'key 1 is not searched without a field list'); + $this->assertSame([2], $s('2', ['key']), 'key is searched when listed'); + $this->assertSame([2, 4, 5], $s('izmir')); + $this->assertSame([2, 4, 5], $s('İZMİR')); + $this->assertSame([2, 4], $s('strasse')); + $this->assertSame([2, 4], $s('STRAßE')); + $this->assertSame([7], $s('ğÜNEŞ')); + $this->assertSame([], $s('kırmızı'), 'dotless ı and I don\'t match each other'); + $this->assertSame([7], $s('kirmizi')); + $this->assertSame([0, 1, 2, 3, 4, 5, 6, 7], $s(''), 'the empty term is in every value'); + $this->assertSame([5], self::keysOf($db->query('se')->search('izmir')->where(['body' => ['$exists' => false]])->get())); + // 2: "İzmir guide"/"STRASSE", 4: "IZMIR"/"straße"; 5 has no body + $this->assertSame([2, 4], self::keysOf($db->query('se')->search('izmir')->search('strasse')->get()), 'several search() calls all apply'); + $this->assertSame([2, 4], self::keysOf($db->query('se')->search('ß', ['body'])->get()), 'ß folds to ss: "STRASSE" contains it'); + $this->assertSame([2], self::keysOf($db->query('se')->search('izmir')->search('strasse')->search('GUIDE')->get())); + } + + /** @test like()/$like/noneDB::like(): Unicode folding with mbstring; ß = ss, İ = i, ı != I */ + public function likeFoldsUnicodeCase(): void + { + if (!function_exists('mb_convert_case')) { + $this->markTestSkipped('needs mbstring (the no-mbstring rules are tested with the hook below)'); + } + $db = $this->db(); + $this->seedFolding($db, 'lf'); + $like = function (string $field, string $pattern) use ($db) { + $a = self::keysOf($db->query('lf')->like($field, $pattern)->get()); + $this->assertSame($a, self::keysOf($db->query('lf')->where([$field => ['$like' => $pattern]])->get()), "\$like $pattern"); + $this->assertSame($a, self::keysOf($db->like('lf', $field, $pattern)), "noneDB::like $pattern"); + return $a; + }; + $this->assertSame([2, 4, 5], $like('t', 'izmir')); + $this->assertSame([2, 4, 5], $like('t', '^İZM')); + $this->assertSame([4, 5], $like('t', 'İzmir$'), '"İzmir guide" doesn\'t end with it'); + $this->assertSame([2, 4], $like('body', 'strasse')); + $this->assertSame([2, 4], $like('body', '^STRAßE$')); + $this->assertSame([7], $like('body', 'ğüneş')); + $this->assertSame([], $like('t', 'kırmızı')); + $this->assertSame([7], $like('t', 'KİRMİZİ'), 'İ folds to i'); + } + + /** @test without mbstring: like() uses PCRE /iu (misses ß/ss and İ/i), search() folds only ASCII letters */ + public function foldingWithoutMbstring(): void + { + $this->setMbstringHook(function () { return false; }); + try { + $db = $this->db(); + $this->seedFolding($db, 'nm'); + $like = function (string $field, string $pattern) use ($db) { + $a = self::keysOf($db->query('nm')->like($field, $pattern)->get()); + $this->assertSame($a, self::keysOf($db->like('nm', $field, $pattern)), "noneDB::like $pattern"); + return $a; + }; + $this->assertSame([4, 5], $like('t', 'izmir'), 'İ is not i without mbstring'); + $this->assertSame([2], $like('body', 'strasse'), 'ß is not ss without mbstring'); + $this->assertSame([7], $like('body', 'ğüneş'), 'PCRE /iu folds Ğ/ğ'); + $this->assertSame([0, 1], $like('t', 'php')); + $s = function (string $term) use ($db) { return self::keysOf($db->query('nm')->search($term)->get()); }; + $this->assertSame([4, 5], $s('izmir')); + $this->assertSame([2], $s('strasse')); + $this->assertSame([], $s('ğüneş'), 'only ASCII letters are folded'); + $this->assertSame([7], $s('Ğüneş')); + $this->assertSame([0, 1], $s('pHp')); + } finally { + $this->setMbstringHook(null); + } + } + + /** @test like()'s end anchor means "ends with": a trailing newline is not the end */ + public function likeEndAnchorMeansEndsWith(): void + { + $db = $this->db(); + $db->insert('nl', [['n' => 'Jackson'], ['n' => "Jackson\n"], ['n' => 'abc'], ['n' => "abc\n"]]); + $this->assertSame([0], self::keysOf($db->query('nl')->like('n', 'son$')->get()), 'QUERY.md: names ending with "son"'); + $this->assertSame([0], self::keysOf($db->like('nl', 'n', 'son$'))); + $this->assertSame([2], self::keysOf($db->query('nl')->where(['n' => ['$like' => '^abc$']])->get())); + $this->assertSame([1, 3], self::keysOf($db->query('nl')->notLike('n', 'son$')->notLike('n', '^abc$')->get())); + } + + /** @test noneDB::like(): booleans never match; a null or array pattern returns false; numbers are matched as text */ + public function noneDbLikeArguments(): void + { + $db = $this->db(); + $db->insert('la', [['f' => true], ['f' => 1], ['f' => false], ['f' => ''], ['f' => '1'], ['f' => [1]], ['f' => null], ['f' => 2.5], ['g' => 1]]); + $this->assertSame([1, 4], self::keysOf($db->like('la', 'f', '1'))); + $this->assertSame([1, 4], self::keysOf($db->like('la', 'f', 1)), 'a number pattern is its text'); + $this->assertSame([7], self::keysOf($db->like('la', 'f', '.5'))); + $this->assertSame([1, 3, 4, 7], self::keysOf($db->like('la', 'f', ''))); + $this->assertFalse($db->like('la', 'f', null)); + $this->assertFalse($db->like('la', 'f', ['1'])); + $this->assertSame([], $db->like('la', 'nothere', '')); + $this->assertSame([], $db->like('ghost', 'f', 'x')); + $this->assertFalse($this->db(['autoCreateDB' => false])->like('ghost', 'f', 'x')); + // The builder: a non-string $like operand is rejected + try { + $db->query('la')->where(['f' => ['$like' => null]])->get(); + $this->fail('null pattern'); + } catch (\InvalidArgumentException $e) { + $this->assertSame('noneDB: like pattern must be a string', $e->getMessage()); + } + } + + // ====================================================================== + // The same answer on every PHP version (query results that don't depend on the PHP version) + // ====================================================================== + + /** @test numeric strings with trailing whitespace are numbers for comparisons, between() and sums, as they are for sort() */ + public function trailingWhitespaceNumericStringsAreNumbersOnEveryPhpVersion(): void + { + $db = $this->db(); + $db->insert('ws', [['v' => '7 '], ['v' => 7], ['v' => ' 7'], ['v' => '7'], ['v' => "7\n"], ['v' => 'abc '], ['v' => "8\t"]]); + // sort(): the number 7, then the numeric strings 7 (ties in key order) and 8, then "abc " + $this->assertSame([1, 0, 2, 3, 4, 6, 5], self::keysOf($db->query('ws')->sort('v')->get()), 'sort(): documented on every PHP version'); + $this->assertSame([0, 1, 2, 3, 4, 6], self::keysOf($db->query('ws')->where(['v' => ['$gt' => 6]])->get()), '$gt'); + $this->assertSame([0, 1, 2, 3, 4], self::keysOf($db->query('ws')->where(['v' => ['$lt' => 8]])->get()), '$lt'); + $this->assertSame([0, 1, 2, 3, 4], self::keysOf($db->query('ws')->between('v', 7, 7)->get()), 'between()'); + $this->assertSame([0, 1, 2, 3, 4], self::keysOf($db->between('ws', 'v', 7, 7)), 'noneDB::between()'); + $this->assertSame(43.0, $db->query('ws')->sum('v'), 'sum: 7 * 5 + 8'); + $this->assertNumber(43, $db->sum('ws', 'v')); + $this->assertSame(43 / 6, $db->query('ws')->avg('v')); + $this->assertSame(7, $db->query('ws')->min('v')); + $this->assertSame("8\t", $db->query('ws')->where(['v' => ['$ne' => 'abc ']])->max('v')); + } + + // ====================================================================== + // Spatial filters in the builder, explain(), unusual membership values + // ====================================================================== + + /** @test nearest(k <= 0) returns nothing; several spatial filters must all match; _distance comes from withinDistance() */ + public function spatialFiltersCombineInTheBuilder(): void + { + foreach ([false, true] as $indexed) { + $db = $this->db(); + $name = $indexed ? 'spi' : 'sp'; + $rows = []; + for ($i = 0; $i < 10; $i++) { + $rows[] = ['i' => $i, 'loc' => ['type' => 'Point', 'coordinates' => [$i * 0.01, 0]]]; + } + $db->insert($name, $rows); + if ($indexed) { + $this->assertTrue($db->createSpatialIndex($name, 'loc')['success']); + } + $this->assertSame([], $db->query($name)->nearest('loc', 0, 0, 0)->get()); + $this->assertSame([], $db->query($name)->nearest('loc', 0, 0, -3)->get()); + $this->assertSame(0, $db->query($name)->nearest('loc', 0, 0, 0)->count()); + $this->assertFalse($db->query($name)->nearest('loc', 0, 0, 0)->exists()); + // Box: longitudes 0.015..0.075 = i 2..7; within 5000 m of (0, 0): i 0..4 (0.01 degree on the equator = 1111.95 m) + $meters = function (int $i) { return 6371000.0 * deg2rad($i * 0.01); }; + foreach ([ + $db->query($name)->withinBBox('loc', 0.015, -1, 0.075, 1)->withinDistance('loc', 0, 0, 5000), + $db->query($name)->withinDistance('loc', 0, 0, 5000)->withinBBox('loc', 0.015, -1, 0.075, 1), + ] as $n => $query) { + $rowsFound = $query->get(); + $keys = self::keysOf($rowsFound); + sort($keys); + $this->assertSame([2, 3, 4], $keys, "$name order $n"); + foreach ($rowsFound as $row) { + $this->assertEqualsWithDelta($meters($row['i']), $row['_distance'], 0.011, "$name order $n: meters to withinDistance()'s center"); + } + $plan = $query->explain(); + $this->assertSame('spatial', $plan['access']); + $this->assertFalse($plan['earlyExit']); + } + $this->assertSame(['withinBBox', 'withinDistance'], + $db->query($name)->withinBBox('loc', 0, 0, 1, 1)->withinDistance('loc', 0, 0, 1)->explain()['spatial']); + // nearest() among other filters: its k after the where() filter, nearest first + $this->assertSame([5, 7], self::keysOf($db->query($name)->nearest('loc', 0.051, 0, 2)->where(['i' => ['$in' => [1, 5, 7, 9]]])->get())); + } + } + + /** @test explain(): "access" is index, scan, mixed (one shard's index file missing), none (nothing to read) or spatial */ + public function explainReportsHowTheQueryReads(): void + { + $db = $this->db(['shardSize' => 4]); + $rows = []; + for ($i = 0; $i < 12; $i++) { + $rows[] = ['v' => $i % 3, 'w' => $i]; + } + $db->insert('ex', $rows); + $this->assertTrue($db->createFieldIndex('ex', 'v')['success']); + $plan = $db->query('ex')->where(['v' => 1])->explain(); + $this->assertSame('index', $plan['access']); + $this->assertSame(['total' => 3, 'skipped' => 0, 'indexed' => 3, 'scanned' => 0], $plan['segments']); + $this->assertSame(4, $plan['estimatedRead'], 'keys 1, 4, 7, 10'); + $this->assertTrue($plan['sharded']); + $this->assertSame('scan', $db->query('ex')->where(['w' => 5])->explain()['access']); + $this->assertSame('none', $db->query('ex')->where(['v' => 5])->explain()['access'], 'the global index rules every shard out'); + $this->assertSame('none', $db->query('nodb')->where(['v' => 1])->explain()['access']); + $keys = $db->query('ex')->where(['key' => [1, 9]])->explain(); + $this->assertSame('index', $keys['access']); + $this->assertSame(['total' => 3, 'skipped' => 1, 'indexed' => 2, 'scanned' => 0], $keys['segments']); + $this->assertSame(['keys-in-filter-order'], $keys['filter']); + $this->assertSame('spatial', $db->query('ex')->withinBBox('w', 0, 0, 1, 1)->explain()['access']); + $this->assertDirectoryDoesNotExist($this->dbPath('nodb', 'home'), 'explain() builds and creates nothing'); + + // Shard 1 loses its field index file: that shard is scanned until the index is built again + $file = $this->dbPath('ex', 'fx', 'v', 1); + $this->assertFileExists($file); + \noneDB::clearStaticCache(); + $this->assertTrue(unlink($file)); + $db = $this->db(['shardSize' => 4]); + $plan = $db->query('ex')->where(['v' => 1])->explain(); + $this->assertSame('mixed', $plan['access']); + $this->assertSame(['total' => 3, 'skipped' => 0, 'indexed' => 2, 'scanned' => 1], $plan['segments']); + $this->assertFileDoesNotExist($file, 'explain() builds nothing'); + $analyzed = $db->query('ex')->where(['v' => 1])->explain(true); + $this->assertSame(4, $analyzed['actual']['returned']); + $this->assertSame([1, 4, 7, 10], self::keysOf($db->query('ex')->where(['v' => 1])->get()), 'the result never depends on the indexes'); + } + + /** @test long membership lists (hash set) with objects and arrays: strict, an object equals no stored value */ + public function longMembershipListsWithUnusualValues(): void + { + $db = $this->db(); + $db->insert('ob', [['v' => 1], ['v' => []], ['v' => ['a' => 1]], ['v' => '1'], ['v' => null], ['w' => 1]]); + $list = [new \stdClass(), 1, 2, 3, 4, 5, 6, 7, 8]; + $this->assertSame([0], self::keysOf($db->query('ob')->whereIn('v', $list)->get())); + $this->assertSame([1, 2, 3, 4, 5], self::keysOf($db->query('ob')->whereNotIn('v', $list)->get())); + $this->assertSame([0], self::keysOf($db->query('ob')->where(['v' => ['$in' => $list]])->get())); + $this->assertSame([1, 2, 3, 4, 5], self::keysOf($db->query('ob')->where(['v' => ['$nin' => $list]])->get())); + $arrays = [1, [], ['a' => 1], 'x', 'y', 'z', 'w', 'q', 'r']; + $this->assertSame([0, 1, 2], self::keysOf($db->query('ob')->whereIn('v', $arrays)->get()), 'arrays by value, in a long list'); + $this->assertSame([0, 1, 2], self::keysOf($db->query('ob')->whereIn('v', array_slice($arrays, 0, 3))->get()), 'and in a short one'); + $this->assertSame([], self::keysOf($db->query('ob')->whereIn('v', [['a' => '1'], [1 => 'a'], 1.0, true, 'A', 0, -1, 9, 10])->get())); + } +} diff --git a/tests/Integration/SpecSpatialTest.php b/tests/Integration/SpecSpatialTest.php new file mode 100644 index 0000000..1260602 --- /dev/null +++ b/tests/Integration/SpecSpatialTest.php @@ -0,0 +1,2051 @@ + false (a scan) and with the model, sharded and not, after + * inserts, updates, deletes, compaction and clearStaticCache(). + * + * The model's public static helpers are also used by the other spatial test files. + */ +class SpecSpatialTest extends RegressionTestCase +{ + // ========================================================================================== + // Independent model (written from docs/SPATIAL.md, not from the implementation) + // ========================================================================================== + + /** docs: "Distances in meters, using the Haversine formula (earth radius 6,371 km)" */ + public const EARTH_RADIUS = 6371000.0; + + /** Haversine distance in meters (asin form; noneDB uses the atan2 form) */ + public static function hav(float $lon1, float $lat1, float $lon2, float $lat2): float + { + $p1 = deg2rad($lat1); + $p2 = deg2rad($lat2); + $a = sin(($p2 - $p1) / 2) ** 2 + cos($p1) * cos($p2) * sin(deg2rad($lon2 - $lon1) / 2) ** 2; + return 2 * self::EARTH_RADIUS * asin(min(1.0, sqrt($a))); + } + + /** + * Distance to the nearest point of a straight lon/lat segment: golden-section search of the haversine + * distance along it (unimodal for the short segments used here), both ends included + */ + public static function segmentDistance(float $lon, float $lat, array $a, array $b): float + { + $ax = (float)$a[0]; + $ay = (float)$a[1]; + $bx = (float)$b[0]; + $by = (float)$b[1]; + $f = function (float $t) use ($lon, $lat, $ax, $ay, $bx, $by): float { + return self::hav($lon, $lat, $ax + ($bx - $ax) * $t, $ay + ($by - $ay) * $t); + }; + $g = (sqrt(5) - 1) / 2; + $lo = 0.0; + $hi = 1.0; + $x1 = $hi - $g * ($hi - $lo); + $x2 = $lo + $g * ($hi - $lo); + $f1 = $f($x1); + $f2 = $f($x2); + for ($i = 0; $i < 64; $i++) { // the bracket shrinks to 0.618^64 = 4e-14 of the segment + if ($f1 < $f2) { + $hi = $x2; + $x2 = $x1; + $f2 = $f1; + $x1 = $hi - $g * ($hi - $lo); + $f1 = $f($x1); + } else { + $lo = $x1; + $x1 = $x2; + $f1 = $f2; + $x2 = $lo + $g * ($hi - $lo); + $f2 = $f($x2); + } + } + return min($f(0.0), $f(1.0), $f1, $f2); + } + + public static function pathDistance(float $lon, float $lat, array $points): float + { + $best = INF; + $n = count($points); + for ($i = 0; $i + 1 < $n; $i++) { + $best = min($best, self::segmentDistance($lon, $lat, $points[$i], $points[$i + 1])); + } + return $best; + } + + /** Where a point is against a ring: 1 inside, 0 on its boundary, -1 outside (winding number) */ + public static function locate(float $x, float $y, array $ring): int + { + $winding = 0; + $n = count($ring) - 1; + for ($i = 0; $i < $n; $i++) { + [$ax, $ay] = [(float)$ring[$i][0], (float)$ring[$i][1]]; + [$bx, $by] = [(float)$ring[$i + 1][0], (float)$ring[$i + 1][1]]; + $o = ($bx - $ax) * ($y - $ay) - ($x - $ax) * ($by - $ay); + if ($o == 0 && min($ax, $bx) <= $x && $x <= max($ax, $bx) && min($ay, $by) <= $y && $y <= max($ay, $by)) { + return 0; + } + if ($ay <= $y) { + if ($by > $y && $o > 0) { + $winding++; + } + } elseif ($by <= $y && $o < 0) { + $winding--; + } + } + return $winding !== 0 ? 1 : -1; + } + + /** docs: "The polygon is closed": inside or on the outer ring, and not strictly inside a hole */ + public static function inClosedPolygon(float $x, float $y, array $rings): bool + { + if (self::locate($x, $y, $rings[0]) < 0) { + return false; + } + for ($h = 1; $h < count($rings); $h++) { + if (self::locate($x, $y, $rings[$h]) > 0) { + return false; + } + } + return true; + } + + /** + * docs: "Distance is measured to the nearest point of the geometry: 0 for a point inside a polygon, + * the closest vertex or edge otherwise". INF for a geometry without any position. + */ + public static function distance(float $lon, float $lat, array $g): float + { + $c = $g['coordinates'] ?? []; + switch ($g['type']) { + case 'Point': + return self::hav($lon, $lat, (float)$c[0], (float)$c[1]); + case 'MultiPoint': + $best = INF; + foreach ($c as $p) { + $best = min($best, self::hav($lon, $lat, (float)$p[0], (float)$p[1])); + } + return $best; + case 'LineString': + return self::pathDistance($lon, $lat, $c); + case 'MultiLineString': + $best = INF; + foreach ($c as $line) { + $best = min($best, self::pathDistance($lon, $lat, $line)); + } + return $best; + case 'Polygon': + if (self::inClosedPolygon($lon, $lat, $c)) { + return 0.0; + } + $best = INF; + foreach ($c as $ring) { + $best = min($best, self::pathDistance($lon, $lat, $ring)); + } + return $best; + case 'MultiPolygon': + $best = INF; + foreach ($c as $polygon) { + $best = min($best, self::distance($lon, $lat, ['type' => 'Polygon', 'coordinates' => $polygon])); + } + return $best; + case 'GeometryCollection': + $best = INF; + foreach ($g['geometries'] as $member) { + $best = min($best, self::distance($lon, $lat, $member)); + } + return $best; + } + throw new \LogicException('model: unknown type'); + } + + /** Every [lon, lat] of a geometry (an altitude is dropped) */ + public static function positions(array $g): array + { + $c = $g['coordinates'] ?? []; + switch ($g['type']) { + case 'Point': + return [[(float)$c[0], (float)$c[1]]]; + case 'MultiPoint': + case 'LineString': + return array_map(function ($p) { return [(float)$p[0], (float)$p[1]]; }, $c); + case 'MultiLineString': + case 'Polygon': + $out = []; + foreach ($c as $part) { + foreach ($part as $p) { + $out[] = [(float)$p[0], (float)$p[1]]; + } + } + return $out; + case 'MultiPolygon': + $out = []; + foreach ($c as $polygon) { + foreach ($polygon as $ring) { + foreach ($ring as $p) { + $out[] = [(float)$p[0], (float)$p[1]]; + } + } + } + return $out; + case 'GeometryCollection': + $out = []; + foreach ($g['geometries'] as $member) { + $out = array_merge($out, self::positions($member)); + } + return $out; + } + throw new \LogicException('model: unknown type'); + } + + /** Bounding box [minLon, minLat, maxLon, maxLat]; null without any position */ + public static function bbox(array $g): ?array + { + $box = null; + foreach (self::positions($g) as [$x, $y]) { + $box = $box === null ? [$x, $y, $x, $y] : [min($box[0], $x), min($box[1], $y), max($box[2], $x), max($box[3], $y)]; + } + return $box; + } + + /** Overlap or touch */ + public static function boxesTouch(array $a, array $b): bool + { + return $a[0] <= $b[2] && $b[0] <= $a[2] && $a[1] <= $b[3] && $b[1] <= $a[3]; + } + + /** docs (withinBBox): minLon > maxLon crosses the antimeridian; infinite edges are clamped to +-180/+-90 */ + public static function bboxQueryBoxes(float $minLon, float $minLat, float $maxLon, float $maxLat): array + { + $minLon = max(-180.0, min(180.0, $minLon)); + $maxLon = max(-180.0, min(180.0, $maxLon)); + $minLat = max(-90.0, min(90.0, $minLat)); + $maxLat = max(-90.0, min(90.0, $maxLat)); + return $minLon <= $maxLon ? [[$minLon, $minLat, $maxLon, $maxLat]] + : [[$minLon, $minLat, 180.0, $maxLat], [-180.0, $minLat, $maxLon, $maxLat]]; + } + + /** withinBBox() per the docs: the geometry's bounding box overlaps or touches the box */ + public static function inBBoxQuery(array $g, array $queryBoxes): bool + { + $box = self::bbox($g); + if ($box === null) { + return false; + } + foreach ($queryBoxes as $q) { + if (self::boxesTouch($box, $q)) { + return true; + } + } + return false; + } + + /** + * Components of a record for withinPolygon(): "each component of a multi-part record (each point of a + * MultiPoint, each part of a MultiPolygon or GeometryCollection)"; members without a position dropped + * @return array list of position lists + */ + public static function components(array $g): array + { + switch ($g['type']) { + case 'Point': + case 'LineString': + case 'Polygon': + return [self::positions($g)]; + case 'MultiPoint': + return array_map(function ($p) { return [[(float)$p[0], (float)$p[1]]]; }, $g['coordinates']); + case 'MultiLineString': + return array_map(function ($l) { return self::positions(['type' => 'LineString', 'coordinates' => $l]); }, $g['coordinates']); + case 'MultiPolygon': + return array_map(function ($p) { return self::positions(['type' => 'Polygon', 'coordinates' => $p]); }, $g['coordinates']); + case 'GeometryCollection': + $out = []; + foreach ($g['geometries'] as $member) { + foreach (self::components($member) as $component) { + if (!empty($component)) { + $out[] = $component; + } + } + } + return $out; + } + throw new \LogicException('model: unknown type'); + } + + /** + * withinPolygon() for a query made of convex parts without holes (exact there): a geometry with a + * position matches when every component lies in one part (all its positions inside or on the part) + * @param array $parts list of polygon coordinates (rings), each convex + */ + public static function withinConvexParts(array $g, array $parts): bool + { + $components = self::components($g); + if (empty($components)) { + return false; + } + foreach ($components as $component) { + $fits = false; + foreach ($parts as $rings) { + $all = true; + foreach ($component as [$x, $y]) { + if (!self::inClosedPolygon($x, $y, $rings)) { + $all = false; + break; + } + } + if ($all) { + $fits = true; + break; + } + } + if (!$fits) { + return false; + } + } + return true; + } + + // ========================================================================================== + // Helpers + // ========================================================================================== + + private const LOC = 'loc'; + + private static function point(float $lon, float $lat): array + { + return ['type' => 'Point', 'coordinates' => [$lon, $lat]]; + } + + private static function rect(float $x1, float $y1, float $x2, float $y2): array + { + return [[$x1, $y1], [$x2, $y1], [$x2, $y2], [$x1, $y2], [$x1, $y1]]; + } + + private static function polygon(array ...$rings): array + { + return ['type' => 'Polygon', 'coordinates' => $rings]; + } + + private static function keys(array $rows): array + { + $keys = array_column($rows, 'key'); + sort($keys); + return $keys; + } + + private function stat(string $name): int + { + $property = new \noneDB\Tests\InternalValue($name); + return $property->getValue(); + } + + private function setStatic(string $name, $value): void + { + $property = new \noneDB\Tests\InternalValue($name); + $property->setValue(null, $value); + } + + protected function tearDown(): void + { + $this->setStatic('renameFn', null); + parent::tearDown(); + } + + /** [indexed instance, scanning instance] */ + private function pair(array $config = []): array + { + return [$this->db($config), $this->db(array_merge($config, ['spatialIndexEnabled' => false]))]; + } + + /** The same query on the indexed and the scanning instance: equal results (order included), returned once */ + private function both(\noneDB $idx, \noneDB $scan, callable $query, string $label = ''): array + { + $a = $query($idx); + $b = $query($scan); + $this->assertSame($b, $a, "index == scan $label"); + return $a; + } + + /** Insert records with the field 'loc' (null: no field); returns their keys (keys are 0, 1, 2, ... in order) */ + private function insertGeometries(\noneDB $db, string $name, array $geometries): void + { + $rows = []; + foreach ($geometries as $n => $g) { + $rows[] = $g === null ? ['n' => (string)$n] : ['n' => (string)$n, self::LOC => $g]; + } + $this->assertSame(count($rows), $db->insert($name, $rows)['n']); + } + + /** @var array Model distances already computed (the same center is asked for again and again) */ + private static $distanceMemo = []; + + private static function memoDistance(float $lon, float $lat, array $g): float + { + $id = $lon . '|' . $lat . '|' . md5(serialize($g)); + if (count(self::$distanceMemo) > 200000) { + self::$distanceMemo = []; + } + return self::$distanceMemo[$id] ?? (self::$distanceMemo[$id] = self::distance($lon, $lat, $g)); + } + + /** + * Tolerance for a modelled distance: distances are exact for every geometry (SPATIAL.md: "measured to the + * nearest point of the geometry"), so only the rounding of _distance to 2 decimals is left + */ + private static function tolerance(array $g, float $d = 0.0): float + { + return 0.006; + } + + /** + * withinDistance() and nearest() against the model: distances to the model's, membership exact + * outside a band of +-tolerance around the radius, sorted by distance + * @param array $model key => geometry (valid ones only) + */ + private function assertWithinDistanceMatchesModel(array $rows, array $model, float $lon, float $lat, float $radius, string $label): void + { + $got = []; + $previous = -INF; + foreach ($rows as $row) { + $this->assertArrayHasKey('_distance', $row, $label); + $this->assertGreaterThanOrEqual($previous, $row['_distance'], "$label: sorted by distance"); + $previous = $row['_distance']; + $got[$row['key']] = $row['_distance']; + } + foreach ($model as $key => $g) { + $d = self::memoDistance($lon, $lat, $g); + $eps = self::tolerance($g, min($d, $radius)); + if ($d <= $radius - $eps) { + $this->assertArrayHasKey($key, $got, "$label: key $key at " . $d . " m is within $radius m"); + } elseif ($d > $radius + $eps) { + $this->assertArrayNotHasKey($key, $got, "$label: key $key at " . $d . " m is outside $radius m"); + } + if (isset($got[$key])) { + $this->assertEqualsWithDelta($d, $got[$key], $eps, "$label: _distance of key $key"); + } + } + $this->assertSame([], array_diff(array_keys($got), array_keys($model)), "$label: only valid geometries"); + } + + /** nearest() against the model: the k smallest distances (ties at the cut may go either way) */ + private function assertNearestMatchesModel(array $rows, array $model, float $lon, float $lat, int $k, ?float $max, string $label): void + { + $distances = []; + foreach ($model as $key => $g) { + $d = self::memoDistance($lon, $lat, $g); + if (is_finite($d)) { + $distances[$key] = $d; + } + } + asort($distances); + $sure = []; + $maybe = []; + foreach ($distances as $key => $d) { + $eps = self::tolerance($model[$key], $d); + if ($max === null || $d <= $max - $eps) { + $sure[$key] = $d; + } + if ($max === null || $d <= $max + $eps) { + $maybe[$key] = $d; + } + } + $this->assertGreaterThanOrEqual(min($k, count($sure)), count($rows), "$label: count"); + $this->assertLessThanOrEqual(min($k, count($maybe)), count($rows), "$label: count"); + if (empty($rows)) { + return; + } + $cut = array_values($maybe)[count($rows) - 1]; // the model's distance at the cut + $previous = -INF; + $got = []; + foreach ($rows as $row) { + $key = $row['key']; + $this->assertArrayHasKey($key, $maybe, "$label: key $key may be returned"); + $eps = self::tolerance($model[$key], $distances[$key]); + $this->assertEqualsWithDelta($distances[$key], $row['_distance'], $eps, "$label: _distance of key $key"); + $this->assertLessThanOrEqual($cut + 2 * $eps, $distances[$key], "$label: key $key is among the nearest"); + $this->assertGreaterThanOrEqual($previous, $row['_distance'], "$label: sorted"); + $previous = $row['_distance']; + $got[$key] = true; + } + foreach ($maybe as $key => $d) { + if ($d < $cut - 2 * self::tolerance($model[$key], $d)) { + $this->assertArrayHasKey($key, $got, "$label: key $key at $d m is nearer than the cut $cut m"); + } + } + } + + // ========================================================================================== + // Distances + // ========================================================================================== + + /** @test docs: meters, haversine with R = 6,371 km, _distance rounded to 2 decimals, altitude ignored */ + public function distancesAreHaversineMetersOnTheDocumentedEarthRadius(): void + { + $db = $this->db(); + // 1 degree of a great circle is R * pi / 180 = 111194.9266 m; antipodes are pi * R = 20015086.796 m apart + $this->assertEqualsWithDelta(self::EARTH_RADIUS * M_PI / 180, $db->haversineDistance(0, 0, 1, 0), 1e-6); + $this->assertEqualsWithDelta(self::EARTH_RADIUS * M_PI / 180, $db->haversineDistance(0, 0, 0, 1), 1e-6); + $this->assertEqualsWithDelta(self::EARTH_RADIUS * M_PI, $db->haversineDistance(0, 0, 180, 0), 1e-6); + $this->assertEqualsWithDelta(self::EARTH_RADIUS * M_PI / 2, $db->haversineDistance(0, 0, 0, 90), 1e-6); + $this->assertSame(0.0, $db->haversineDistance(28.9784, 41.0082, 28.9784, 41.0082)); + // across the antimeridian: 179.5 E and 179.5 W on the equator are 1 degree apart + $this->assertEqualsWithDelta(self::EARTH_RADIUS * M_PI / 180, $db->haversineDistance(179.5, 0, -179.5, 0), 1e-6); + mt_srand(11); + for ($i = 0; $i < 200; $i++) { + $p = [mt_rand(-1800, 1800) / 10, mt_rand(-900, 900) / 10, mt_rand(-1800, 1800) / 10, mt_rand(-900, 900) / 10]; + $this->assertEqualsWithDelta(self::hav(...$p), $db->haversineDistance(...$p), 1e-6, json_encode($p)); + } + + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'hv', [ + self::point(1, 0), // 0: 111194.93 m + self::point(0, 1), // 1: 111194.93 m + self::point(180, 0), // 2: 20015086.80 m (antipode) + ['type' => 'Point', 'coordinates' => [0, 0, 1000]], // 3: 0 m, altitude ignored + ]); + $idx->createSpatialIndex('hv', self::LOC); + foreach ([$idx, $scan] as $db) { + $get = function ($radius) use ($db) { + return array_column($db->withinDistance('hv', self::LOC, 0, 0, $radius, ['includeDistance' => true]), '_distance', 'key'); + }; + $this->assertSame([3 => 0.0], $get(111194.92), 'just below one degree'); + $within = $get(111194.93); + $this->assertSame([3, 0, 1], array_keys($within), 'one degree: both, after the record at 0 m'); + $this->assertSame([0.0, 111194.93, 111194.93], array_values($within), '2 decimals'); + $this->assertCount(3, $get(20015086.79), 'the antipode is 20015086.796 m away'); + $this->assertSame([3, 0, 1, 2], array_keys($get(20015086.80))); + $this->assertSame(20015086.8, $get(INF)[2]); + $this->assertSame([0.0, 111194.93, 111194.93, 20015086.8], array_column($db->nearest('hv', self::LOC, 0, 0, 4), '_distance')); + } + } + } + + /** @test docs: distance to the nearest point of each geometry type; 0 inside a polygon; withDistance 4 decimals */ + public function distanceIsToTheNearestPointOfEveryGeometryType(): void + { + $c = [29.5, 41.5]; + $geometries = [ + 'polygon around' => self::polygon(self::rect(29.25, 41.25, 29.75, 41.75)), + 'center in hole' => self::polygon(self::rect(29.25, 41.25, 29.75, 41.75), self::rect(29.375, 41.375, 29.625, 41.625)), + 'line east' => ['type' => 'LineString', 'coordinates' => [[29.625, 41.375], [29.625, 41.625]]], + 'line diagonal' => ['type' => 'LineString', 'coordinates' => [[29.4, 41.2], [29.8, 41.4]]], + 'line 3d' => ['type' => 'LineString', 'coordinates' => [[29.5, 41.48, 10], [29.6, 41.48, 20]]], + 'multipoint' => ['type' => 'MultiPoint', 'coordinates' => [[29.5, 41.6], [29.5, 41.45], [31, 41]]], + 'multiline' => ['type' => 'MultiLineString', 'coordinates' => [[[30, 42], [30.1, 42]], [[29.45, 41.55], [29.6, 41.55]]]], + 'multipolygon' => ['type' => 'MultiPolygon', 'coordinates' => [[self::rect(31, 41, 31.5, 41.5)], [self::rect(29.51, 41.51, 29.52, 41.52)]]], + 'collection' => ['type' => 'GeometryCollection', 'geometries' => [self::point(35, 35), ['type' => 'LineString', 'coordinates' => [[29.4, 41.53], [29.6, 41.53]]]]], + 'empty member' => ['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => []], self::point(29.5, 41.52)]], + 'point 3d' => ['type' => 'Point', 'coordinates' => [29.5, 41.49, 1234.5]], + 'empty' => ['type' => 'GeometryCollection', 'geometries' => []], + ]; + $names = array_keys($geometries); + $model = []; + foreach ($names as $key => $label) { + if ($label !== 'empty') { + $model[$key] = $geometries[$label]; + } + } + // hand checks of the model itself + $this->assertSame(0.0, self::distance($c[0], $c[1], $geometries['polygon around'])); + $this->assertEqualsWithDelta(self::hav(29.5, 41.5, 29.5, 41.45), self::distance($c[0], $c[1], $geometries['multipoint']), 1e-9); + $this->assertEqualsWithDelta(self::hav(29.5, 41.5, 29.5, 41.55), self::distance($c[0], $c[1], $geometries['multiline']), 0.01); + + foreach ([['shardingEnabled' => false], ['shardSize' => 4]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'dt', array_values($geometries)); + $this->assertSame(count($model), $idx->createSpatialIndex('dt', self::LOC)['indexed'], 'the empty collection is not indexed'); + $rows = $this->both($idx, $scan, function ($db) use ($c) { + return $db->withinDistance('dt', self::LOC, $c[0], $c[1], 1000000, ['includeDistance' => true]); + }); + $this->assertWithinDistanceMatchesModel($rows, $model, $c[0], $c[1], 1000000, 'withinDistance'); + $this->assertNotContains(array_search('empty', $names, true), array_column($rows, 'key'), 'empty collection never matches'); + $this->assertSame(0.0, array_column($rows, '_distance', 'key')[0], 'a point inside a polygon is at 0 m'); + + // each geometry is found exactly when the radius reaches it + foreach ($model as $key => $g) { + $d = self::distance($c[0], $c[1], $g); + $in = $this->both($idx, $scan, function ($db) use ($c, $d) { + return self::keys($db->withinDistance('dt', self::LOC, $c[0], $c[1], $d + 1)); + }); + $this->assertContains($key, $in, "{$names[$key]} within d + 1 m"); + if ($d > 1) { + $out = $this->both($idx, $scan, function ($db) use ($c, $d) { + return self::keys($db->withinDistance('dt', self::LOC, $c[0], $c[1], $d - 1)); + }); + $this->assertNotContains($key, $out, "{$names[$key]} not within d - 1 m"); + } + } + + $near = $this->both($idx, $scan, function ($db) use ($c) { + return $db->nearest('dt', self::LOC, $c[0], $c[1], 100); + }); + $this->assertNearestMatchesModel($near, $model, $c[0], $c[1], 100, null, 'nearest'); + + // withDistance(): measured the same way, rounded to 4 decimals; the empty collection gets none + $all = $this->both($idx, $scan, function ($db) use ($c) { + return $db->query('dt')->withDistance(self::LOC, $c[0], $c[1])->get(); + }); + foreach ($all as $row) { + if (!isset($model[$row['key']])) { + $this->assertArrayNotHasKey('_distance', $row, 'no distance without a position'); + continue; + } + $this->assertEqualsWithDelta(self::distance($c[0], $c[1], $model[$row['key']]), $row['_distance'], + 0.00006, 'withDistance ' . $names[$row['key']]); + $this->assertSame(round($row['_distance'], 4), $row['_distance'], '4 decimals'); + } + } + } + + /** + * @test BUG CANDIDATE (docs: "Distance is measured to the nearest point of the geometry ... Circles + * that cross the antimeridian are searched correctly"): an edge on the other side of the antimeridian + * is measured to one of its ends, not to its nearest point, so a line 7.2 km away is reported at + * 111.9 km and withinDistance(50 km) misses it. + */ + public function distanceToEdgesAcrossTheAntimeridian(): void + { + $geometries = [ + ['type' => 'LineString', 'coordinates' => [[179.9, -1], [179.99, 1]]], + self::polygon([[179.9, -1], [179.99, 1], [179.9, 1], [179.9, -1]]), + ['type' => 'LineString', 'coordinates' => [[-179.99, -1], [-179.9, 1]]], + ]; + [$idx, $scan] = $this->pair(); + $this->insertGeometries($idx, 'amd', $geometries); + $idx->createSpatialIndex('amd', self::LOC); + foreach ([[-179.99, 0.0], [179.99, 0.0]] as [$lon, $lat]) { + $expected = [0, 1, 2]; + foreach ($expected as $key) { + $this->assertLessThan(10000, self::distance($lon, $lat, $geometries[$key]), 'model: every geometry is within 10 km'); + } + $rows = $this->both($idx, $scan, function ($db) use ($lon, $lat) { + return $db->withinDistance('amd', self::LOC, $lon, $lat, 50000, ['includeDistance' => true]); + }); + $this->assertSame($expected, self::keys($rows), "within 50 km of ($lon, $lat)"); + $model = array_intersect_key($geometries, array_flip($expected)); + $this->assertWithinDistanceMatchesModel($rows, $model, $lon, $lat, 50000, "($lon, $lat)"); + } + } + + /** + * @test BUG CANDIDATE (docs: "Distance is measured to the nearest point of the geometry"): the point of an + * edge is chosen in a local equirectangular projection around the center, so a long edge far from the + * center is measured to a point that isn't its nearest one (72.7 m too far at 409 km here). + */ + public function distanceToLongEdgesIsToTheirNearestPoint(): void + { + $line = ['type' => 'LineString', 'coordinates' => [[0, 60], [20, 70]]]; + $polygon = self::polygon([[0, 60], [20, 70], [0, 70], [0, 60]]); + $model = self::distance(15, 62, $line); + $this->assertEqualsWithDelta($model, self::distance(15, 62, $polygon), 1e-6, 'model: the same nearest edge'); + $this->assertLessThan(self::hav(15, 62, 20, 70), $model, 'model: nearer than both ends'); + $this->assertLessThan(self::hav(15, 62, 0, 60), $model); + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $this->insertGeometries($idx, 'le', [$line, $polygon]); + $idx->createSpatialIndex('le', self::LOC); + $rows = $this->both($idx, $scan, function ($db) { return $db->nearest('le', self::LOC, 15, 62, 2); }); + foreach ($rows as $row) { + $this->assertEqualsWithDelta($model, $row['_distance'], 1.0, 'nearest: _distance of key ' . $row['key']); + } + $this->assertSame([0, 1], self::keys($this->both($idx, $scan, function ($db) use ($model) { + return $db->withinDistance('le', self::LOC, 15, 62, $model + 10); + })), 'within 10 m more than the nearest point'); + } + + /** @test docs: NaN -> [], infinite radius -> every geometry, infinite center or negative radius -> [] */ + public function withinDistanceArgumentRules(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'wa', [ + self::point(0.01, 0), self::point(-0.01, 0), self::point(0.02, 0), self::point(0, 0.03), // 0-3 + ['type' => 'LineString', 'coordinates' => [1, 2]], // 4 invalid + ['type' => 'GeometryCollection', 'geometries' => []], // 5 no position + null, // 6 no field + self::point(-179.99, -89.99), // 7 far away + ]); + $idx->createSpatialIndex('wa', self::LOC); + $q = function (float $lon, float $lat, float $r, array $o = []) use ($idx, $scan) { + return $this->both($idx, $scan, function ($db) use ($lon, $lat, $r, $o) { + return $db->withinDistance('wa', self::LOC, $lon, $lat, $r, $o); + }, json_encode([$lon, $lat, $r])); + }; + $this->assertSame([], $q(0, 0, -1)); + $this->assertSame([], $q(0, 0, NAN)); + $this->assertSame([], $q(NAN, 0, 1000)); + $this->assertSame([], $q(0, NAN, 1000)); + $this->assertSame([], $q(INF, 0, 1000)); + $this->assertSame([], $q(0, -INF, INF)); + $this->assertSame([], $q(0, 0, 0), 'no record at the center'); + $this->assertSame([0], self::keys($q(0.01, 0, 0)), 'radius 0: the record at the center'); + $this->assertSame([0, 1, 2, 3, 7], self::keys($q(0, 0, INF)), 'INF: every valid geometry with a position'); + $this->assertSame([0, 1, 2, 3, 7], self::keys($q(0, 0, 1e12)), 'beyond half the earth: everything'); + $default = $q(0, 0, 5000); + $this->assertSame([0, 1, 2, 3], self::keys($default)); + foreach ($default as $row) { + $this->assertArrayNotHasKey('_distance', $row, 'includeDistance defaults to false'); + } + $this->assertSame([], $idx->query('wa')->withinDistance(self::LOC, 0, 0, NAN)->get()); + $this->assertSame([], $idx->query('wa')->withinDistance(self::LOC, 0, 0, -5)->get()); + } + } + + /** + * @test docs/SPATIAL.md nearest(): without an index every record is measured and the k nearest are returned, + * also when there are more records than are kept while measuring (the best are kept as they come, in one + * segment or across shards). 3000 points along the equator, each 0.001 degree further than the one before: + * the nearest are the first keys, measured before the others. + */ + public function nearestWithoutAnIndexKeepsTheBestOfManyRecords(): void + { + foreach ([['shardingEnabled' => false], ['shardSize' => 1000]] as $config) { + $this->cleanTestDirectory(); + $scan = $this->db(array_merge($config, ['spatialIndexEnabled' => false])); + $this->insertGeometries($scan, 'nk', array_map(function ($i) { return self::point(0.001 * ($i + 1), 0); }, range(0, 2999))); + foreach ([1, 5, 300, 2999] as $k) { + $rows = $scan->nearest('nk', self::LOC, 0, 0, $k); + $this->assertSame(range(0, $k - 1), array_column($rows, 'key'), json_encode($config) . " k = $k"); + } + } + } + + /** @test docs: nearest() k, ordering, ties, includeDistance, maxDistance (NaN, INF, negative, 0) */ + public function nearestRules(): void + { + $one = self::EARTH_RADIUS * 0.01 * M_PI / 180; // 1111.9493 m: 0.01 degree + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'nr', [ + self::point(0.01, 0), // 0: 1111.95 m east + self::point(-0.01, 0), // 1: 1111.95 m west (tie) + self::point(0.02, 0), // 2: 2223.90 m + self::point(0, 0.03), // 3: 3335.85 m + ['type' => 'Point', 'coordinates' => [200, 0]], // 4 invalid + ['type' => 'GeometryCollection', 'geometries' => []], // 5 no position + null, // 6 no field + ]); + $idx->createSpatialIndex('nr', self::LOC); + $q = function (int $k, array $o = [], float $lon = 0.0, float $lat = 0.0) use ($idx, $scan) { + return $this->both($idx, $scan, function ($db) use ($k, $o, $lon, $lat) { + return $db->nearest('nr', self::LOC, $lon, $lat, $k, $o); + }, json_encode([$k, $o])); + }; + $expected = [round($one, 2), round($one, 2), round(2 * $one, 2), round(3 * $one, 2)]; + foreach ([3, 4, 6, 10, 1000] as $k) { // below and above the record count (the index path and the scan path) + $rows = $q($k); + $this->assertSame(array_slice($expected, 0, min($k, 4)), array_column($rows, '_distance'), "k = $k"); + $this->assertEqualsCanonicalizing([0, 1], array_slice(array_column($rows, 'key'), 0, 2), "k = $k: the tie first"); + if ($k >= 4) { + $this->assertSame([2, 3], array_slice(array_column($rows, 'key'), 2), "k = $k"); + } + } + $first = $q(1); + $this->assertCount(1, $first); + $this->assertContains($first[0]['key'], [0, 1], 'k = 1: one of the two nearest'); + $this->assertSame([], $q(0)); + $this->assertSame([], $q(-1)); + + foreach ($q(10, ['includeDistance' => false]) as $row) { + $this->assertArrayNotHasKey('_distance', $row); + } + $this->assertSame([0, 1, 2, 3], self::keys($q(10, ['includeDistance' => false]))); + + $this->assertSame([0, 1], self::keys($q(10, ['maxDistance' => 2000])), 'fewer than k within maxDistance'); + $this->assertSame([], $q(10, ['maxDistance' => 1111.9492]), 'just below 1111.94927 m'); + $this->assertSame([0, 1], self::keys($q(10, ['maxDistance' => 1111.9493]))); + $this->assertSame([0], array_column($q(10, ['maxDistance' => 1000], 0.01, 0), 'key')); + $this->assertSame([], $q(10, ['maxDistance' => NAN])); + $this->assertSame([], $q(10, ['maxDistance' => -1])); + $this->assertSame([], $q(10, ['maxDistance' => 0])); + $this->assertSame([0], array_column($q(10, ['maxDistance' => 0], 0.01, 0), 'key'), 'maxDistance 0: the record at the center'); + $this->assertSame(array_column($q(10), '_distance'), array_column($q(10, ['maxDistance' => INF]), '_distance'), 'INF: no limit'); + $this->assertSame([], $q(3, [], INF, 0)); + $this->assertSame([], $q(3, [], 0, NAN)); + } + } + + // ========================================================================================== + // withinBBox + // ========================================================================================== + + /** @test docs: bounding boxes overlap or touch; antimeridian boxes; INF clamped; NaN -> [] */ + public function withinBBoxComparesBoundingBoxes(): void + { + $geometries = [ + self::point(0, 0), // 0 + self::point(10, 10), // 1 corner + self::point(10.0000001, 5), // 2 just east of the box + ['type' => 'LineString', 'coordinates' => [[0, 0], [10, 10]]], // 3 diagonal + self::polygon(self::rect(20, 20, 30, 30), self::rect(22, 22, 28, 28)), // 4 with a hole + ['type' => 'MultiPolygon', 'coordinates' => [[self::rect(40, 40, 41, 41)], [self::rect(45, 45, 46, 46)]]], // 5 + self::point(175, 5), self::point(-175, 5), self::point(180, 5), self::point(-180, 5), // 6-9 + ['type' => 'GeometryCollection', 'geometries' => []], // 10 no position + ['type' => 'LineString', 'coordinates' => [1, 2]], // 11 invalid + ['type' => 'GeometryCollection', 'geometries' => [self::point(50, 50), ['type' => 'LineString', 'coordinates' => [[60, 60], [61, 61]]]]], // 12 + ['type' => 'Point', 'coordinates' => [0, 89.5]], // 13 + ['type' => true, 'coordinates' => [3, 3]], // 14 invalid type + ]; + $cases = [ + [[0, 0, 10, 10], [0, 1, 3]], + [[6, 0, 10, 4], [3]], // the diagonal's box overlaps; the line doesn't enter + [[24, 24, 26, 26], [4]], // inside the hole: the box still overlaps + [[42, 42, 44, 44], [5]], // between the two parts of the MultiPolygon + [[55, 55, 56, 56], [12]], // between the members of the collection + [[10, 10, 12, 12], [1, 3]], // touching corners + [[10.00000005, 0, 11, 6], [2]], + [[170, 0, -170, 10], [6, 7, 8, 9]], // crosses the antimeridian + [[180, 0, -180, 10], [8, 9]], // only the meridian itself + [[-INF, -INF, INF, INF], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13]], + [[-1, 89, 1, INF], [13]], + [[NAN, 0, 10, 10], []], + [[0, 0, 10, NAN], []], + ]; + foreach ([['shardingEnabled' => false], ['shardSize' => 4]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'bb', $geometries); + $this->assertSame(12, $idx->createSpatialIndex('bb', self::LOC)['indexed']); + foreach ($cases as [$box, $expected]) { + $got = $this->both($idx, $scan, function ($db) use ($box) { + return self::keys($db->withinBBox('bb', self::LOC, ...$box)); + }, json_encode($box)); + $this->assertSame($expected, $got, json_encode($box)); + if (array_filter($box, 'is_nan') === []) { + $model = []; + foreach ($geometries as $key => $g) { + if (!in_array($key, [11, 14], true) && self::inBBoxQuery($g, self::bboxQueryBoxes(...$box))) { + $model[] = $key; + } + } + $this->assertSame($model, $got, 'model ' . json_encode($box)); + } + } + $this->assertSame([0, 1, 3], self::keys($idx->query('bb')->withinBBox(self::LOC, 0, 0, 10, 10)->get()), 'builder'); + } + } + + /** @test docs: integer boxes in 1e-7 degree units rounded outwards: sub-grid differences still count */ + public function quantizedBoxesRoundOutwardsAndQueriesStayExact(): void + { + $db = $this->db(['shardingEnabled' => false]); + $quantize = function (array $box) use ($db) { + return (function () use ($box) { return $this->quantizeBox($box); })->call(\noneDB\Tests\Support\Testing::engine($db)); + }; + $this->assertSame([12345678, -12345679, 12345679, -12345678], $quantize([1.23456789, -1.23456789, 1.23456789, -1.23456789])); + $this->assertSame([5000000, -5000000, 5000000, -5000000], $quantize([0.5, -0.5, 0.5, -0.5]), 'grid values stay'); + // 29 + 1/1024 degrees is 290009765.625 units + $this->assertSame([290009765, 410009765, 290009766, 410009766], $quantize([29 + 1 / 1024, 41 + 1 / 1024, 29 + 1 / 1024, 41 + 1 / 1024])); + $this->assertSame([-1800000000, -900000000, 1800000000, 900000000], $quantize([-200, -100, 200, 100]), 'clamped to the globe'); + $this->assertSame([-1800000000, -900000000, 1800000000, 900000000], $quantize([-INF, -INF, INF, INF])); + + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $geometries = [ + self::point(1.23456789, -1.23456789), + self::point(29.00000004, 41.0), + self::point(29.00000001, 41.0), + self::point(-29.00000004, -41.0), + ]; + $this->insertGeometries($idx, 'qz', $geometries); + $idx->createSpatialIndex('qz', self::LOC); + [$header, $nodes] = $this->readRTreeFile($this->dbPath('qz', 'rx', 'loc')); + $this->assertSame(10000000, $header['q']); + $entries = []; + foreach ($nodes as [$level, , $items]) { + if ($level === 0) { + foreach ($items as $item) { + $entries[$item[0]] = array_slice($item, 1); + } + } + } + ksort($entries); + $this->assertSame([12345678, -12345679, 12345679, -12345678], $entries[0], 'stored rounded outwards'); + foreach ($entries as $key => $box) { + foreach ($box as $v) { + $this->assertIsInt($v); + } + [$x, $y] = $geometries[$key]['coordinates']; + $this->assertLessThanOrEqual($x * 1e7, $box[0]); + $this->assertGreaterThanOrEqual($x * 1e7, $box[2]); + $this->assertLessThanOrEqual($y * 1e7, $box[1]); + $this->assertGreaterThanOrEqual($y * 1e7, $box[3]); + } + $cases = [ + [[29.0, 40.9, 29.00000001, 41.1], [2]], // 29.00000004 shares the grid cell, but lies east + [[29.00000002, 40.9, 29.00000003, 41.1], []], // between the two points + [[29.00000004, 41.0, 29.00000004, 41.0], [1]], + [[-29.00000005, -41.0, -29.00000004, -41.0], [3]], + [[-29.00000003, -41.1, -29, -40.9], []], + [[1.23456789, -1.23456789, 1.23456789, -1.23456789], [0]], + ]; + foreach ($cases as [$box, $expected]) { + $got = $this->both($idx, $scan, function ($db) use ($box) { + return self::keys($db->withinBBox('qz', self::LOC, ...$box)); + }); + $this->assertSame($expected, $got, json_encode($box)); + } + } + + // ========================================================================================== + // withinPolygon + // ========================================================================================== + + /** Keys of the records within $query, the indexed and the scanning instance agreeing */ + private function withinKeys(\noneDB $idx, \noneDB $scan, string $name, array $query): array + { + return $this->both($idx, $scan, function ($db) use ($name, $query) { + return self::keys($db->withinPolygon($name, self::LOC, $query)); + }, json_encode($query)); + } + + /** @test docs: closed boundary on every edge, holes (touching allowed), concave polygons */ + public function withinPolygonBoundaryAndHoles(): void + { + $query = self::polygon(self::rect(0, 0, 10, 10), self::rect(4, 4, 6, 6)); + $cases = [ + // points: every edge and vertex of the outer ring is inside + [self::point(10, 5), true], [self::point(5, 10), true], [self::point(10, 10), true], [self::point(0, 5), true], + [self::point(5, 0), true], [self::point(0, 0), true], [self::point(0, 10), true], [self::point(3, 3), true], + [self::point(10.0000001, 5), false], [self::point(-0.0000001, 5), false], [self::point(5, 10.0000001), false], + // the hole: inside it is outside the polygon, its boundary belongs to the polygon + [self::point(5, 5), false], [self::point(4, 5), true], [self::point(5, 4), true], [self::point(6, 6), true], + [self::point(4.0000001, 5), false], + // lines + [['type' => 'LineString', 'coordinates' => [[1, 5], [4, 5]]], true], // touches the hole + [['type' => 'LineString', 'coordinates' => [[1, 4], [9, 4]]], true], // along the hole's edge + [['type' => 'LineString', 'coordinates' => [[0, 0], [10, 0], [10, 10]]], true], // along the outer boundary + [['type' => 'LineString', 'coordinates' => [[3, 5], [9.5, 5]]], false], // through the hole + [['type' => 'LineString', 'coordinates' => [[1, 1], [11, 1]]], false], // leaves + // polygons + [self::polygon(self::rect(1, 1, 4, 4)), true], // touches a hole corner + [self::polygon(self::rect(1, 4, 4, 6)), true], // shares the hole's west edge + [self::polygon(self::rect(0, 0, 10, 10)), false], // covers the hole + [self::polygon(self::rect(2, 2, 8, 8)), false], // a ring around the hole + [self::polygon(self::rect(2, 2, 8, 8), self::rect(4, 4, 6, 6)), true], // the same hole + [self::polygon(self::rect(5, 5, 8, 8)), false], // a vertex in the hole + [self::polygon(self::rect(7, 7, 10, 10)), true], + // multi-part records: every component inside + [['type' => 'MultiPoint', 'coordinates' => [[1, 1], [9, 9], [4, 4]]], true], + [['type' => 'MultiPoint', 'coordinates' => [[1, 1], [5, 5]]], false], + [['type' => 'MultiLineString', 'coordinates' => [[[1, 1], [2, 2]], [[7, 7], [9, 9]]]], true], + [['type' => 'MultiLineString', 'coordinates' => [[[1, 1], [2, 2]], [[3, 5], [9, 5]]]], false], + [['type' => 'MultiPolygon', 'coordinates' => [[self::rect(1, 1, 2, 2)], [self::rect(7, 7, 9, 9)]]], true], + [['type' => 'MultiPolygon', 'coordinates' => [[self::rect(1, 1, 2, 2)], [self::rect(9, 9, 11, 11)]]], false], + [['type' => 'GeometryCollection', 'geometries' => [self::point(1, 1), ['type' => 'LineString', 'coordinates' => [[2, 2], [3, 3]]], + ['type' => 'GeometryCollection', 'geometries' => []]]], true], // empty members are ignored + [['type' => 'GeometryCollection', 'geometries' => [self::point(1, 1), self::point(5, 5)]], false], + [['type' => 'GeometryCollection', 'geometries' => []], false], // never within anything + [['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => []]]], false], + [['type' => 'Point', 'coordinates' => [3, 3, 99]], true], // altitude ignored + ]; + $concave = self::polygon([[0, 0], [10, 0], [10, 3], [3, 3], [3, 7], [10, 7], [10, 10], [0, 10], [0, 0]]); // a "C" + $concaveCases = [ + [['type' => 'LineString', 'coordinates' => [[5, 3], [5, 7]]], false], // a chord across the notch + [['type' => 'LineString', 'coordinates' => [[10, 3], [10, 7]]], false], // between two boundary vertices + [['type' => 'LineString', 'coordinates' => [[5, 2], [5, 8]]], false], // leaves and re-enters + [['type' => 'LineString', 'coordinates' => [[1, 1], [1, 9]]], true], + [['type' => 'LineString', 'coordinates' => [[3, 3], [3, 7]]], true], // along the notch's edge + [self::point(10, 5), false], [self::point(3, 5), true], [self::point(6.5, 3), true], + ]; + // a zero-area hole (all its vertices on one line): nothing is excluded, its boundary belongs to the polygon + $flatHole = self::polygon(self::rect(0, 0, 10, 10), [[2, 5], [8, 5], [5, 5], [2, 5]]); + $flatCases = [[self::point(5, 5), true], [self::point(3, 5), true], [self::point(5, 6), true]]; + + foreach ([['shardingEnabled' => false], ['shardSize' => 5]] as $config) { + foreach ([[$query, $cases], [$concave, $concaveCases], [$flatHole, $flatCases]] as $n => [$q, $list]) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'wp', array_column($list, 0)); + $idx->createSpatialIndex('wp', self::LOC); + $expected = array_keys(array_filter(array_column($list, 1))); + $got = $this->withinKeys($idx, $scan, 'wp', $q); + foreach ($list as $key => [$g, $in]) { + $this->assertSame($in, in_array($key, $got, true), "query $n, record $key " . json_encode($g)); + } + $this->assertSame($expected, $got); + $this->assertSame($expected, self::keys($idx->query('wp')->withinPolygon(self::LOC, $q)->get()), 'builder'); + } + } + $this->assertTrue($this->db()->validateGeoJSON($flatHole)['valid'], 'setup: the flat hole is a valid polygon'); + } + + /** @test docs: MultiPolygon query: components may lie in different parts, one component in one part */ + public function withinMultiPolygonComponentRule(): void + { + $query = ['type' => 'MultiPolygon', 'coordinates' => [[self::rect(0, 0, 5, 5)], [self::rect(5, 0, 10, 5)], [self::rect(20, 20, 30, 30)]]]; + $cases = [ + [['type' => 'MultiPoint', 'coordinates' => [[2, 2], [7, 2], [25, 25]]], true], + [['type' => 'LineString', 'coordinates' => [[2, 2], [7, 2]]], false], // spans two adjacent parts + [['type' => 'LineString', 'coordinates' => [[5, 1], [5, 4]]], true], // on the shared edge + [['type' => 'MultiLineString', 'coordinates' => [[[1, 1], [4, 1]], [[6, 1], [9, 1]]]], true], + [['type' => 'MultiPolygon', 'coordinates' => [[self::rect(1, 1, 2, 2)], [self::rect(21, 21, 22, 22)]]], true], + [['type' => 'MultiPolygon', 'coordinates' => [[self::rect(1, 1, 2, 2)], [self::rect(12, 12, 13, 13)]]], false], + [self::polygon(self::rect(4, 1, 6, 2)), false], // spans two parts + [self::polygon(self::rect(21, 21, 29, 29)), true], + [['type' => 'GeometryCollection', 'geometries' => [self::point(1, 1), self::polygon(self::rect(6, 1, 7, 2)), + ['type' => 'LineString', 'coordinates' => [[20, 20], [30, 30]]]]], true], + [['type' => 'GeometryCollection', 'geometries' => [self::point(1, 1), self::point(15, 15)]], false], + [self::point(15, 15), false], + [self::point(30, 30), true], + ]; + $parts = array_map(function ($p) { return $p; }, $query['coordinates']); + foreach ($cases as [$g, $in]) { + $this->assertSame($in, self::withinConvexParts($g, $parts), 'model agrees with the hand answer: ' . json_encode($g)); + } + foreach ([['shardingEnabled' => false], ['shardSize' => 3]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $this->insertGeometries($idx, 'mp', array_column($cases, 0)); + $idx->createSpatialIndex('mp', self::LOC); + $this->assertSame(array_keys(array_filter(array_column($cases, 1))), $this->withinKeys($idx, $scan, 'mp', $query)); + } + + // docs: "A polygon can't cross the antimeridian ...; split it into a MultiPolygon at +-180" + $pacific = ['type' => 'MultiPolygon', 'coordinates' => [[self::rect(170, -10, 180, 10)], [self::rect(-180, -10, -170, 10)]]]; + $cases = [ + [self::point(175, 0), true], [self::point(-175, 5), true], [self::point(180, 0), true], [self::point(-180, -10), true], + [self::point(0, 0), false], [self::point(-169.9, 0), false], + [['type' => 'MultiPoint', 'coordinates' => [[175, 0], [-175, 0]]], true], + [['type' => 'LineString', 'coordinates' => [[179, 0], [-179, 0]]], false], // plain coordinates: the long way round + [['type' => 'MultiLineString', 'coordinates' => [[[171, 1], [179, 1]], [[-179, 1], [-171, 1]]]], true], + ]; + foreach ($cases as [$g, $in]) { + $this->assertSame($in, self::withinConvexParts($g, $pacific['coordinates']), 'model: ' . json_encode($g)); + } + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $this->insertGeometries($idx, 'pc', array_column($cases, 0)); + $idx->createSpatialIndex('pc', self::LOC); + $this->assertSame(array_keys(array_filter(array_column($cases, 1))), $this->withinKeys($idx, $scan, 'pc', $pacific)); + } + + /** @test docs: a query that isn't a valid Polygon or MultiPolygon returns [] */ + public function withinPolygonRejectsInvalidQueries(): void + { + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $this->insertGeometries($idx, 'iq', [self::point(1, 1), self::point(5, 5)]); + $idx->createSpatialIndex('iq', self::LOC); + $valid = self::polygon(self::rect(0, 0, 10, 10)); + $this->assertSame([0, 1], $this->withinKeys($idx, $scan, 'iq', $valid), 'setup'); + $invalid = [ + 'unclosed' => self::polygon([[0, 0], [10, 0], [10, 10], [0, 10]]), + 'bow-tie' => self::polygon([[0, 0], [10, 10], [10, 0], [0, 10], [0, 0]]), + 'hole outside' => self::polygon(self::rect(0, 0, 10, 10), self::rect(20, 20, 21, 21)), + 'NaN vertex' => self::polygon([[0, 0], [10, 0], [NAN, 10], [0, 10], [0, 0]]), + 'flat' => ['type' => 'Polygon', 'coordinates' => [0, 0, 10, 10]], + 'line' => ['type' => 'LineString', 'coordinates' => [[0, 0], [10, 10]]], + 'point' => self::point(1, 1), + 'collection' => ['type' => 'GeometryCollection', 'geometries' => [$valid]], + 'bad part' => ['type' => 'MultiPolygon', 'coordinates' => [$valid['coordinates'], [[[0, 0], [1, 0], [0, 0]]]]], + 'no type' => ['coordinates' => $valid['coordinates']], + 'empty' => [], + ]; + foreach ($invalid as $label => $q) { + $this->assertSame([], $this->withinKeys($idx, $scan, 'iq', $q), $label); + $this->assertSame([], $idx->query('iq')->withinPolygon(self::LOC, $q)->get(), "builder $label"); + } + } + + // ========================================================================================== + // Query builder rules (docs: "Rules") + // ========================================================================================== + + /** @test */ + public function builderRulesFromTheDocs(): void + { + $points = []; + mt_srand(31); + for ($i = 0; $i < 40; $i++) { + $points[] = ['t' => $i % 3, 'name' => "p$i", self::LOC => self::point(29.5 + mt_rand(-200, 200) / 2048, 41.5 + mt_rand(-200, 200) / 2048)]; + } + $points[] = ['t' => 0, 'name' => 'area', self::LOC => self::polygon(self::rect(29.45, 41.45, 29.55, 41.55))]; + $model = array_map(function ($r) { return $r[self::LOC]; }, $points); + $a = [29.5, 41.5]; + $b = [29.55, 41.47]; + foreach ([['shardingEnabled' => false], ['shardSize' => 7]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $idx->insert('br', $points); + $idx->createSpatialIndex('br', self::LOC); + foreach ([$idx, $scan] as $which => $db) { + $label = $which === 0 ? 'indexed' : 'scan'; + // the spatial filter runs first, whatever the order of the calls + $inA = array_keys(array_filter($model, function ($g) use ($a) { return self::distance($a[0], $a[1], $g) <= 5000; })); + $t1 = array_values(array_filter($inA, function ($k) use ($points) { return $points[$k]['t'] === 1; })); + $x = $db->query('br')->where(['t' => 1])->withinDistance(self::LOC, $a[0], $a[1], 5000)->get(); + $y = $db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->where(['t' => 1])->get(); + $this->assertSame($x, $y, "$label: call order"); + $this->assertSame($t1, self::keys($x), "$label: model"); + foreach ($x as $row) { + $this->assertEqualsWithDelta(self::distance($a[0], $a[1], $model[$row['key']]), $row['_distance'], 0.006, "$label: _distance of withinDistance"); + } + // limit() and sort() apply to the spatial results, wherever they were called + $t1ByDistance = []; + foreach ($t1 as $k) { + $t1ByDistance[$k] = self::distance($a[0], $a[1], $model[$k]); + } + asort($t1ByDistance); + $limited = $db->query('br')->limit(3)->sort('_distance', 'asc')->where(['t' => 1])->withinDistance(self::LOC, $a[0], $a[1], 5000)->get(); + $this->assertSame(array_slice(array_keys($t1ByDistance), 0, 3), array_column($limited, 'key'), "$label: limit after the spatial filter"); + + // nearest() takes its k after the other filters + $byDistance = []; + foreach ($model as $k => $g) { + if ($points[$k]['t'] === 2) { + $byDistance[$k] = self::distance($b[0], $b[1], $g); + } + } + asort($byDistance); + $near = $db->query('br')->nearest(self::LOC, $b[0], $b[1], 4)->where(['t' => 2])->get(); + $this->assertSame(array_slice(array_keys($byDistance), 0, 4), array_column($near, 'key'), "$label: k after where()"); + + // withinDistance(A)->nearest(B): ordered and cut by the distance to B, _distance to B + $candidates = []; + foreach ($inA as $k) { + $candidates[$k] = self::distance($b[0], $b[1], $model[$k]); + } + asort($candidates); + foreach ([ + 'A then B' => $db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->nearest(self::LOC, $b[0], $b[1], 5)->get(), + 'B then A' => $db->query('br')->nearest(self::LOC, $b[0], $b[1], 5)->withinDistance(self::LOC, $a[0], $a[1], 5000)->get(), + ] as $order => $rows) { + $this->assertSame(array_slice(array_keys($candidates), 0, 5), array_column($rows, 'key'), "$label $order"); + foreach ($rows as $row) { + $this->assertEqualsWithDelta($candidates[$row['key']], $row['_distance'], 0.006, "$label $order: distance to B"); + } + } + + // withDistance() replaces _distance (4 decimals), is computed before sort(); select() keeps it + $rows = $db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->withDistance(self::LOC, $b[0], $b[1])->sort('_distance', 'desc')->get(); + $this->assertSame($inA, self::keys($rows), "$label: withDistance doesn't filter"); + $previous = INF; + foreach ($rows as $row) { + $this->assertEqualsWithDelta(self::distance($b[0], $b[1], $model[$row['key']]), $row['_distance'], + $row['name'] === 'area' ? 0.5 : 0.00006, "$label: withDistance replaced _distance"); + $this->assertLessThanOrEqual($previous, $row['_distance'], "$label: sorted by withDistance's _distance"); + $previous = $row['_distance']; + } + $selected = $db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->select(['name'])->get(); + $this->assertNotEmpty($selected); + foreach ($selected as $row) { + $this->assertSame(['name', 'key'], array_keys($row), "$label: select() drops withinDistance's _distance"); + } + foreach ($db->query('br')->nearest(self::LOC, $a[0], $a[1], 3)->select(['name'])->get() as $row) { + $this->assertSame(['name', 'key'], array_keys($row), "$label: select() drops nearest's _distance"); + } + foreach ($db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->withDistance(self::LOC, $b[0], $b[1])->select(['name'])->get() as $row) { + $this->assertSame(['name', 'key', '_distance'], array_keys($row), "$label: select() keeps withDistance's _distance"); + } + $area = $db->query('br')->where(['name' => 'area'])->withDistance(self::LOC, $a[0], $a[1])->first(); + $this->assertSame(0.0, $area['_distance'], "$label: 0 inside a polygon"); + + // several spatial filters must all match; a later withinDistance() adds its _distance + $box = [29.45, 41.45, 29.52, 41.6]; + $inBox = array_keys(array_filter($model, function ($g) use ($box) { return self::inBBoxQuery($g, [$box]); })); + $both = $db->query('br')->withinBBox(self::LOC, ...$box)->withinDistance(self::LOC, $a[0], $a[1], 3000)->get(); + $within3 = array_keys(array_filter($model, function ($g) use ($a) { return self::distance($a[0], $a[1], $g) <= 3000; })); + $this->assertSame(array_values(array_intersect($inBox, $within3)), self::keys($both), "$label: bbox AND distance"); + foreach ($both as $row) { + $this->assertEqualsWithDelta(self::distance($a[0], $a[1], $model[$row['key']]), $row['_distance'], 0.006, "$label: _distance of the later withinDistance()"); + } + $poly = self::polygon(self::rect(29.4, 41.4, 29.5, 41.6)); + $inPoly = array_keys(array_filter($model, function ($g) use ($poly) { return self::withinConvexParts($g, [$poly['coordinates']]); })); + $this->assertSame(array_values(array_intersect($inBox, $inPoly)), + self::keys($db->query('br')->withinBBox(self::LOC, ...$box)->withinPolygon(self::LOC, $poly)->get()), "$label: bbox AND polygon"); + + // k = 0 (or negative) returns nothing; count()/exists()/first() agree with get() + $this->assertSame([], $db->query('br')->nearest(self::LOC, $a[0], $a[1], 0)->get(), "$label: k = 0"); + $this->assertSame([], $db->query('br')->nearest(self::LOC, $a[0], $a[1], -2)->get(), "$label: k < 0"); + $this->assertSame(count($y), $db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->where(['t' => 1])->count(), "$label: count()"); + $this->assertTrue($db->query('br')->withinDistance(self::LOC, $a[0], $a[1], 5000)->exists()); + $this->assertFalse($db->query('br')->withinDistance(self::LOC, 0, 0, 5000)->exists()); + $this->assertSame(array_keys($byDistance)[0], $db->query('br')->nearest(self::LOC, $b[0], $b[1], 1)->where(['t' => 2])->first()['key'], "$label: first()"); + } + } + } + + // ========================================================================================== + // Index management + // ========================================================================================== + + /** @test docs: return shapes and errors of createSpatialIndex / dropSpatialIndex / rebuildSpatialIndex */ + public function indexManagementResultsAndErrors(): void + { + $db = $this->db(['shardingEnabled' => false]); + $this->insertGeometries($db, 'mg', [ + self::point(1, 1), self::point(2, 2), + null, // missing field + 'not array', // not an array + ['x' => 1], // missing type + self::point(200, 100), // lon > 180 + ['type' => 'LineString', 'coordinates' => [1, 2]], // flat coordinates + ['type' => 'GeometryCollection', 'geometries' => []], // valid, no location: never indexed + ]); + $this->assertSame(['success' => true, 'indexed' => 2, 'error' => null], $db->createSpatialIndex('mg', self::LOC)); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => "Spatial index already exists for field 'loc'"], $db->createSpatialIndex('mg', self::LOC)); + $this->assertSame([self::LOC], $db->getSpatialIndexes('mg')); + $this->assertTrue($db->hasSpatialIndex('mg', self::LOC)); + $this->assertFalse($db->hasSpatialIndex('mg', self::LOC, 0), 'a database without shards has no shard 0'); + $this->assertFalse($db->hasSpatialIndex('mg', 'other')); + foreach (['', 'key'] as $field) { + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Invalid field name'], $db->createSpatialIndex('mg', $field)); + } + + $disabled = $this->db(['shardingEnabled' => false, 'spatialIndexEnabled' => false]); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled'], $disabled->createSpatialIndex('mg', 'other')); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Spatial indexing is disabled'], $disabled->rebuildSpatialIndex('mg', self::LOC)); + $this->assertSame([self::LOC], $disabled->getSpatialIndexes('mg'), 'existing indexes are kept'); + $this->assertSame([0, 1], self::keys($disabled->withinBBox('mg', self::LOC, 0, 0, 3, 3)), 'disabled: queries scan'); + + $this->assertSame(['success' => true, 'indexed' => 2, 'error' => null], $db->rebuildSpatialIndex('mg', self::LOC)); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->rebuildSpatialIndex('mg', 'other'), 'a field without an index gets one'); + $this->assertSame([self::LOC, 'other'], $db->getSpatialIndexes('mg')); + $this->assertSame(['success' => true, 'error' => null], $db->dropSpatialIndex('mg', 'other')); + $this->assertSame(['success' => true, 'error' => null], $db->dropSpatialIndex('mg', 'other'), 'also when there was no such index'); + $this->assertSame(['success' => true, 'error' => null], $db->dropSpatialIndex('mg', self::LOC)); + $this->assertSame([], $db->getSpatialIndexes('mg')); + $this->assertFalse($db->hasSpatialIndex('mg', self::LOC)); + $this->assertFileDoesNotExist($this->dbPath('mg', 'rx', 'loc')); + $this->assertSame([0, 1], self::keys($db->withinBBox('mg', self::LOC, 0, 0, 3, 3)), 'no index: a scan'); + + // a missing database: created with autoCreateDB, an error without + $strict = $this->db(['autoCreateDB' => false]); + $this->assertSame(['success' => false, 'indexed' => 0, 'error' => 'Database does not exist'], $strict->createSpatialIndex('nodb', self::LOC)); + $this->assertSame([], $strict->getSpatialIndexes('nodb')); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->createSpatialIndex('newdb', self::LOC)); + $this->assertSame([self::LOC], $db->getSpatialIndexes('newdb')); + + // any field name; documented file name token (STORAGE_FORMAT.md §5.1: spatial/{segment}.{token}.rx) + $this->assertTrue($db->createSpatialIndex('mg', 'home-loc')['success']); + $this->assertTrue($db->createSpatialIndex('mg', 'konum.ev')['success']); + $this->assertFileExists($this->dbPath('mg', 'gen') . 'spatial/main.home_loc-17e07f2f0d56.rx'); + + // index file names hold no database name (STORAGE_FORMAT.md §4, §5.1): a long name indexes any field + $field = str_repeat('b', 40); + $ok = str_repeat('a', 158); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->createSpatialIndex($ok, $field)); + $long = str_repeat('a', 159); + $this->assertSame(['success' => true, 'indexed' => 0, 'error' => null], $db->createSpatialIndex($long, $field)); + $this->assertSame([$field], $db->getSpatialIndexes($long)); + $this->assertTrue($db->createSpatialIndex($long, 'loc')['success'], 'a second field'); + $this->assertSame([$field, 'loc'], $db->getSpatialIndexes($long)); + } + + /** @test docs: hasSpatialIndex with a shard id: the index exists and the database has that shard */ + public function hasSpatialIndexWithShardIds(): void + { + $db = $this->db(['shardSize' => 5]); + $rows = []; + for ($i = 0; $i < 12; $i++) { + $rows[] = [self::LOC => self::point($i, $i)]; + } + $db->insert('hs', $rows); + $this->assertFalse($db->hasSpatialIndex('hs', self::LOC, 0)); + $db->createSpatialIndex('hs', self::LOC); + $this->assertTrue($db->hasSpatialIndex('hs', self::LOC)); + foreach ([0, 1, 2] as $shard) { + $this->assertTrue($db->hasSpatialIndex('hs', self::LOC, $shard), "shard $shard"); + $this->assertFileExists($this->dbPath('hs', 'rx', 'loc', $shard)); + } + $this->assertFalse($db->hasSpatialIndex('hs', self::LOC, 3)); + $this->assertFalse($db->hasSpatialIndex('hs', 'other', 0)); + } + + /** @test CHANGES 9: spatial reads of a database that doesn't exist return nothing and create nothing */ + public function spatialReadsOfAMissingDatabaseCreateNothing(): void + { + $db = $this->db(); + $before = array_values(array_diff(scandir($this->testDbDir), ['.', '..'])); + $this->assertSame([], $db->withinDistance('ghost', self::LOC, 0, 0, 1000)); + $this->assertSame([], $db->withinBBox('ghost', self::LOC, -1, -1, 1, 1)); + $this->assertSame([], $db->nearest('ghost', self::LOC, 0, 0, 3)); + $this->assertSame([], $db->withinPolygon('ghost', self::LOC, self::polygon(self::rect(0, 0, 1, 1)))); + $this->assertSame([], $db->query('ghost')->withinDistance(self::LOC, 0, 0, 1000)->withDistance(self::LOC, 0, 0)->get()); + $this->assertFalse($db->hasSpatialIndex('ghost', self::LOC)); + $this->assertSame([], $db->getSpatialIndexes('ghost')); + clearstatcache(); + $this->assertSame($before, array_values(array_diff(scandir($this->testDbDir), ['.', '..']))); + } + + /** @test docs: invalid GeoJSON is stored, never indexed or matched, and blocks nothing */ + public function invalidGeometriesAreKeptAndNeverBlockTheDatabase(): void + { + $bad = [ + ['type' => 'LineString', 'coordinates' => [1, 2]], + ['type' => 'MultiPoint', 'coordinates' => ['a' => [1, 2]]], + ['type' => true, 'coordinates' => [1, 2]], + ['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], [1, 1], [0, 0], [0, 1]]]], + ['type' => 'Point', 'coordinates' => ['1', '2']], + ['type' => 'GeometryCollection', 'geometries' => [5]], + ]; + foreach ([['shardingEnabled' => false], ['shardSize' => 2]] as $config) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair($config); + $idx->createSpatialIndex('ib', self::LOC); + $this->insertGeometries($idx, 'ib', array_merge($bad, [self::point(1, 1)])); + foreach ([$idx, $scan] as $db) { + $this->assertSame([6], self::keys($db->withinBBox('ib', self::LOC, -10, -10, 10, 10))); + $this->assertSame([6], self::keys($db->withinDistance('ib', self::LOC, 1, 1, 1e6))); + $this->assertSame([6], self::keys($db->nearest('ib', self::LOC, 1, 1, 10))); + $this->assertSame([6], self::keys($db->withinPolygon('ib', self::LOC, self::polygon(self::rect(-10, -10, 10, 10))))); + } + $this->assertCount(7, $idx->find('ib', 0), 'such records are kept'); + $this->assertSame(['success' => true, 'indexed' => 1, 'error' => null], $idx->rebuildSpatialIndex('ib', self::LOC)); + $this->assertSame(1, $idx->update('ib', [['key' => 0], ['set' => ['n' => 'x']]])['n']); + $this->assertSame(1, $idx->delete('ib', ['key' => 1])['n']); + $this->assertTrue($idx->checkIntegrity('ib')['ok']); + $this->assertTrue($idx->dropDB('ib')['success']); + } + } + + /** + * @test docs (Validation Cost): inserts and updates don't validate; geometries are validated when an + * R-tree is built or adds new data lines (the first query of a process after a write), and by queries + * that don't find them in an R-tree + */ + public function geometriesAreValidatedWhenTheTreeTakesThemNotOnWrite(): void + { + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $idx->createSpatialIndex('vc', self::LOC); + $idx->withinBBox('vc', self::LOC, 0, 0, 1, 1); // the (empty) tree is loaded + $polygons = []; + for ($i = 0; $i < 10; $i++) { + $polygons[] = self::polygon(self::rect($i, 0, $i + 0.5, 0.5)); + } + $before = $this->stat('geometryValidations'); + $this->insertGeometries($idx, 'vc', $polygons); + $this->assertSame(0, $this->stat('geometryValidations') - $before, 'insert: no validation'); + $this->assertCount(10, $idx->withinBBox('vc', self::LOC, -1, -1, 11, 1)); + $this->assertSame(10, $this->stat('geometryValidations') - $before, 'the first query validates the 10 new lines once'); + $before = $this->stat('geometryValidations'); + $this->assertCount(10, $idx->withinBBox('vc', self::LOC, -1, -1, 11, 1)); + $this->assertSame(0, $this->stat('geometryValidations') - $before, 'candidates of the tree are not validated again'); + $this->assertCount(3, $idx->withinPolygon('vc', self::LOC, self::polygon(self::rect(-1, -1, 2.5, 1)))); + $this->assertSame(1, $this->stat('geometryValidations') - $before, 'withinPolygon(): only the query polygon is validated'); + $before = $this->stat('geometryValidations'); + $this->assertSame(1, $idx->update('vc', [['key' => 3], ['set' => [self::LOC => self::polygon(self::rect(20, 0, 21, 1))]]])['n']); + $this->assertSame(0, $this->stat('geometryValidations') - $before, 'update: no validation'); + $this->assertSame([3], self::keys($idx->withinBBox('vc', self::LOC, 19, -1, 22, 2))); + $this->assertSame(1, $this->stat('geometryValidations') - $before, 'the changed line, once'); + $before = $this->stat('geometryValidations'); + $this->assertCount(10, $scan->withinBBox('vc', self::LOC, -1, -1, 22, 2)); + $this->assertSame(10, $this->stat('geometryValidations') - $before, 'a scan validates every record'); + } + + // ========================================================================================== + // R-tree format + // ========================================================================================== + + /** [header, nodes] of an R-tree file */ + private function readTree(string $path): array + { + return $this->readRTreeFile($path); + } + + /** Entries per leaf, root level and children of the root */ + private function treeShape(array $nodes, int $root): array + { + $leaves = []; + foreach ($nodes as [$level, , $items]) { + if ($level === 0) { + $leaves[] = count($items); + } + } + sort($leaves); + return ['leaves' => $leaves, 'rootLevel' => $nodes[$root][0], 'rootChildren' => count($nodes[$root][2])]; + } + + /** + * Structure of an in-memory R-tree: root a leaf or with 2+ children, every node at most $nodeSize items, + * boxes the union of their items, levels decreasing by one, every key in one leaf + */ + private function assertTreeIsSound(array $tree, int $nodeSize, array $expectedKeys, string $label): void + { + $root = $tree['nodes'][$tree['root']]; + if ($root[0] > 0) { + $this->assertGreaterThanOrEqual(2, count($root[2]), "$label: an inner root has 2+ children"); + } + $seen = []; + $visited = 0; + $stack = [$tree['root']]; + while ($stack) { + $id = array_pop($stack); + $visited++; + [$level, $box, $items] = $tree['nodes'][$id]; + $this->assertLessThanOrEqual($nodeSize, count($items), "$label: node $id holds at most nodeSize entries"); + $union = null; + foreach ($items as $item) { + if ($level === 0) { + $itemBox = array_slice($item, 1); + $this->assertArrayNotHasKey($item[0], $seen, "$label: key {$item[0]} once"); + $seen[$item[0]] = true; + } else { + $this->assertSame($level - 1, $tree['nodes'][$item][0], "$label: child level"); + $itemBox = $tree['nodes'][$item][1]; + $stack[] = $item; + } + $union = $union === null ? $itemBox : [min($union[0], $itemBox[0]), min($union[1], $itemBox[1]), max($union[2], $itemBox[2]), max($union[3], $itemBox[3])]; + } + $this->assertSame($union, $box, "$label: box of node $id"); + } + $this->assertSame(count($tree['nodes']), $visited, "$label: no unreachable node"); + $keys = array_keys($seen); + sort($keys); + $this->assertSame($expectedKeys, $keys, "$label: keys"); + $this->assertSame(count($expectedKeys), $tree['cnt'], "$label: count"); + } + + private function loadedTree(\noneDB $db, string $name): array + { + return (function () use ($name) { + return $this->withReadLock($name, function () use ($name) { + return $this->readSpatialIndex($name, 'loc'); + }); + })->call(\noneDB\Tests\Support\Testing::engine($db)); + } + + /** @test docs: STR bulk load fills nodes to 3/4; nodeSize option (>= 4) kept by rebuilds */ + public function bulkLoadFillsThreeQuartersAndNodeSizeIsKept(): void + { + $grid = function (int $n): array { + $out = []; + for ($i = 0; $i < $n; $i++) { + $out[] = self::point(29 + ($i % 12) / 64, 41 + intdiv($i, 12) / 64); + } + return $out; + }; + $db = $this->db(['shardingEnabled' => false]); + // default nodeSize 32: 96 entries in 4 leaves of 24, one root above them + $this->insertGeometries($db, 'st', $grid(96)); + $this->assertSame(['success' => true, 'indexed' => 96, 'error' => null], $db->createSpatialIndex('st', self::LOC)); + [$header, $nodes] = $this->readTree($this->dbPath('st', 'rx', 'loc')); + $this->assertSame(32, $header['ns']); + $this->assertSame(['leaves' => [24, 24, 24, 24], 'rootLevel' => 1, 'rootChildren' => 4], $this->treeShape($nodes, $header['root'])); + + // nodeSize 16: leaves of 12 + $this->insertGeometries($db, 'st16', $grid(48)); + $db->createSpatialIndex('st16', self::LOC, ['nodeSize' => 16]); + $path16 = $this->dbPath('st16', 'rx', 'loc'); + [$header, $nodes] = $this->readTree($path16); + $this->assertSame(16, $header['ns']); + $this->assertSame([12, 12, 12, 12], $this->treeShape($nodes, $header['root'])['leaves']); + $this->assertSame(['success' => true, 'indexed' => 48, 'error' => null], $db->rebuildSpatialIndex('st16', self::LOC)); + $this->assertSame(16, $this->readTree($path16)[0]['ns'], 'rebuildSpatialIndex keeps nodeSize'); + \noneDB::clearStaticCache(); + unlink($path16); + $db = $this->db(['shardingEnabled' => false]); + $this->assertCount(48, $db->withinBBox('st16', self::LOC, 28, 40, 30, 42)); + $this->assertSame(16, $this->readTree($path16)[0]['ns'], 'an index built again after a loss keeps nodeSize'); + + // below 4: refused, nothing made (API.md "Options are the documented ones"); without the option: 32; 4 is + // accepted: leaves of 3 + $this->insertGeometries($db, 'st3', $grid(12)); + try { + $db->createSpatialIndex('st3', self::LOC, ['nodeSize' => 3]); + $this->fail('nodeSize 3 accepted'); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString("'nodeSize' must be an integer of at least 4", $e->getMessage()); + } + $this->assertSame([], $db->getSpatialIndexes('st3'), 'nothing made'); + $db->createSpatialIndex('st3', self::LOC); + $this->assertSame(32, $this->readTree($this->dbPath('st3', 'rx', 'loc'))[0]['ns']); + $db->rebuildSpatialIndex('st3', self::LOC); + $this->assertSame(32, $this->readTree($this->dbPath('st3', 'rx', 'loc'))[0]['ns']); + $this->insertGeometries($db, 'st4', $grid(12)); + $db->createSpatialIndex('st4', self::LOC, ['nodeSize' => 4]); + [$header, $nodes] = $this->readTree($this->dbPath('st4', 'rx', 'loc')); + $this->assertSame(4, $header['ns']); + $this->assertSame([3, 3, 3, 3], $this->treeShape($nodes, $header['root'])['leaves']); + + // every tree answers like the model + foreach (['st' => 96, 'st16' => 48, 'st3' => 12, 'st4' => 12] as $name => $n) { + $model = $grid($n); + foreach ([[29, 41, 29.05, 41.05], [29.1, 41.02, 29.2, 41.2], [28, 40, 30, 42]] as $box) { + $expected = array_keys(array_filter($model, function ($g) use ($box) { return self::inBBoxQuery($g, [$box]); })); + $this->assertSame($expected, self::keys($db->withinBBox($name, self::LOC, ...$box)), "$name " . json_encode($box)); + } + } + } + + /** @test the R-tree follows inserts (splits, identical boxes), deletes (root collapse) and inserts again */ + public function rtreeStaysSoundThroughSplitsAndRootCollapse(): void + { + // no compaction: the tombstones are replayed into the loaded tree instead of a rebuild + $config = ['shardingEnabled' => false, 'autoCompactThreshold' => 1000]; + $db = $this->db($config); + $scan = $this->db($config + ['spatialIndexEnabled' => false]); + $db->createSpatialIndex('rc', self::LOC, ['nodeSize' => 4]); + $model = []; + $check = function (string $label) use (&$db, $scan, &$model) { + foreach ([[0, 0, 100, 100], [10, 10, 10, 10], [0, 0, 5, 5], [40, 40, 60, 60]] as $box) { + $expected = array_keys(array_filter($model, function ($g) use ($box) { return self::inBBoxQuery($g, [$box]); })); + $this->assertSame($expected, self::keys($this->both($db, $scan, function ($d) use ($box) { return $d->withinBBox('rc', self::LOC, ...$box); })), "$label " . json_encode($box)); + } + $this->assertTreeIsSound($this->loadedTree($db, 'rc'), 4, array_keys($model), $label); + }; + // identical boxes overflow a node: the split must still divide them + for ($i = 0; $i < 9; $i++) { + $db->insert('rc', [self::LOC => self::point(10, 10)]); + $model[$i] = self::point(10, 10); + } + $check('9 identical points'); + mt_srand(3); + for ($i = 9; $i < 120; $i++) { + $g = self::point(mt_rand(0, 100), mt_rand(0, 80)); + $db->insert('rc', [self::LOC => $g]); + $model[$i] = $g; + if ($i % 37 === 0) { + $check("insert $i"); + } + } + $check('120 inserted'); + // rebuilt by bulk loading, then emptied through the replayed tombstones: the root collapses + $db->rebuildSpatialIndex('rc', self::LOC); + $check('bulk loaded'); + $keys = array_keys($model); + foreach (array_slice($keys, 0, 119) as $n => $key) { + $db->delete('rc', ['key' => $key]); + unset($model[$key]); + if ($n % 40 === 0 || $n > 112) { + $check("deleted $n"); + } + } + $tree = $this->loadedTree($db, 'rc'); + $this->assertSame(0, $tree['nodes'][$tree['root']][0], 'one record left: the root is a leaf'); + $db->delete('rc', ['key' => 119]); + unset($model[119]); + $check('empty'); + for ($i = 0; $i < 30; $i++) { + $g = self::point(50 + $i / 10, 50); + $db->insert('rc', [self::LOC => $g]); + $model[120 + $i] = $g; + } + $check('grown again'); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $check('after clearStaticCache'); + } + + // ========================================================================================== + // Index, scan and model through the whole life of a database + // ========================================================================================== + + private const G = 1 / 256; // grid of the generated coordinates (exact binary fractions) + + private function gx(): float + { + return 29 + mt_rand(0, 256) * self::G; + } + + private function gy(): float + { + return 41 + mt_rand(0, 256) * self::G; + } + + /** A random record value: [present, value, valid] */ + private function randomValue(): array + { + $r = mt_rand(0, 99); + $x = $this->gx(); + $y = $this->gy(); + $d = function () { return mt_rand(1, 6) * self::G; }; + if ($r < 25) { + return [true, self::point($x, $y), true]; + } + if ($r < 30) { + return [true, ['type' => 'Point', 'coordinates' => [$x, $y, 12.5]], true]; + } + if ($r < 34) { + // far away: near the antimeridian or a pole + $edge = [[180, $y], [-180, $y], [179.75, $y], [-179.75, $y], [$x, 89.75], [$x - 180, 89.875]][mt_rand(0, 5)]; + return [true, self::point($edge[0], $edge[1]), true]; + } + if ($r < 46) { + $coords = [[$x, $y]]; + for ($i = mt_rand(1, 3); $i > 0; $i--) { + $coords[] = [$x + mt_rand(-4, 4) * self::G, $y + mt_rand(-4, 4) * self::G]; + } + return [true, ['type' => 'LineString', 'coordinates' => $coords], true]; + } + if ($r < 58) { + $w = $d() + 2 * self::G; + $h = $d() + 2 * self::G; + $rings = [self::rect($x, $y, $x + $w, $y + $h)]; + if (mt_rand(0, 2) === 0) { + $rings[] = self::rect($x + self::G, $y + self::G, $x + $w - self::G, $y + $h - self::G); + } + return [true, ['type' => 'Polygon', 'coordinates' => $rings], true]; + } + if ($r < 64) { + return [true, ['type' => 'MultiPoint', 'coordinates' => [[$x, $y], [$this->gx(), $this->gy()]]], true]; + } + if ($r < 70) { + return [true, ['type' => 'MultiLineString', 'coordinates' => [[[$x, $y], [$x + $d(), $y]], [[$x, $y + $d()], [$x + $d(), $y + 2 * $d()]]]], true]; + } + if ($r < 76) { + $x2 = $this->gx(); + $y2 = $this->gy(); + return [true, ['type' => 'MultiPolygon', 'coordinates' => [[self::rect($x, $y, $x + $d(), $y + $d())], [self::rect($x2, $y2, $x2 + $d(), $y2 + $d())]]], true]; + } + if ($r < 84) { + $members = [[], [self::point($x, $y), ['type' => 'LineString', 'coordinates' => [[$x, $y], [$x + $d(), $y + $d()]]]], + [['type' => 'GeometryCollection', 'geometries' => []], self::polygon(self::rect($x, $y, $x + $d(), $y + $d()))], + [self::point($x, $y), self::point($this->gx(), $this->gy())]][mt_rand(0, 3)]; + return [true, ['type' => 'GeometryCollection', 'geometries' => $members], true]; + } + switch (mt_rand(0, 6)) { + case 0: + return [false, null, false]; // no field + case 1: + return [true, ['type' => 'LineString', 'coordinates' => [$x, $y]], false]; // flat + case 2: + return [true, self::point($x + 200, $y), false]; // lon > 180 + case 3: + return [true, ['type' => true, 'coordinates' => [$x, $y]], false]; + case 4: + return [true, ['type' => 'Polygon', 'coordinates' => [[[$x, $y], [$x + 1, $y], [$x + 1, $y + 1], [$x, $y + 1]]]], false]; // unclosed + case 5: + return [true, ['type' => 'MultiPoint', 'coordinates' => ['a' => [$x, $y]]], false]; + default: + return [true, null, false]; // null + } + } + + /** Every kind of query on both instances, compared with each other and with the model */ + private function checkLife(\noneDB $idx, \noneDB $scan, string $name, array $model, string $label): void + { + $valid = []; + foreach ($model as $key => [, $value, $isValid]) { + if ($isValid) { + $valid[$key] = $value; + } + } + // the model's bookkeeping matches what is stored + $stored = array_column($idx->find($name, 0), null, 'key'); + ksort($stored); + $this->assertSame(array_keys($model), array_keys($stored), "$label: keys"); + foreach ($model as $key => [$present, $value]) { + $this->assertSame($present ? $value : null, $stored[$key][self::LOC] ?? null, "$label: stored value $key"); + } + + $boxes = [[170, 41, -170, 42], [-INF, -INF, INF, INF], [29.25, 41.25, 29.25, 41.75]]; + for ($i = 0; $i < 6; $i++) { + $x = $this->gx() - 0.1; + $y = $this->gy() - 0.1; + $boxes[] = [$x, $y, $x + mt_rand(0, 80) * self::G, $y + mt_rand(0, 80) * self::G]; + } + foreach ($boxes as $box) { + $got = $this->both($idx, $scan, function ($db) use ($name, $box) { return self::keys($db->withinBBox($name, self::LOC, ...$box)); }, "$label bbox"); + $expected = array_keys(array_filter($valid, function ($g) use ($box) { return self::inBBoxQuery($g, self::bboxQueryBoxes(...$box)); })); + $this->assertSame($expected, $got, "$label: withinBBox " . json_encode($box)); + } + + $centers = [[$this->gx(), $this->gy()], [$this->gx(), $this->gy()], [$this->gx(), $this->gy()], [-179.875, 41.5], [179.875, 41.25], [12.5, 89.9]]; + foreach ($centers as $n => [$lon, $lat]) { + $radius = (float)mt_rand(0, 40000); + $rows = $this->both($idx, $scan, function ($db) use ($name, $lon, $lat, $radius) { + return $db->withinDistance($name, self::LOC, $lon, $lat, $radius, ['includeDistance' => true]); + }, "$label distance"); + $this->assertWithinDistanceMatchesModel($rows, $valid, $lon, $lat, $radius, "$label: withinDistance($lon, $lat, $radius)"); + if ($n < 3) { + foreach ([[1, null], [7, null], [25, 4000.0], [1000, null]] as [$k, $max]) { + $options = $max === null ? [] : ['maxDistance' => $max]; + $near = $this->both($idx, $scan, function ($db) use ($name, $lon, $lat, $k, $options) { + return $db->nearest($name, self::LOC, $lon, $lat, $k, $options); + }, "$label nearest"); + $this->assertNearestMatchesModel($near, $valid, $lon, $lat, $k, $max, "$label: nearest($lon, $lat, $k, $max)"); + } + } + } + + $polygons = []; + for ($i = 0; $i < 3; $i++) { + $x = $this->gx() - 0.2; + $y = $this->gy() - 0.2; + $polygons[] = [self::rect($x, $y, $x + mt_rand(1, 100) * self::G, $y + mt_rand(1, 100) * self::G)]; + } + $polygons[] = [[[29, 41], [30, 41], [29.5, 42], [29, 41]]]; // a triangle + $queries = []; + foreach ($polygons as $rings) { + $queries[] = [self::polygon(...$rings), [$rings]]; + } + $queries[] = [['type' => 'MultiPolygon', 'coordinates' => [$polygons[0], $polygons[1]]], [$polygons[0], $polygons[1]]]; + foreach ($queries as [$query, $parts]) { + $got = $this->withinKeys($idx, $scan, $name, $query); + $expected = array_keys(array_filter($valid, function ($g) use ($parts) { return self::withinConvexParts($g, $parts); })); + $this->assertSame($expected, $got, "$label: withinPolygon " . json_encode($query)); + } + } + + /** @test */ + public function indexScanAndModelAgreeThroughWritesCompactionAndRestarts(): void + { + foreach (['plain' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 30]] as $label => $config) { + $this->cleanTestDirectory(); + mt_srand(20261027); + $name = "life $label"; + [$idx, $scan] = $this->pair($config); + $model = []; + $next = 0; + $insert = function (int $count) use (&$model, &$next, &$idx, $name) { + $rows = []; + foreach (range(1, $count) as $unused) { + $value = $this->randomValue(); + $rows[] = $value[0] ? ['i' => $next, self::LOC => $value[1]] : ['i' => $next]; + $model[$next++] = $value; + } + $this->assertSame($count, $idx->insert($name, count($rows) === 1 ? $rows[0] : $rows)['n']); + }; + $indexedCount = function () use (&$model) { + return count(array_filter($model, function ($m) { return $m[2] && self::bbox($m[1]) !== null; })); + }; + + $insert(120); + $this->assertSame(['success' => true, 'indexed' => $indexedCount(), 'error' => null], $idx->createSpatialIndex($name, self::LOC)); + $this->checkLife($idx, $scan, $name, $model, "$label created"); + + for ($i = 0; $i < 25; $i++) { + $insert(1); + } + $insert(20); + $this->checkLife($idx, $scan, $name, $model, "$label inserted"); + + $keys = array_keys($model); + for ($i = 0; $i < 30; $i++) { + $key = $keys[mt_rand(0, count($keys) - 1)]; + $value = $this->randomValue(); + $this->assertSame(1, $idx->update($name, [['key' => $key], ['set' => [self::LOC => $value[1]]]])['n']); + $model[$key] = [true, $value[1], $value[2]]; + } + $this->checkLife($idx, $scan, $name, $model, "$label updated"); + + for ($i = 0; $i < 25; $i++) { + $keys = array_keys($model); + $key = $keys[mt_rand(0, count($keys) - 1)]; + $this->assertSame(1, $idx->delete($name, ['key' => $key])['n']); + unset($model[$key]); + } + $this->checkLife($idx, $scan, $name, $model, "$label deleted"); + + $this->assertTrue($idx->compact($name)['success']); + $this->checkLife($idx, $scan, $name, $model, "$label compacted"); + + \noneDB::clearStaticCache(); + [$idx, $scan] = $this->pair($config); + $this->checkLife($idx, $scan, $name, $model, "$label restarted"); + + $insert(25); + $this->checkLife($idx, $scan, $name, $model, "$label inserted after restart"); + + $this->assertSame(['success' => true, 'indexed' => $indexedCount(), 'error' => null], $idx->rebuildSpatialIndex($name, self::LOC)); + $this->checkLife($idx, $scan, $name, $model, "$label rebuilt"); + $this->assertTrue($idx->checkIntegrity($name)['ok'], "$label integrity"); + } + } + + // ========================================================================================== + // Sharded databases: the global .grx file per field + // ========================================================================================== + + /** Integer box of a dyadic coordinate list: floor/ceil of degrees * 1e7, in integer arithmetic */ + private static function unitsFloor(int $whole, int $k1024): int + { + return $whole * 10000000 + intdiv($k1024 * 78125, 8); // k / 1024 degrees = k * 9765.625 units + } + + private static function unitsCeil(int $whole, int $k1024): int + { + return $whole * 10000000 + intdiv($k1024 * 78125 + 7, 8); + } + + /** @test docs: one global file per field with each shard's bounding box (integers, rounded outwards) */ + public function globalFilePerFieldHoldsEachShardsBox(): void + { + $db = $this->db(['shardSize' => 20]); + $rows = []; + $expected = ['loc' => [], 'home-loc' => []]; + mt_srand(17); + for ($i = 0; $i < 60; $i++) { + $shard = intdiv($i, 20); + $kx = mt_rand(1, 1023); + $ky = mt_rand(1, 1023); + $row = [self::LOC => self::point(29 + $kx / 1024, 41 + $ky / 1024)]; + $box = [self::unitsFloor(29, $kx), self::unitsFloor(41, $ky), self::unitsCeil(29, $kx), self::unitsCeil(41, $ky)]; + $prev = $expected['loc'][$shard] ?? null; + $expected['loc'][$shard] = $prev === null ? $box : [min($prev[0], $box[0]), min($prev[1], $box[1]), max($prev[2], $box[2]), max($prev[3], $box[3])]; + if ($shard === 1) { + $row['home-loc'] = self::point(-(10 + $kx / 1024), 5); // only shard 1 has this field + $hb = [-self::unitsCeil(10, $kx), 50000000, -self::unitsFloor(10, $kx), 50000000]; + $prev = $expected['home-loc'][1] ?? null; + $expected['home-loc'][1] = $prev === null ? $hb : [min($prev[0], $hb[0]), 50000000, max($prev[2], $hb[2]), 50000000]; + } + $rows[] = $row; + } + $expected['home-loc'][0] = null; // "null = empty" + $expected['home-loc'][2] = null; + ksort($expected['home-loc']); + $db->insert('gl', $rows); + $db->createSpatialIndex('gl', 'loc'); + $db->createSpatialIndex('gl', 'home-loc'); + $base = $this->dbPath('gl', 'gen') . 'spatial/'; + foreach (['loc' => 'loc', 'home-loc' => 'home_loc-17e07f2f0d56'] as $field => $token) { + $file = "$base$token.grx"; + $this->assertFileExists($file, "global file of $field"); + $header = json_decode(strtok(file_get_contents($file), "\n"), true); + $this->assertSame(['grx', $field], [$header['kind'], $header['field']]); + $boxes = []; + foreach ($header['sh'] as $shard => $entry) { + $boxes[(int)$shard] = $entry['b']; + } + ksort($boxes); + $this->assertSame($expected[$field], $boxes, "shard boxes of $field"); + foreach ([0, 1, 2] as $shard) { + $this->assertFileExists("{$base}s$shard.$token.rx"); + } + } + $this->assertCount(20, $db->withinBBox('gl', 'home-loc', -11, 4, -10, 6)); + $this->assertSame([], $db->withinBBox('gl', 'home-loc', 28, 40, 31, 43)); + } + + /** + * @test docs: "A shard is skipped only when its box and its newer lines can't match; a shard without an + * entry is always searched." Writes don't rewrite the global file, so its boxes are older than the data. + */ + public function staleOrMissingGlobalEntriesNeverSkipAShard(): void + { + $config = ['shardSize' => 30]; + [$idx, $scan] = $this->pair($config); + $model = []; + $rows = []; + for ($i = 0; $i < 80; $i++) { // shards 0, 1 and 2 (2 not full) + $model[$i] = self::point(29 + ($i % 10) / 64, 41 + intdiv($i, 10) / 64); + $rows[] = [self::LOC => $model[$i]]; + } + $idx->insert('sk', $rows); + $idx->createSpatialIndex('sk', self::LOC); + $global = $this->dbPath('sk', 'grx', 'loc'); + $before = file_get_contents($global); + $this->assertSame([0, 1, 2, 3, 4], self::keys($idx->withinBBox('sk', self::LOC, 28.9, 40.9, 29.07, 41.01)), 'setup, loads the trees'); + + $idx->insert('sk', [self::LOC => self::point(-70, -30)]); // key 80, shard 2 + $model[80] = self::point(-70, -30); + $idx->update('sk', [['key' => 5], ['set' => [self::LOC => self::point(-70.01, -30)]]]); // shard 0 + $model[5] = self::point(-70.01, -30); + $idx->update('sk', [['key' => 40], ['set' => [self::LOC => ['type' => 'LineString', 'coordinates' => [[100, 10], [101, 11]]]]]]); // shard 1 + $model[40] = ['type' => 'LineString', 'coordinates' => [[100, 10], [101, 11]]]; + clearstatcache(); + $this->assertSame($before, file_get_contents($global), 'setup: writes leave the global boxes as they were'); + + $check = function (string $label) use (&$idx, &$scan, &$model) { + foreach ([[-71, -31, -69, -29], [99, 9, 102, 12], [28.9, 40.9, 29.2, 41.2], [-INF, -INF, INF, INF]] as $box) { + $expected = array_keys(array_filter($model, function ($g) use ($box) { return self::inBBoxQuery($g, self::bboxQueryBoxes(...$box)); })); + $this->assertSame($expected, self::keys($this->both($idx, $scan, function ($db) use ($box) { return $db->withinBBox('sk', self::LOC, ...$box); })), "$label " . json_encode($box)); + } + $rows = $this->both($idx, $scan, function ($db) { return $db->withinDistance('sk', self::LOC, -70, -30, 5000, ['includeDistance' => true]); }); + $this->assertWithinDistanceMatchesModel($rows, $model, -70, -30, 5000, "$label distance"); + $near = $this->both($idx, $scan, function ($db) { return $db->nearest('sk', self::LOC, -70, -30, 3); }); + $this->assertNearestMatchesModel($near, $model, -70, -30, 3, null, "$label nearest"); + $this->assertSame(80, $near[0]['key'], "$label nearest: the record at the center"); + $this->assertSame([40], self::keys($this->both($idx, $scan, function ($db) { + return $db->withinPolygon('sk', self::LOC, self::polygon(self::rect(99, 9, 102, 12))); + })), "$label polygon"); + }; + $check('stale entries'); + \noneDB::clearStaticCache(); + [$idx, $scan] = $this->pair($config); + $check('stale entries, fresh process'); + + // a new shard has no entry: always searched + $more = []; + for ($i = 81; $i < 95; $i++) { + $model[$i] = self::point(-70 + ($i - 80) / 1000, -30.001); + $more[] = [self::LOC => $model[$i]]; + } + $idx->insert('sk', $more); + $this->assertTrue($idx->hasSpatialIndex('sk', self::LOC, 3), 'setup: shard 3 exists'); + $check('new shard'); + // no global file at all: every shard is searched + \noneDB::clearStaticCache(); + unlink($global); + [$idx, $scan] = $this->pair($config); + $check('no global file'); + } + + /** @test compaction that removes emptied shards removes their spatial files; queries stay right */ + public function compactionRemovingShardsRemovesTheirSpatialFiles(): void + { + $config = ['shardSize' => 20]; + [$idx, $scan] = $this->pair($config); + $model = []; + $rows = []; + for ($i = 0; $i < 60; $i++) { + $model[$i] = self::point($i / 10, $i / 20); + $rows[] = [self::LOC => $model[$i]]; + } + $idx->insert('cp', $rows); + $idx->createSpatialIndex('cp', self::LOC); + $idx->createSpatialIndex('cp', 'other'); + for ($i = 0; $i < 40; $i++) { + $idx->delete('cp', ['key' => $i]); + unset($model[$i]); + } + $this->assertTrue($idx->compact('cp')['success']); + foreach ([0, 1] as $shard) { + $this->assertFileDoesNotExist($this->dbPath('cp', 'rx', 'loc', $shard)); + $this->assertFileDoesNotExist($this->dbPath('cp', 'rx', 'other', $shard)); + $this->assertFalse($idx->hasSpatialIndex('cp', self::LOC, $shard)); + } + $this->assertFileExists($this->dbPath('cp', 'rx', 'loc', 2)); + $this->assertTrue($idx->hasSpatialIndex('cp', self::LOC, 2)); + foreach (['loc', 'other'] as $token) { + $header = json_decode(strtok(file_get_contents($this->dbPath('cp', 'grx', $token)), "\n"), true); + $this->assertSame(['2'], array_map('strval', array_keys($header['sh'])), "only the remaining shard in the global file of $token"); + } + foreach ([false, true] as $restart) { + if ($restart) { + \noneDB::clearStaticCache(); + [$idx, $scan] = $this->pair($config); + } + $this->assertSame(array_keys($model), self::keys($this->both($idx, $scan, function ($db) { return $db->withinBBox('cp', self::LOC, -1, -1, 10, 10); }))); + $rows = $this->both($idx, $scan, function ($db) { return $db->nearest('cp', self::LOC, 0, 0, 3); }); + $this->assertNearestMatchesModel($rows, $model, 0, 0, 3, null, 'nearest after compaction'); + $this->assertSame([40, 41, 42], array_column($rows, 'key')); + } + } + + // ========================================================================================== + // Damaged and missing index files + // ========================================================================================== + + /** @test CHANGES: damaged index files are detected and rebuilt from the data */ + public function damagedRtreeIsRebuiltFromTheData(): void + { + foreach (['body' => function ($content) { return strtok($content, "\n") . "\n{\"0\": broken"; }, + 'root' => function ($content) { + $header = json_decode(strtok($content, "\n"), true); + $header['root'] = 999999; + return json_encode($header) . "\n" . substr($content, strpos($content, "\n") + 1); + }, + 'header' => function ($content) { return "not json\n" . substr($content, strpos($content, "\n") + 1); }] as $label => $damage) { + $this->cleanTestDirectory(); + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $model = []; + for ($i = 0; $i < 30; $i++) { + $model[$i] = self::point($i, $i / 2); + } + $this->insertGeometries($idx, 'dm', $model); + $idx->createSpatialIndex('dm', self::LOC); + $path = $this->dbPath('dm', 'rx', 'loc'); + \noneDB::clearStaticCache(); + file_put_contents($path, $damage(file_get_contents($path))); + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $expected = array_keys(array_filter($model, function ($g) { return self::inBBoxQuery($g, [[5, 0, 12, 100]]); })); + $this->assertSame($expected, self::keys($this->both($idx, $scan, function ($db) { return $db->withinBBox('dm', self::LOC, 5, 0, 12, 100); })), $label); + [$header, $nodes] = $this->readTree($path); + $this->assertIsArray($nodes, "$label: rebuilt"); + $this->assertSame(30, $header['cnt'], "$label: every geometry indexed again"); + } + } + + /** @test a lost shard R-tree is built again from the data; the global file keeps the same shard boxes */ + public function lostShardRtreeIsRebuiltAndTheGlobalBoxesStay(): void + { + $config = ['shardSize' => 10]; + [$idx, $scan] = $this->pair($config); + $model = []; + for ($i = 0; $i < 30; $i++) { + $model[$i] = self::point($i, -$i); + } + $this->insertGeometries($idx, 'ls', $model); + $idx->createSpatialIndex('ls', self::LOC); + $global = file_get_contents($this->dbPath('ls', 'grx', 'loc')); + \noneDB::clearStaticCache(); + unlink($this->dbPath('ls', 'rx', 'loc', 1)); + [$idx, $scan] = $this->pair($config); + foreach ([[-1, -40, 40, 1], [12, -15, 14, -11], [9.5, -30, 20.5, 0]] as $box) { + $expected = array_keys(array_filter($model, function ($g) use ($box) { return self::inBBoxQuery($g, [$box]); })); + $this->assertSame($expected, self::keys($this->both($idx, $scan, function ($db) use ($box) { return $db->withinBBox('ls', self::LOC, ...$box); })), json_encode($box)); + } + clearstatcache(); + $this->assertFileExists($this->dbPath('ls', 'rx', 'loc', 1), 'built again'); + $header = json_decode(strtok(file_get_contents($this->dbPath('ls', 'rx', 'loc', 1)), "\n"), true); + $this->assertSame(10, $header['cnt']); + $this->assertSame($global, file_get_contents($this->dbPath('ls', 'grx', 'loc')), 'the same shard boxes'); + } + + /** + * @test CHANGES (Locking): "A lock that couldn't be taken throws RuntimeException with code + * noneDB::LOCK_FAILED. Fallback paths never continue without the lock": a spatial query that must build + * its index doesn't scan instead when the exclusive lock can't be had + */ + public function indexBuildWithoutTheLockThrowsLockFailed(): void + { + $db = $this->db(['shardingEnabled' => false]); + $this->insertGeometries($db, 'lk', [self::point(1, 1), self::point(2, 2)]); + $db->createSpatialIndex('lk', self::LOC); + \noneDB::clearStaticCache(); + $rx = $this->dbPath('lk', 'rx', 'loc'); + unlink($rx); + $script = $this->childScript('$fp = fopen(' . var_export($this->dbPath('lk', 'lock'), true) . ', "r");' + . ' flock($fp, LOCK_SH); echo "locked\n"; usleep(5000000);'); + $proc = proc_open([PHP_BINARY, $script], [1 => ['pipe', 'w']], $pipes); + try { + $this->assertSame("locked\n", fgets($pipes[1]), 'setup: another process holds a shared lock'); + $db = $this->db(['shardingEnabled' => false, 'lockTimeout' => 0.3]); + try { + $db->withinBBox('lk', self::LOC, 0, 0, 3, 3); + $this->fail('a query whose index needs the exclusive lock must not go on without it'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::LOCK_FAILED, $e->getCode(), $e->getMessage()); + } + $this->assertFileDoesNotExist($rx); + } finally { + proc_terminate($proc); + fclose($pipes[1]); + proc_close($proc); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([0, 1], self::keys($db->withinBBox('lk', self::LOC, 0, 0, 3, 3)), 'once the lock is free'); + $this->assertFileExists($rx); + } + + /** @test B-1: an index that can't be built (its file can't be written) leaves the query to a scan */ + public function indexThatCannotBeWrittenFallsBackToAScan(): void + { + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $model = []; + for ($i = 0; $i < 20; $i++) { + $model[$i] = self::point($i / 10, 0); + } + $this->insertGeometries($idx, 'fb', $model); + $idx->createSpatialIndex('fb', self::LOC); + $path = $this->dbPath('fb', 'rx', 'loc'); + \noneDB::clearStaticCache(); + unlink($path); + $this->setStatic('renameFn', function ($from, $to) { + return substr($to, -3) !== '.rx' && rename($from, $to); + }); + [$idx, $scan] = $this->pair(['shardingEnabled' => false]); + $this->assertSame([5, 6, 7], self::keys($idx->withinBBox('fb', self::LOC, 0.5, -1, 0.7, 1))); + $rows = $idx->withinDistance('fb', self::LOC, 0, 0, 25000, ['includeDistance' => true]); + $this->assertWithinDistanceMatchesModel($rows, $model, 0, 0, 25000, 'scan fallback'); + $this->assertSame([0, 1, 2], self::keys($rows)); + $this->assertFileDoesNotExist($path, 'setup: the index could not be written'); + $this->setStatic('renameFn', null); + $this->assertSame([5, 6, 7], self::keys($idx->withinBBox('fb', self::LOC, 0.5, -1, 0.7, 1))); + $this->assertFileExists($path, 'built once it can be written'); + } +} diff --git a/tests/Integration/SpecStorageTest.php b/tests/Integration/SpecStorageTest.php new file mode 100644 index 0000000..a221ecd --- /dev/null +++ b/tests/Integration/SpecStorageTest.php @@ -0,0 +1,1602 @@ +setStatic($hook, null); + } + if ($this->precision !== false) { + ini_set('serialize_precision', (string)$this->precision); + $this->precision = false; + } + parent::tearDown(); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + private function setStatic(string $name, $value): void + { + $property = new \noneDB\Tests\InternalValue($name); + $property->setValue(null, $value); + } + + private function dataFile(string $name, ?int $shard = null): string + { + return $this->dbPath($name, 'data', $shard); + } + + /** Quarantine files of a database (STORAGE_FORMAT.md 7.2: recovery/q.{segment}.{offset}.{length}.{sha256}.bin) */ + private function quarantineFiles(string $name): array + { + $files = array_map('basename', glob($this->dbPath($name, 'quarantine') . 'q.*') ?: []); + sort($files); + return $files; + } + + /** Name of a quarantine file in the database's recovery/ directory */ + private function quarantineName(string $name, string $segment, int $offset, string $bytes): string + { + return 'q.' . $segment . '.' . $offset . '.' . strlen($bytes) . '.' . hash('sha256', $bytes) . '.bin'; + } + + /** Removes the key-index checkpoints of every database */ + private function deleteCheckpoints(): void + { + foreach (glob(rtrim($this->testDbDir, '/') . '/*.ndb4/generations/*/segments/*.kix') as $index) { + unlink($index); + } + } + + /** noneDB's in-memory key index of a segment: its counters and p/l (offset/length per live key) */ + private function memIndex(\noneDB $db, string $name, ?int $shard = null): ?array + { + $index = (function () use ($name, $shard) { + return $this->readJsonlIndex($name, $shard); + })->call(\noneDB\Tests\Support\Testing::engine($db)); + if ($index === null) { + return null; + } + // Every live key's offset and length, ascending by key + [$offsets, $lengths] = \noneDB\Tests\Support\Testing::callStatic('KeyIndex::locate', $index, null); + ksort($offsets); + $index['p'] = $offsets; + $index['l'] = []; + foreach ($offsets as $key => $offset) { + $index['l'][$key] = $lengths[$key]; + } + return $index; + } + + /** A key-index checkpoint file (STORAGE_FORMAT.md 6.4) read by the test's own decoder: its header and p/l */ + private function diskIndex(string $path): array + { + $file = $this->readKeyIndexFile($path); + $index = $file['header'] + ['p' => [], 'l' => []]; + foreach ($file['entries'] as $key => [$offset, $length]) { + $index['p'][$key] = $offset; + $index['l'][$key] = $length; + } + return $index; + } + + /** The parts of an index the format defines (STORAGE_FORMAT.md 6.4), keys sorted */ + private static function core(?array $index, bool $withN = true): array + { + $p = array_map('intval', $index['p'] ?? []); + $l = array_map('intval', $index['l'] ?? []); + ksort($p); + ksort($l); + $core = ['p' => $p, 'l' => $l, 'd' => (int)($index['d'] ?? -1), 't' => (int)($index['t'] ?? -1), 's' => (int)($index['s'] ?? -1)]; + if ($withN) { + $core['n'] = (int)($index['n'] ?? -1); + } + return $core; + } + + /** + * Independent model of a segment index, from the documented data format only: every complete line is a + * record (a JSON object with an integer "key" of this segment) or a tombstone {"__nonedb_deleted":K}; + * the last line of a key wins, a tombstone deletes. d = record lines that are no longer current (replaced + * or deleted) + lines that are neither; t = tombstone lines; n = highest key seen + 1 (local to the + * shard); s = bytes up to the end of the last complete line. An unterminated tail is no line. + */ + private function modelIndex(string $file, int $base = 0, ?int $limit = null): array + { + clearstatcache(true, $file); + $bytes = is_file($file) ? (string)file_get_contents($file) : ''; + $p = []; + $l = []; + $d = 0; + $t = 0; + $n = 0; + $pos = 0; + while (($nl = strpos($bytes, "\n", $pos)) !== false) { + $json = substr($bytes, $pos, $nl - $pos); + $rec = json_decode($json, true); + $key = null; + $tomb = false; + if (is_array($rec) && array_keys($rec) === ['__nonedb_deleted'] && is_int($rec['__nonedb_deleted'])) { + $key = $rec['__nonedb_deleted']; + $tomb = true; + } elseif (is_array($rec) && isset($rec['key']) && is_int($rec['key'])) { + $key = $rec['key']; + } + if ($key !== null && ($key < $base || ($limit !== null && $key >= $limit))) { + $key = null; // a key of another segment is not a line of this one + } + if ($key === null) { + $d++; + } elseif ($tomb) { + if (isset($p[$key])) { + unset($p[$key], $l[$key]); + $d++; + } + $t++; + } else { + if (isset($p[$key])) { + $d++; + } + $p[$key] = $pos; + $l[$key] = strlen($json); + } + if ($key !== null) { + $n = max($n, $key - $base + 1); + } + $pos = $nl + 1; + } + ksort($p); + ksort($l); + return ['p' => $p, 'l' => $l, 'd' => $d, 't' => $t, 's' => $pos, 'n' => $n]; + } + + /** Complete lines of a file (without "\n"), and what follows the last one */ + private function linesOf(string $file): array + { + clearstatcache(true, $file); + $bytes = is_file($file) ? (string)file_get_contents($file) : ''; + $parts = explode("\n", $bytes); + $tail = array_pop($parts); + return [$parts, $tail]; + } + + private static function sorted(array $record): array + { + ksort($record); + return $record; + } + + /** Every file of a database (LOCK included): path relative to its directory => sha1 */ + private function snapshot(string $name): array + { + clearstatcache(); + $out = []; + foreach ($this->dbFiles($name) as $file) { + $out[$file] = sha1_file($this->dbPath($name, 'home') . $file); + } + ksort($out); + return $out; + } + + private function generation(string $name): int + { + return $this->lockState($name)['gen']; + } + + /** Temp files anywhere in the store (atomic writes: {target}.tmp{8 hex}; quarantine: q.tmp{8 hex}) */ + private function tempFiles(): array + { + $found = []; + $walk = function (string $dir) use (&$walk, &$found) { + foreach (scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($dir . $entry)) { + $walk($dir . $entry . '/'); + } elseif (preg_match('/\.tmp[0-9a-f]{8}$|^q\.tmp/', $entry) === 1) { + $found[] = $entry; + } + } + }; + $walk(rtrim($this->testDbDir, '/') . '/'); + return $found; + } + + // ------------------------------------------------------------------ + // Data file format: an append-only log of records and tombstones + // ------------------------------------------------------------------ + + /** @test STORAGE_FORMAT.md 6.1: updates append a line, deletes append {"__nonedb_deleted":K} */ + public function dataFileIsAnAppendOnlyLogOfRecordsAndTombstones(): void + { + // autoCompactThreshold 1: garbage can never exceed 100% of the record versions, so no compaction + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $file = $this->dataFile('log'); + + $this->assertSame(['n' => 3], $db->insert('log', [['v' => 'a'], ['v' => 'b'], ['v' => 'c']])); + [$lines, $tail] = $this->linesOf($file); + $this->assertSame('', $tail, 'every line ends with "\n"'); + $this->assertSame([['key' => 0, 'v' => 'a'], ['key' => 1, 'v' => 'b'], ['key' => 2, 'v' => 'c']], + array_map(function ($l) { return self::sorted(json_decode($l, true)); }, $lines), 'one line per record, carrying its key'); + $inserted = $lines; + + $this->assertSame(['n' => 1], $db->update('log', [['key' => 1], ['set' => ['v' => 'B']]])); + [$lines] = $this->linesOf($file); + $this->assertCount(4, $lines); + $this->assertSame($inserted, array_slice($lines, 0, 3), 'the old line stays until compaction'); + $this->assertSame(['key' => 1, 'v' => 'B'], self::sorted(json_decode($lines[3], true)), 'the new version is appended'); + + $this->assertSame(['n' => 1], $db->delete('log', ['key' => 2])); + [$lines] = $this->linesOf($file); + $this->assertCount(5, $lines); + $this->assertSame('{"__nonedb_deleted":2}', $lines[4], 'tombstone format'); + + clearstatcache(); + $bytes = file_get_contents($file); + $this->assertSame(['n' => 0], $db->delete('log', ['key' => 2]), 'already deleted'); + $this->assertSame(['n' => 0], $db->update('log', [['v' => 'nothing'], ['set' => ['v' => 'x']]])); + $this->assertSame(['n' => 0], $db->delete('log', ['v' => 'nothing'])); + clearstatcache(); + $this->assertSame($bytes, file_get_contents($file), 'writes that match nothing append nothing'); + + // In memory: 2 garbage record lines (key 1 replaced, key 2 deleted), 1 tombstone, next key 3 + $this->assertSame($this->modelIndex($file), self::core($this->memIndex($db, 'log'))); + $this->assertSame(['d' => 2, 't' => 1, 'n' => 3], array_intersect_key(self::core($this->memIndex($db, 'log')), ['d' => 0, 't' => 0, 'n' => 0])); + $this->assertSame([['key' => 0, 'v' => 'a'], ['key' => 1, 'v' => 'B']], array_map([self::class, 'sorted'], $db->find('log', 0))); + + // The data alone gives the same (an index rebuilt from it never resurrects key 2) + \noneDB::clearStaticCache(); + unlink($this->dbPath('log', 'kix')); + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $this->assertSame([['key' => 0, 'v' => 'a'], ['key' => 1, 'v' => 'B']], array_map([self::class, 'sorted'], $db->find('log', 0))); + $this->assertSame($this->modelIndex($file), self::core($this->memIndex($db, 'log'))); + $db->insert('log', ['v' => 'd']); + $this->assertSame(3, $db->find('log', ['v' => 'd'])[0]['key'], 'key 2 is not handed out again'); + } + + /** + * @test STORAGE_FORMAT.md 6.4: the in-memory index, a checkpoint plus replay and a rebuild from the data + * all equal an independent model of the data file (offsets, lengths, d, t, s, n) + */ + public function keyIndexAlwaysEqualsAnIndependentModelOfTheDataFile(): void + { + foreach ([ + 'plain, no compaction' => ['shardingEnabled' => false, 'autoCompactThreshold' => 1], + 'plain, auto-compaction' => ['shardingEnabled' => false], + 'sharded' => ['shardSize' => 6], + ] as $case => $config) { + $this->cleanTestDirectory(); + mt_srand(77); + $db = $this->db($config); + $live = 0; + for ($op = 0; $op < 220; $op++) { + $kind = mt_rand(0, 9); + if ($kind < 5 || $live < 3) { + $batch = mt_rand(1, 3); + $rows = []; + for ($i = 0; $i < $batch; $i++) { + $rows[] = ['g' => mt_rand(0, 6), 'pad' => str_repeat('p', mt_rand(0, 20))]; + } + $live += $db->insert('mx', $rows)['n']; + } elseif ($kind < 8) { + $db->update('mx', [['g' => mt_rand(0, 6)], ['set' => ['u' => $op]]]); + } else { + $live -= $db->delete('mx', ['g' => mt_rand(0, 6)])['n']; + } + } + $this->assertSame($live, $db->count('mx'), $case); + $check = function (\noneDB $db, string $when) use ($case, $config) { + if (!isset($config['shardSize'])) { + $this->assertSame($this->modelIndex($this->dataFile('mx')), self::core($this->memIndex($db, 'mx')), "$case: $when"); + return; + } + $meta = json_decode(file_get_contents($this->dbPath('mx', 'shards')), true); + $ids = array_column($meta['shards'], 'id'); + $this->assertFileDoesNotExist($this->dataFile('mx'), "$case: switched"); + foreach ($ids as $id) { + // n of a shard other than the last never hands out keys: compared for the last one only + $withN = $id === max($ids); + $model = $this->modelIndex($this->dataFile('mx', $id), $id * 6, ($id + 1) * 6); + if (!$withN) { + unset($model['n']); + } + $this->assertSame($model, self::core($this->memIndex($db, 'mx', $id), $withN), "$case: $when, shard $id"); + } + }; + $check($db, 'in memory'); + $records = $db->find('mx', 0); + \noneDB::clearStaticCache(); + $check($this->db($config), 'checkpoint + replay'); + $this->deleteCheckpoints(); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $check($db, 'rebuilt from the data'); + $this->assertSame($records, $db->find('mx', 0), "$case: same records without checkpoints"); + } + } + + // ------------------------------------------------------------------ + // Checkpoints + // ------------------------------------------------------------------ + + /** + * @test STORAGE_FORMAT.md 6.4: a write session writes a checkpoint once the tail reached 64 lines + live/16. + * Single inserts into a new database (checkpoint at s = 0): after k inserts the tail has k lines and k + * records, k >= 64 + intdiv(k, 16) first holds for k = 68; then m more lines need + * m >= 64 + intdiv(68 + m, 16), first true for m = 72 (insert 140). + */ + public function checkpointIsWrittenWhenTheTailReaches64LinesPlusASixteenthOfTheRecords(): void + { + $db = $this->db(['shardingEnabled' => false]); + $ckpt = $this->dbPath('cp', 'kix'); + $changedAt = []; + $previous = null; + $epochs = []; + for ($i = 1; $i <= 150; $i++) { + $db->insert('cp', ['i' => $i]); + clearstatcache(); + $content = file_get_contents($ckpt); + if ($i === 1) { + $first = $this->diskIndex($ckpt); + $this->assertSame(0, $first['s'], 'the checkpoint of the new database covers nothing yet'); + $this->assertSame(0, $first['n']); + $epochs[] = $first['e']; + } elseif ($content !== $previous) { + $changedAt[] = $i; + $file = $this->diskIndex($ckpt); + $epochs[] = $file['e']; + // Contents of a checkpoint (STORAGE_FORMAT.md 6.4) + $data = (string)file_get_contents($this->dataFile('cp')); + $s = strlen($data); + $this->assertSame($s, $file['s'], "insert $i: covers the whole data file"); + $this->assertSame(hash('crc32b', substr($data, max(0, $s - 256), min(256, $s))), $file['h'], "insert $i: crc32b of the 256 bytes before s"); + $this->assertSame('kix', $file['kind']); + $this->assertSame(1, $file['v']); + $model = $this->modelIndex($this->dataFile('cp')); + $this->assertSame($model, self::core($file), "insert $i: offsets, lengths and counters"); + $this->assertSame($i, $file['n']); + // Only the documented header members (z and sid belong to shards); no runtime member (ck, tl) on disk + $members = array_keys($this->readKeyIndexFile($ckpt)['header']); + sort($members); + $this->assertSame(['base', 'created', 'd', 'e', 'h', 'kind', 'layout', 'live', 'n', 's', 'slots', 't', 'updated', 'v'], $members, "insert $i: members on disk"); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}$/', $file['e']); + } + $previous = $content; + } + $this->assertSame([68, 140], $changedAt, 'checkpoints exactly when the tail reached 64 + live/16 lines'); + $this->assertCount(1, array_unique($epochs), 'the epoch stays while the data file is the same'); + } + + /** @test STORAGE_FORMAT.md 6.4: ... or 1 MiB of tail */ + public function checkpointIsWrittenWhenTheTailReachesOneMebibyte(): void + { + $db = $this->db(['shardingEnabled' => false]); + $ckpt = $this->dbPath('mib', 'kix'); + $db->insert('mib', ['blob' => str_repeat('x', 1000000)]); // one line of ~1,000,020 bytes < 1,048,576 + $this->assertSame(0, $this->diskIndex($ckpt)['s'], '1 line, below 1 MiB: no checkpoint'); + $db->insert('mib', ['blob' => str_repeat('y', 60000)]); // tail now ~1,060,040 bytes >= 1 MiB + clearstatcache(); + $file = $this->diskIndex($ckpt); + $this->assertSame(filesize($this->dataFile('mib')), $file['s'], 'two lines, above 1 MiB: checkpointed'); + $this->assertSame(2, $file['n']); + } + + /** @test DURABILITY.md 3: checkpoints are derived files (best effort): a failing checkpoint never fails a write */ + public function aFailingCheckpointNeverFailsTheWriteOrTheRead(): void + { + $db = $this->db(['shardingEnabled' => false]); + for ($i = 0; $i < 67; $i++) { + $db->insert('cpf', ['i' => $i]); + } + $armed = true; + $this->setFaultHook(function ($point) use (&$armed) { + if ($armed && $point === 'checkpoint:written') { + throw new \RuntimeException('checkpoint failed'); + } + }); + $this->assertSame(['n' => 1], $db->insert('cpf', ['i' => 67]), 'the 68th insert ends with a checkpoint (see above), which fails'); + $armed = false; + \noneDB::clearStaticCache(); + $this->assertSame(range(0, 67), array_column($this->db(['shardingEnabled' => false])->find('cpf', 0), 'i')); + + // A reader that replays a long tail checkpoints it; that failing doesn't fail the read + $this->runChild('$db = ' . $this->childDb(['shardingEnabled' => false]) . '; + noneDB\Tests\Support\Testing::setFaultHook(function ($point) { if ($point === "scope:commit") { exit(0); } }); + $db->insert("cpf", array_map(function ($n) { return ["i" => $n]; }, range(68, 199)));'); + \noneDB::clearStaticCache(); + $armed = true; + $this->assertSame(range(0, 199), array_column($this->db(['shardingEnabled' => false])->find('cpf', 0), 'i')); + } + + /** + * @test Records are verified against their key when read; a mismatch rebuilds the index once. + * A checkpoint that fits its data file (s, h) but points key 1 at the line of key 2. + */ + public function anIndexPointingAtTheWrongLineIsRebuiltByReadsAndByCompaction(): void + { + foreach (['read', 'compact'] as $case) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('wl', [['v' => 'a'], ['v' => 'b'], ['v' => 'c']]); + $file = $this->dataFile('wl'); + $data = file_get_contents($file); + $model = $this->modelIndex($file); + $bad = $model; + $bad['p'][1] = $model['p'][2]; + $bad['l'][1] = $model['l'][2]; + $s = strlen($data); + $entries = []; + foreach ($bad['p'] as $key => $offset) { + $entries[$key] = [$offset, $bad['l'][$key]]; + } + $this->writeKeyIndexFile($this->dbPath('wl', 'kix'), ['created' => 1, 'updated' => 1, 'n' => 3, 'd' => 0, 't' => 0, + 's' => $s, 'h' => hash('crc32b', substr($data, max(0, $s - 256))), 'e' => 'abcdef01'], $entries); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + if ($case === 'compact') { + $this->assertTrue($db->compact('wl')['success']); + $this->assertSame($data, file_get_contents($file), 'compaction of a file without garbage keeps every line'); + } + $this->assertSame([['v' => 'b', 'key' => 1]], $db->find('wl', ['key' => 1]), $case); + $this->assertSame(['a', 'b', 'c'], array_column($db->find('wl', 0), 'v'), $case); + $this->assertSame(self::core($this->modelIndex($file)), self::core($this->memIndex($db, 'wl')), $case); + $disk = $this->diskIndex($this->dbPath('wl', 'kix')); + $this->assertSame($this->modelIndex($file)['p'], self::core($disk)['p'], "$case: the rebuilt checkpoint is on disk"); + $this->assertNotSame('abcdef01', $disk['e'], "$case: an index rebuilt without a checkpoint gets a new epoch"); + } + } + + // ------------------------------------------------------------------ + // Compaction + // ------------------------------------------------------------------ + + /** + * @test CONFIGURATION.md: compact "when deleted/superseded records exceed this share of its records" + * (garbage d > threshold * (live + d); exceed = strictly more) + */ + public function compactionStartsWhenGarbageExceedsTheThresholdShareOfTheRecords(): void + { + // threshold 0.5, 10 records, k deletes: d = k, live = 10 - k: k > 0.5 * 10 first for k = 6 + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 0.5]); + $db->insert('ct', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + [$original] = $this->linesOf($this->dataFile('ct')); + for ($k = 1; $k <= 5; $k++) { + $db->delete('ct', ['key' => $k - 1]); + $this->assertSame($k, $this->memIndex($db, 'ct')['d'], "$k deletes: $k of 10 records is not more than half"); + $this->assertCount(10 + $k, $this->linesOf($this->dataFile('ct'))[0]); + } + $db->delete('ct', ['key' => 5]); + $this->assertSame(0, $this->memIndex($db, 'ct')['d'], '6 of 10: compacted'); + clearstatcache(); + $this->assertSame(implode("\n", array_slice($original, 6)) . "\n", file_get_contents($this->dataFile('ct')), + 'only the live records, in key order, as they were written (the highest key 9 is live: no tombstone)'); + + // threshold 0.3 (default), 10 records, k updates of different keys: d = k, live = 10: + // k > 0.3 * (10 + k) first for k = 5 (4 > 4.2 is false, 5 > 4.5 is true) + $db = $this->db(['shardingEnabled' => false]); + $db->insert('cu', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + for ($k = 1; $k <= 4; $k++) { + $db->update('cu', [['key' => $k - 1], ['set' => ['u' => true]]]); + $this->assertSame($k, $this->memIndex($db, 'cu')['d'], "$k updates"); + } + $db->update('cu', [['key' => 4], ['set' => ['u' => true]]]); + $this->assertSame(0, $this->memIndex($db, 'cu')['d'], '5 updates: compacted'); + [$lines] = $this->linesOf($this->dataFile('cu')); + $expected = []; + foreach (range(0, 9) as $n) { + $expected[] = $n < 5 ? ['key' => $n, 'n' => $n, 'u' => true] : ['key' => $n, 'n' => $n]; + } + $this->assertSame($expected, array_map(function ($l) { return self::sorted(json_decode($l, true)); }, $lines), + 'one line per record, the current version, in key order'); + } + + /** @test docs/API.md compact(): keys never change; STORAGE_FORMAT.md 6.2: compaction keeps a tombstone for the highest key */ + public function compactionKeepsTheHighestKeySoKeysAreNeverReusedAfterAnIndexLoss(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $db->insert('hk', [['v' => 'a'], ['v' => 'b'], ['v' => 'c']]); + $db->update('hk', [['key' => 0], ['set' => ['v' => 'A']]]); + $db->delete('hk', ['key' => 2]); + [$lines] = $this->linesOf($this->dataFile('hk')); + // Lines: a, b, c, A, tombstone 2. Garbage: a (replaced) and c (deleted) + $this->assertSame(['success' => true, 'freedSlots' => 2, 'totalRecords' => 2, 'sharded' => false], $db->compact('hk')); + $this->assertSame($lines[3] . "\n" . $lines[1] . "\n" . '{"__nonedb_deleted":2}' . "\n", file_get_contents($this->dataFile('hk')), + 'live records in key order (key 0 is the line of A), then the tombstone of the highest key ever used'); + $this->assertSame(['d' => 0, 't' => 1, 'n' => 3], array_intersect_key(self::core($this->memIndex($db, 'hk')), ['d' => 0, 't' => 0, 'n' => 0])); + + $this->deleteCheckpoints(); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('hk', ['v' => 'd']); + $this->assertSame([0, 1, 3], array_column($db->find('hk', 0), 'key'), 'key 2 is not reused after compaction and index loss'); + + // Sharded: shard 1 (keys 5..9) loses 7, 8, 9; its auto-compaction keeps tombstone 9 + $db = $this->db(['shardSize' => 5]); + $db->insert('hks', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + [$shard1] = $this->linesOf($this->dataFile('hks', 1)); + $this->assertSame(['n' => 3], $db->delete('hks', ['key' => [7, 8, 9]])); + // shard 1: d = 3 > 0.3 * (2 + 3): compacted + $this->assertSame($shard1[0] . "\n" . $shard1[1] . "\n" . '{"__nonedb_deleted":9}' . "\n", file_get_contents($this->dataFile('hks', 1))); + foreach (['indexes lost' => false, 'indexes and meta lost' => true] as $case => $loseMeta) { + $this->deleteCheckpoints(); + if ($loseMeta) { + file_put_contents($this->dbPath('hks', 'shards'), ''); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $db->insert('hks', ['n' => $case]); + $key = $db->find('hks', ['n' => $case])[0]['key']; + $this->assertSame($loseMeta ? 11 : 10, $key, "$case: no key used before is handed out again"); + } + } + + /** + * @test CONFIGURATION.md "Recovery log and quarantine": damaged lines are quarantined by compaction, a torn + * tail by the next write (here: compaction itself); nothing is dropped without a verified copy + */ + public function compactionQuarantinesDamagedLinesAndTheTornTail(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('cq', [['v' => 'a'], ['v' => 'b'], ['v' => 'c']]); + $file = $this->dataFile('cq'); + $clean = file_get_contents($file); + $garbage = "garbage line\n"; + $torn = '{"torn'; + file_put_contents($file, $garbage . $torn, FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertTrue($db->compact('cq')['success']); + + $this->assertSame($clean, file_get_contents($file), 'only the three records are left'); + $expected = [ + $this->quarantineName('cq', 'main', strlen($clean), $garbage), + $this->quarantineName('cq', 'main', strlen($clean) + strlen($garbage), $torn), + ]; + sort($expected); + $this->assertSame($expected, $this->quarantineFiles('cq')); + $this->assertSame($garbage, file_get_contents($this->dbPath('cq', 'quarantine') . $this->quarantineName('cq', 'main', strlen($clean), $garbage))); + $this->assertSame($torn, file_get_contents($this->dbPath('cq', 'quarantine') . $this->quarantineName('cq', 'main', strlen($clean) + strlen($garbage), $torn))); + $log = $db->getRecoveryLog('cq'); + $actions = array_column($log, 'action'); + sort($actions); + $this->assertSame(['cut-tail', 'drop-line', 'skip-line'], $actions); + foreach ($log as $entry) { + if ($entry['action'] !== 'skip-line') { + $this->assertTrue($entry['bytesVerified'], $entry['action']); + $this->assertContains($entry['quarantine'], $expected, $entry['action']); + } + } + $this->assertSame(['a', 'b', 'c'], array_column($db->find('cq', 0), 'v')); + } + + /** + * @test compaction never keeps an unterminated tail, also one that appeared after this process checked its + * index against the file: the bytes are removed into quarantine (cut-tail or drop-line, both documented as + * "bytes removed from a data or buffer file, kept in the quarantine file") + */ + public function compactionNeverKeepsATornTailItFinds(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ct2', [['v' => 'a'], ['v' => 'b']]); + $file = $this->dataFile('ct2'); + $clean = file_get_contents($file); + file_put_contents($file, '{"torn', FILE_APPEND); + $this->assertTrue($db->compact('ct2')['success']); + clearstatcache(); + $this->assertSame($clean, file_get_contents($file)); + $name = $this->quarantineName('ct2', 'main', strlen($clean), '{"torn'); + $this->assertSame([$name], $this->quarantineFiles('ct2')); + $this->assertSame('{"torn', file_get_contents($this->dbPath('ct2', 'quarantine') . $name)); + $log = $db->getRecoveryLog('ct2'); + $this->assertCount(1, $log); + $this->assertContains($log[0]['action'], ['cut-tail', 'drop-line']); + $this->assertSame($name, $log[0]['quarantine']); + $db->insert('ct2', ['v' => 'c']); + $this->assertSame([0, 1, 2], array_column($db->find('ct2', 0), 'key')); + } + + // ------------------------------------------------------------------ + // Torn tails and damaged lines + // ------------------------------------------------------------------ + + /** + * @test STORAGE_FORMAT.md 6.1, 7.2 (recovery never drops bytes silently): the next writer copies an + * unterminated tail to recovery/q.{segment}.{offset}.{length}.{sha256}.bin (main, s{N}, buffer), verifies it, + * cuts it, logs cut-tail, and appends where the tail started + */ + public function tornTailIsQuarantinedExactlyAndTheNextLineStartsWhereItStarted(): void + { + $fragment = '{"v":"half' . "\xE2\x82"; // cut inside a UTF-8 sequence, like a torn write + $cases = [ + 'main file' => [['shardingEnabled' => false], null, 'main'], + 'shard 1' => [['shardSize' => 5], 1, 's1'], + ]; + foreach ($cases as $case => [$config, $shard, $segment]) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('tq', array_map(function ($n) { return ['v' => $n]; }, range(0, 6))); + $file = $this->dataFile('tq', $shard); + $before = file_get_contents($file); + file_put_contents($file, $fragment, FILE_APPEND); + + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(range(0, 6), array_column($db->find('tq', 0), 'v'), "$case: the tail is not a record"); + clearstatcache(); + $this->assertSame(strlen($before) + strlen($fragment), filesize($file), "$case: readers never cut"); + + $db->insert('tq', ['v' => 'next']); + $name = $this->quarantineName('tq', $segment, strlen($before), $fragment); + $this->assertSame([$name], $this->quarantineFiles('tq'), $case); + $this->assertSame($fragment, file_get_contents($this->dbPath('tq', 'quarantine') . $name), "$case: the exact bytes"); + $after = file_get_contents($file); + $this->assertSame($before, substr($after, 0, strlen($before)), "$case: nothing before the tail changed"); + $rest = substr($after, strlen($before)); + $this->assertSame("\n", substr($rest, -1)); + $this->assertSame(1, substr_count($rest, "\n"), "$case: exactly one new line where the tail was"); + $this->assertSame(['key' => 7, 'v' => 'next'], self::sorted(json_decode($rest, true))); + $log = $db->getRecoveryLog('tq'); + $this->assertSame(['cut-tail'], array_column($log, 'action'), $case); + $this->assertSame($name, $log[0]['quarantine'], $case); + $this->assertTrue($log[0]['bytesVerified'], $case); + } + + // Buffer (segment buffer): the next buffered insert cuts the tail where the last complete line ends + $this->cleanTestDirectory(); + $config = ['bufferEnabled' => true, 'shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('tb', [['v' => 0], ['v' => 1]]); + $buffer = $this->dbPath('tb', 'buffer'); + $before = file_get_contents($buffer); + file_put_contents($buffer, $fragment, FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(['n' => 1], $db->insert('tb', ['v' => 2])); + $name = $this->quarantineName('tb', 'buffer', strlen($before), $fragment); + $this->assertSame([$name], $this->quarantineFiles('tb'), 'buffer'); + $this->assertSame($fragment, file_get_contents($this->dbPath('tb', 'quarantine') . $name)); + $this->assertSame($before . '{"v":2}' . "\n", file_get_contents($buffer), 'buffered records carry no key yet'); + $this->assertSame(['cut-tail'], array_column($db->getRecoveryLog('tb'), 'action')); + $this->assertSame([['v' => 0, 'key' => 0], ['v' => 1, 'key' => 1], ['v' => 2, 'key' => 2]], $db->find('tb', 0)); + + // A read flushes a buffer whose tail is torn: its records are stored, the whole buffer file is kept in + // quarantine before it is deleted (drop-buffer) + $db->insert('tb2', [['v' => 0], ['v' => 1]]); + $buffer = $this->dbPath('tb2', 'buffer'); + file_put_contents($buffer, $fragment, FILE_APPEND); + $whole = file_get_contents($buffer); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame([['v' => 0, 'key' => 0], ['v' => 1, 'key' => 1]], $db->find('tb2', 0)); + $this->assertSame([], glob($this->dbPath('tb2', 'gen') . 'buffer/*')); + $this->assertSame([$this->quarantineName('tb2', 'buffer', 0, $whole)], $this->quarantineFiles('tb2')); + $this->assertSame($whole, file_get_contents($this->dbPath('tb2', 'quarantine') . $this->quarantineName('tb2', 'buffer', 0, $whole))); + $this->assertSame(['drop-buffer'], array_column($db->getRecoveryLog('tb2'), 'action')); + } + + /** @test the next buffered insert cuts a torn buffer tail at its last complete line, also beyond 64 KiB */ + public function aTornBufferTailLongerThan64KiBIsCutAtTheLastCompleteLine(): void + { + $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false]); + $db->insert('bt', ['v' => 'one']); + $buffer = $this->dbPath('bt', 'buffer'); + $before = file_get_contents($buffer); + $fragment = '{"v":"' . str_repeat('z', 70000); // no "\n" in the last 64 KiB of the file + file_put_contents($buffer, $fragment, FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false]); + $db->insert('bt', ['v' => 'two']); + $this->assertSame([$this->quarantineName('bt', 'buffer', strlen($before), $fragment)], $this->quarantineFiles('bt')); + [$lines, $tail] = $this->linesOf($buffer); + $this->assertSame('', $tail); + $this->assertSame([['v' => 'one'], ['v' => 'two']], array_map(function ($l) { return json_decode($l, true); }, $lines)); + $this->assertSame(['one', 'two'], array_column($db->find('bt', 0), 'v')); + } + + /** + * @test STORAGE_FORMAT.md 6.1: a record line is a JSON object with an integer key of the segment; a tombstone is exactly + * {"__nonedb_deleted":K}. Anything else is a damaged line: never a record, never a deletion. + */ + public function linesThatAreNeitherRecordsNorTombstonesAreNeverApplied(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('nl', [['v' => 'zero'], ['v' => 'one']]); + $bad = [ + '{"v":"no key"}', + '{"key":"1","v":"string key"}', + '{"key":1.0,"v":"float key"}', + '{"key":-1,"v":"negative key"}', + '{"__nonedb_deleted":"1"}', + '{"__nonedb_deleted":1,"x":2}', + '[1,2,3]', + ]; + file_put_contents($this->dataFile('nl'), implode("\n", $bad) . "\n", FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([['v' => 'zero', 'key' => 0], ['v' => 'one', 'key' => 1]], $db->find('nl', 0)); + $index = $this->memIndex($db, 'nl'); + $this->assertSame(count($bad), $index['d'], 'each damaged line is garbage'); + $this->assertSame(0, $index['t'], 'no tombstone among them'); + $this->assertSame(2, $index['n']); + $db->insert('nl', ['v' => 'two']); + $this->assertSame([0, 1, 2], array_column($db->find('nl', 0), 'key')); + $this->assertSame(array_fill(0, count($bad), 'skip-line'), array_column($db->getRecoveryLog('nl'), 'action'), + 'each one reported, logged by the write session'); + + // Shards: a line whose key belongs to another shard (key K lives in shard intdiv(K, shardSize)) + $db = $this->db(['shardSize' => 5]); + $db->insert('nls', array_map(function ($n) { return ['n' => $n]; }, range(0, 6))); + file_put_contents($this->dataFile('nls', 1), '{"n":"intruder","key":2}' . "\n", FILE_APPEND); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $this->assertSame([['n' => 2, 'key' => 2]], $db->find('nls', ['key' => 2])); + $this->assertSame(range(0, 6), array_column($db->find('nls', 0), 'n')); + $this->assertSame(1, $this->memIndex($db, 'nls', 1)['d']); + $this->assertSame(['n' => 1], $db->delete('nls', ['key' => 2])); + $this->assertSame([0, 1, 3, 4, 5, 6], array_column($db->find('nls', 0), 'key'), 'a deleted key never comes back'); + // checkIntegrity() reports it as a damaged line of shard 1: after {"n":5,"key":5} and {"n":6,"key":6} + // (16 bytes each with "\n") at offset 32, 24 bytes + "\n" + $report = $db->checkIntegrity('nls'); + $this->assertFalse($report['ok']); + $this->assertSame([[1, 32, 25]], $report['badLines']); + // compaction quarantines it (drop-line); then nothing is left to report + $this->assertTrue($db->compact('nls')['success']); + $this->assertSame([0, 1, 3, 4, 5, 6], array_column($db->find('nls', 0), 'key')); + $this->assertTrue($db->checkIntegrity('nls')['ok']); + $this->assertContains('drop-line', array_column($db->getRecoveryLog('nls'), 'action')); + } + + /** + * @test CONFIGURATION.md: "An existing file with the same name is reused only if its bytes match"; if the + * copy can't be verified, nothing is cut and the write fails + */ + public function aQuarantineFileWithTheSameNameButOtherBytesRefusesTheCut(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('qm', ['v' => 'first']); + $file = $this->dataFile('qm'); + $before = file_get_contents($file); + $fragment = '{"half'; + file_put_contents($file, $fragment, FILE_APPEND); + $planted = $this->dbPath('qm', 'quarantine') . $this->quarantineName('qm', 'main', strlen($before), $fragment); + @mkdir(dirname($planted), 0777, true); + file_put_contents($planted, 'XXXXXX'); // same length, other bytes + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + try { + $db->insert('qm', ['v' => 'second']); + $this->fail('the write must be refused'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('quarantine', $e->getMessage()); + } + clearstatcache(); + $this->assertSame($before . $fragment, file_get_contents($file), 'nothing cut, nothing appended'); + $this->assertSame('XXXXXX', file_get_contents($planted), 'the existing file is left alone'); + $this->assertSame([], $this->tempFiles(), 'no temp file left'); + $this->assertSame(['first'], array_column($db->find('qm', 0), 'v')); + + unlink($planted); + $db->insert('qm', ['v' => 'second']); + $this->assertSame($fragment, file_get_contents($planted)); + $this->assertSame(['first', 'second'], array_column($db->find('qm', 0), 'v')); + } + + // ------------------------------------------------------------------ + // Write buffer + // ------------------------------------------------------------------ + + private function bufferLines(string $name): array + { + return $this->linesOf($this->dbPath($name, 'buffer'))[0]; + } + + private function dataRecords(string $name): array + { + return array_map(function ($l) { return json_decode($l, true); }, $this->linesOf($this->dataFile($name))[0]); + } + + /** + * @test buffer flush triggers (docs/API.md buffer limits): the count + * limit, the size limit, a buffer already over a limit before an append, and the flush interval + */ + public function theBufferIsFlushedAtItsCountSizeAndTimeLimits(): void + { + $config = ['bufferEnabled' => true, 'shardingEnabled' => false]; + + // Count limit 10: nine inserts stay buffered, the tenth flushes all of them in order + $db = $this->db($config); + $db->setBufferCountLimit(10); + for ($i = 0; $i < 9; $i++) { + $this->assertSame(['n' => 1], $db->insert('bc', ['i' => $i])); + } + $this->assertCount(9, $this->bufferLines('bc')); + $this->assertSame([], $this->dataRecords('bc'), 'nothing in the data file yet'); + $db->insert('bc', ['i' => 9]); + $this->assertSame([], $this->bufferLines('bc')); + $expected = []; + foreach (range(0, 9) as $i) { + $expected[] = ['i' => $i, 'key' => $i]; + } + $this->assertSame($expected, array_map([self::class, 'sorted'], $this->dataRecords('bc')), 'buffer order = key order'); + + // Size limit 1024 bytes: lines {"s":"<191 x>"}\n are 200 bytes; 5 = 1000 < 1024, 6 = 1200 >= 1024 + $db = $this->db($config); + $db->setBufferSizeLimit(1024); + for ($i = 0; $i < 5; $i++) { + $db->insert('bs', ['s' => str_repeat('x', 191)]); + } + clearstatcache(); + $this->assertSame(1000, filesize($this->dbPath('bs', 'buffer'))); + $this->assertSame([], $this->dataRecords('bs')); + $db->insert('bs', ['s' => str_repeat('x', 191)]); + $this->assertSame([], $this->bufferLines('bs')); + $this->assertSame(range(0, 5), array_column($this->dataRecords('bs'), 'key')); + + // Already over the limit before the append (limit lowered): flushed first, the new record starts a buffer + $db = $this->db($config); + for ($i = 0; $i < 4; $i++) { + $db->insert('bo', ['s' => str_repeat('y', 291), 'i' => $i]); // 4 lines of 306 bytes (1224), default 1 MiB limit + } + $this->assertCount(4, $this->bufferLines('bo')); + $db->setBufferSizeLimit(1024); + $db->insert('bo', ['i' => 4]); + $this->assertSame([0, 1, 2, 3], array_column($this->dataRecords('bo'), 'i')); + $this->assertSame([['i' => 4]], array_map(function ($l) { return json_decode($l, true); }, $this->bufferLines('bo'))); + + // Flush interval 30 s: a buffer whose last flush is 31 s old is flushed by the next insert (before its + // append, which then starts a new buffer) + $db = $this->db($config); + $db->setBufferFlushInterval(30); + $db->insert('bi', ['i' => 0]); + $this->setInstanceProperty($db, 'bufferLastFlush', ['bi' => time() - 31]); + $db->insert('bi', ['i' => 1]); + $this->assertSame([0], array_column($this->dataRecords('bi'), 'i')); + $this->assertCount(1, $this->bufferLines('bi')); + + // flush(): documented result, then nothing left to flush + $this->assertSame(['success' => true, 'flushed' => 1, 'error' => null], $db->flush('bi')); + $this->assertSame(['success' => true, 'flushed' => 0, 'error' => null], $db->flush('bi')); + $this->assertSame([0, 1], array_column($db->find('bi', 0), 'i')); + $this->assertSame([0, 1], array_column($db->find('bi', 0), 'key')); + } + + /** @test buffered records are flushed when the script ends normally (bufferAutoFlushOnShutdown) */ + public function bufferedRecordsAreFlushedWhenTheProcessEnds(): void + { + $out = $this->runChild('$db = ' . $this->childDb(['bufferEnabled' => true, 'shardingEnabled' => false]) . '; + foreach (["a", "b", "c"] as $v) { $db->insert("sd", ["v" => $v]); } + clearstatcache(); + echo is_file(' . var_export($this->dbPath('sd', 'buffer'), true) . ') ? "buffered" : "not buffered";'); + $this->assertSame('buffered', $out, 'the records were in the buffer when the script ended'); + $this->assertFileDoesNotExist($this->dbPath('sd', 'buffer')); + $this->assertSame([['key' => 0, 'v' => 'a'], ['key' => 1, 'v' => 'b'], ['key' => 2, 'v' => 'c']], + array_map([self::class, 'sorted'], $this->dataRecords('sd')), 'flushed at shutdown, in order'); + } + + /** + * @test STORAGE_FORMAT.md 5.1: a flush renames the buffer to buffer/flushing.{startKey}.jsonl; replaying it + * after a crash inserts each record exactly once (keys startKey, startKey+1, ...; stored keys are skipped) + */ + public function anInterruptedBufferFlushIsReplayedExactlyOnce(): void + { + $hook = ''; + + // Non-sharded: the writer dies right after appending every buffered record to the data file + $config = ['bufferEnabled' => true, 'shardingEnabled' => false]; + $this->runChild('$db = ' . $this->childDb($config) . '; + foreach (["a", "b", "c"] as $v) { $db->insert("fx", ["v" => $v]); } + ' . $hook . ' + noneDB\Tests\Support\Testing::setFaultHook(function ($point, $ctx) { if ($point === "append:after" && substr($ctx["path"] ?? "", -20) === "/segments/main.jsonl") { exit(0); } }); + $db->flush("fx");'); + $this->assertFileExists($this->dbPath('fx', 'flushing', 0), 'precondition: the flush was interrupted after its rename'); + $this->assertCount(3, $this->dataRecords('fx'), 'precondition: the records were appended'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertSame([['v' => 'a', 'key' => 0], ['v' => 'b', 'key' => 1], ['v' => 'c', 'key' => 2]], $db->find('fx', 0)); + $this->assertCount(3, $this->dataRecords('fx'), 'nothing appended twice'); + $this->assertSame([], glob($this->dbPath('fx', 'gen') . 'buffer/*')); + $this->assertNotContains('b', $this->lockState('fx')['flags'], 'buffer flag cleared'); + + // Sharded: keys 7, 8, 9 go to shard 1 and 10 to shard 2; the writer dies after the shard 1 append + $config = ['bufferEnabled' => true, 'shardSize' => 5]; + $this->db(['shardSize' => 5])->insert('fxs', array_map(function ($n) { return ['v' => $n]; }, range(0, 6))); + \noneDB::clearStaticCache(); + $this->runChild('$db = ' . $this->childDb($config) . '; + foreach ([7, 8, 9, 10] as $v) { $db->insert("fxs", ["v" => $v]); } + ' . $hook . ' + noneDB\Tests\Support\Testing::setFaultHook(function ($point, $ctx) { if ($point === "append:after" && preg_match("#/segments/s\\\\d+\\\\.jsonl$#", $ctx["path"] ?? "")) { exit(0); } }); + $db->flush("fxs");'); + $this->assertFileExists($this->dbPath('fxs', 'flushing', 7), 'precondition: interrupted flush starting at key 7'); + $this->assertSame([5, 6, 7, 8, 9], array_column(array_map(function ($l) { return json_decode($l, true); }, + $this->linesOf($this->dataFile('fxs', 1))[0]), 'key'), 'precondition: only the shard 1 append happened'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $all = $db->find('fxs', 0); + $this->assertSame(range(0, 10), array_column($all, 'key')); + $this->assertSame(range(0, 10), array_column($all, 'v'), 'every buffered record exactly once, in buffer order'); + $this->assertSame(11, $db->count('fxs')); + $this->assertSame(11, $db->getShardInfo('fxs')['nextKey']); + $this->assertSame([], glob($this->dbPath('fxs', 'gen') . 'buffer/*')); + } + + /** + * @test docs/DURABILITY.md §3, §4: in durable mode appended buffer lines are synced; a failed sync of an + * append refuses the write and its bytes are cut back (DURABILITY_FAILED, rolled back) + */ + public function aBufferedInsertWhoseSyncFailsIsRefusedAndUndone(): void + { + $this->setStatic('osFamily', 'Linux'); // durable mode's rules, on every platform + $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false, 'durability' => 'durable']); + $db->insert('bf', ['v' => 'a']); + $buffer = $this->dbPath('bf', 'buffer'); + $bytes = file_get_contents($buffer); + $failed = 0; + $this->setStatic('syncFn', function ($handle, string $kind, $path) use (&$failed) { + // the append's sync of the buffer file fails; the sync of its cut succeeds + return !($kind === 'file' && substr((string)$path, -21) === '/buffer/pending.jsonl' && ++$failed === 1); + }); + try { + $db->insert('bf', ['v' => 'b']); + $this->fail('the insert must be refused'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('fsync', $e->getMessage()); + $this->assertSame(\noneDB::DURABILITY_FAILED, $e->getCode()); + } + $this->assertSame(2, $failed, 'the append and its cut were synced'); + $this->setStatic('syncFn', null); + clearstatcache(); + $this->assertSame($bytes, file_get_contents($buffer), 'the appended line was undone'); + $this->assertSame([['v' => 'a', 'key' => 0]], $db->find('bf', 0)); + } + + /** @test buffered records survive a flush that can't rename the buffer (the flush fails, nothing is lost) */ + public function aBufferThatCannotBeRenamedForItsFlushKeepsItsRecords(): void + { + $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false]); + $db->insert('br', [['v' => 'a'], ['v' => 'b']]); + $buffer = $this->dbPath('br', 'buffer'); + $bytes = file_get_contents($buffer); + $this->setStatic('renameFn', function ($from, $to) { + return strpos($from, '/buffer/pending.jsonl') !== false ? false : rename($from, $to); + }); + try { + $db->flush('br'); + $this->fail('the flush must fail'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('buffer', $e->getMessage()); + } + $this->setStatic('renameFn', null); + clearstatcache(); + $this->assertSame($bytes, file_get_contents($buffer), 'the buffer is untouched'); + $this->assertSame([], $this->dataRecords('br')); + $this->assertSame([['v' => 'a', 'key' => 0], ['v' => 'b', 'key' => 1]], $db->find('br', 0)); + } + + // ------------------------------------------------------------------ + // JSON encoding + // ------------------------------------------------------------------ + + /** + * @test docs/API.md "Floats keep their type"; serialize_precision never truncates stored floats + * (STORAGE_FORMAT.md 2: serialize_precision = -1); "Values JSON can't represent (invalid UTF-8, NAN, INF): nothing is written" + */ + public function everyValueIsStoredExactlyWhateverTheSerializePrecision(): void + { + $this->precision = ini_get('serialize_precision'); + ini_set('serialize_precision', '5'); + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); // keep both lines + $values = [ + 'f' => 28.980345, + 'sum' => 0.1 + 0.2, + 'huge' => 1e100, + 'max' => 1.7976931348623157e308, + 'tiny' => 5e-324, + 'two' => 2.0, + 'negzero' => -0.0, + 'imax' => PHP_INT_MAX, + 'imin' => PHP_INT_MIN, + 'tr' => 'şğüöçıİ', + 'emoji' => "😀 👩\u{200D}💻", + 'cjk' => '中文字符', + 'ctrl' => "a\nb\r\nc\td\0e\u{2028}f\"g\\h/i", + "field\nwith\nnewlines" => 'ok', + 'ключ' => 'значение', + 'nested' => ['a' => [1, 2.5, ['b' => -0.5]]], + 'empty' => [], + 'emptyObject' => new \stdClass(), + ]; + $this->assertSame(['n' => 1], $db->insert('fl', $values)); + $this->assertSame(['n' => 1], $db->update('fl', [['key' => 0], ['set' => ['upd' => 1.0000000000000002, 'f2' => 0.30000000000000004]]])); + ini_set('serialize_precision', (string)$this->precision); + + [$lines, $tail] = $this->linesOf($this->dataFile('fl')); + $this->assertCount(2, $lines, 'strings with line breaks never break a line'); + $this->assertSame('', $tail); + $stored = json_decode($lines[1], true); + \noneDB::clearStaticCache(); + $found = $this->db(['shardingEnabled' => false])->find('fl', 0)[0]; + foreach ([$stored, $found] as $which => $record) { + $label = $which === 0 ? 'data file' : 'find()'; + foreach (['f', 'sum', 'huge', 'max', 'tiny', 'two', 'imax', 'imin', 'tr', 'emoji', 'cjk', 'ctrl', "field\nwith\nnewlines", 'ключ', 'nested'] as $field) { + $this->assertSame($values[$field], $record[$field], "$label: $field"); + } + $this->assertIsFloat($record['two'], "$label: 2.0 stays a float"); + $this->assertIsFloat($record['negzero'], "$label: -0.0 stays a float"); + $this->assertSame('-0.0', json_encode($record['negzero'], JSON_PRESERVE_ZERO_FRACTION), "$label: and keeps its sign"); + $this->assertSame([], $record['empty']); + $this->assertSame([], $record['emptyObject'], "$label: an object is stored as JSON"); + $this->assertSame(1.0000000000000002, $record['upd'], $label); + $this->assertSame(0.30000000000000004, $record['f2'], $label); + $this->assertSame(0, $record['key']); + } + } + + /** @test docs/API.md insert()/update() errors: values JSON can't represent are rejected, nothing is written */ + public function unencodableValuesAreRejectedAndNothingIsWritten(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ue', ['v' => 'ok']); + $db->find('ue', 0); + $before = $this->snapshot('ue'); + $this->precision = ini_get('serialize_precision'); + foreach (['-1', '5'] as $precision) { + ini_set('serialize_precision', $precision); + foreach (['nan' => NAN, 'inf' => INF, '-inf' => -INF, 'utf8' => "bad\xB1", 'nested' => ['a' => ['b' => NAN]]] as $case => $value) { + foreach ([['v' => $value], [['v' => 'fine'], ['v' => $value]]] as $i => $data) { + $r = $db->insert('ue', $data); + $this->assertSame([0, 'n'], [$r['n'], array_key_first($r)], "precision $precision, $case, insert $i"); + $this->assertStringStartsWith('insert data cannot be encoded as JSON', $r['error'], "precision $precision, $case, insert $i"); + } + $r = $db->update('ue', [['v' => 'ok'], ['set' => ['w' => $value]]]); + $this->assertSame(0, $r['n'], "precision $precision, $case, update"); + $this->assertStringStartsWith('update data cannot be encoded as JSON', $r['error'], "precision $precision, $case, update"); + } + } + ini_set('serialize_precision', (string)$this->precision); + $this->assertSame($before, $this->snapshot('ue'), 'no file written, not even the lock state'); + $this->assertSame([['v' => 'ok', 'key' => 0]], $db->find('ue', 0)); + } + + // ------------------------------------------------------------------ + // CRUD contract + // ------------------------------------------------------------------ + + /** @test docs/API.md: the documented error results of insert(), update() and delete(), and no file is touched */ + public function invalidArgumentsReturnTheDocumentedErrorsAndWriteNothing(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('ia', [['v' => 1], ['v' => 2]]); + $db->find('ia', 0); + $before = $this->snapshot('ia'); + $gen = $this->generation('ia'); + + $insert = [ + 'insert data must be array' => ['string', null, 42, 1.5, true, new \stdClass()], + 'insert data must not be empty' => [[]], + 'insert data must be either one record or a list of records' => [ + [0 => ['a' => 1], 'name' => 'x'], // documented mix + [0 => ['a' => 1], 2 => ['b' => 2]], // a gap: not 0..n-1 + [1 => ['a' => 1]], // does not start at 0 + [['a' => 1], 'scalar'], // an element that isn't a record + ], + 'You cannot set key name to key' => [['key' => 'value'], ['key' => null], [['a' => 1], ['key' => 2, 'b' => 3]]], + ]; + foreach ($insert as $error => $cases) { + foreach ($cases as $i => $data) { + $this->assertSame(['n' => 0, 'error' => $error], $db->insert('ia', $data), "insert: $error #$i"); + } + } + $updateError = ['n' => 0, 'error' => 'Please check your update parameters']; + foreach ([ + 'string', null, 5, [], [['v' => 1]], ['v', ['set' => ['a' => 1]]], [1 => ['set' => ['a' => 1]]], + [['v' => 1], 'set'], [['v' => 1], []], [['v' => 1], ['values' => ['a' => 1]]], [['v' => 1], ['set' => 'a']], + [['v' => 1], ['set' => ['key' => 5]]], [[], ['set' => ['a' => 1, 'key' => 5]]], + ] as $i => $data) { + $this->assertSame($updateError, $db->update('ia', $data), "update #$i"); + } + foreach (['string', 0, null, 1.5, true] as $i => $filter) { + $this->assertSame(['n' => 0, 'error' => 'Please check your delete parameters'], $db->delete('ia', $filter), "delete #$i"); + } + $this->assertSame($before, $this->snapshot('ia'), 'no file written'); + $this->assertSame($gen, $this->generation('ia')); + $this->assertSame([1, 2], array_column($db->find('ia', 0), 'v')); + + // A record with a nested "key" and a single record whose first field is numeric are valid + $this->assertSame(['n' => 1], $db->insert('ia', ['data' => ['key' => 'nested']])); + $this->assertSame(['n' => 1], $db->insert('ia', ['a' => 1, 0 => ['x']])); + $this->assertSame([['data' => ['key' => 'nested'], 'key' => 2]], $db->find('ia', ['key' => 2])); + $this->assertSame([['a' => 1, 0 => ['x'], 'key' => 3]], $db->find('ia', ['key' => 3])); + } + + /** + * @test The generation moves only when a file is written: sessions that write nothing (an update that matches + * nothing, checkDB of an existing database, an empty flush) keep the other processes' caches + */ + public function writesThatChangeNothingWriteNoFileAndKeepTheGeneration(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('nw', [['v' => 1], ['v' => 2]]); + $db->delete('nw', ['v' => 2]); + $db->find('nw', 0); + $before = $this->snapshot('nw'); + $gen = $this->generation('nw'); + + $this->assertSame(['n' => 0], $db->update('nw', [['v' => 99], ['set' => ['x' => 1]]])); + $this->assertSame(['n' => 0], $db->delete('nw', ['v' => 99])); + $this->assertSame(['n' => 0], $db->delete('nw', ['key' => 1]), 'already deleted'); + $this->assertSame(['n' => 1], $db->update('nw', [['v' => 1], ['set' => []]]), 'empty set: the number of matches'); + $this->assertSame(['success' => true, 'flushed' => 0, 'error' => null], $db->flush('nw')); + $this->assertTrue($db->checkDB('nw')); + $this->assertFalse($db->createDB('nw'), 'already exists'); + + $this->assertSame($before, $this->snapshot('nw'), 'no file written'); + $this->assertSame($gen, $this->generation('nw')); + } + + /** @test docs/CONFIGURATION.md autoCreateDB false: writes don't create a database, reads return false */ + public function withoutAutoCreateNothingCreatesADatabase(): void + { + foreach ([false, true] as $buffered) { + $this->cleanTestDirectory(); + $db = $this->db(['autoCreateDB' => false, 'bufferEnabled' => $buffered]); + $this->assertFalse($db->find('ac', 0)); + // Same wording as the documented dropDB() error + $this->assertSame(['n' => 0, 'error' => 'Database does not exist'], $db->insert('ac', ['v' => 1]), $buffered ? 'buffered' : 'direct'); + $this->assertSame(['n' => 0], $db->update('ac', [[], ['set' => ['v' => 2]]])); + $this->assertSame(['n' => 0], $db->delete('ac', [])); + $this->assertFalse($db->find('ac', 0)); + $this->assertFileDoesNotExist($this->dataFile('ac')); + $this->assertFileDoesNotExist($this->dbPath('ac', 'format')); + $this->assertFileDoesNotExist($this->dbPath('ac', 'buffer')); + } + } + + // ------------------------------------------------------------------ + // Sharding + // ------------------------------------------------------------------ + + /** @test docs/API.md convertToSharded(); STORAGE_FORMAT.md 5.1: key K lives in shard intdiv(K, shardSize), keys never change */ + public function theSwitchToShardsKeepsEveryKeyAndRecordAndSplitsTheDataByKeyRange(): void + { + foreach (['indexes lost' => 'kix', 'meta lost' => 'meta'] as $case => $lost) { + $this->cleanTestDirectory(); + $db = $this->db(['shardSize' => 4, 'shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $model = []; + foreach (range(0, 10) as $k) { + $model[$k] = ['n' => $k, 'tag' => 't' . ($k % 3)]; + } + $db->insert('mg', array_values($model)); + $db->update('mg', [['key' => [1, 9]], ['set' => ['tag' => 'upd']]]); + $model[1]['tag'] = $model[9]['tag'] = 'upd'; + $db->delete('mg', ['key' => [2, 5, 10]]); // 10: the highest key + unset($model[2], $model[5], $model[10]); + $this->assertFalse($db->getShardInfo('mg')['sharded'], 'shardingEnabled off'); + + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('mg')); + // The single layout's segment (generation 1) is gone (STORAGE_FORMAT.md 5.2 CLEANUP) + $this->assertFileDoesNotExist($this->dbPath('mg', 'home') . 'generations/1/segments/main.jsonl'); + $this->assertFileDoesNotExist($this->dbPath('mg', 'home') . 'generations/1/segments/main.kix'); + $this->assertFileDoesNotExist($this->dataFile('mg')); + // nextKey 11: shards intdiv(10, 4) + 1 = 3, holding [0,4), [4,8), [8,12) + foreach ([0 => [0, 1, 3], 1 => [4, 6, 7], 2 => [8, 9]] as $shard => $keys) { + $records = []; + foreach ($this->linesOf($this->dataFile('mg', $shard))[0] as $line) { + $decoded = json_decode($line, true); + if (array_keys($decoded) === ['__nonedb_deleted']) { + // a tombstone may only keep a deleted key of this shard + $this->assertContains($decoded['__nonedb_deleted'], [8, 9, 10, 11], "shard $shard"); + $this->assertArrayNotHasKey($decoded['__nonedb_deleted'], $model); + continue; + } + $records[] = self::sorted($decoded); + } + $expected = []; + foreach ($keys as $k) { + $expected[] = self::sorted($model[$k] + ['key' => $k]); + } + $this->assertSame($expected, $records, "shard $shard: its key range, in key order, current versions only"); + } + $this->assertSame(['sharded' => true, 'shards' => 3, 'totalRecords' => 8, 'deletedCount' => 0, 'shardSize' => 4, 'nextKey' => 11], + $db->getShardInfo('mg')); + $expectedAll = []; + foreach ($model as $k => $r) { + $expectedAll[] = $r + ['key' => $k]; + } + $this->assertSame($expectedAll, $db->find('mg', 0)); + $this->assertSame(['success' => true, 'status' => 'already_sharded'], $db->convertToSharded('mg')); + + // Keys aren't reused after an index loss, nor with a lost shards.json + if ($lost === 'kix') { + $this->deleteCheckpoints(); + } else { + file_put_contents($this->dbPath('mg', 'shards'), ''); + } + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + $this->assertSame($expectedAll, $db->find('mg', 0), $case); + $db->insert('mg', ['n' => 'new']); + $this->assertSame(11, $db->find('mg', ['n' => 'new'])[0]['key'], "$case: the deleted highest key 10 is not reused"); + } + } + + /** + * @test Compaction and the switch to shards keep a tombstone for the highest key, so keys aren't reused after an index + * loss either (STORAGE_FORMAT.md 6.2). After a switch whose highest keys (9, 10) were deleted, the shard + * files alone (shards.json and checkpoints lost, as a power loss can leave them) must still keep key 10: the + * next key is 11. compactionKeepsTheHighestKeySoKeysAreNeverReusedAfterAnIndexLoss checks + * the same for compaction. + */ + public function theSwitchToShardsKeepsATombstoneForTheHighestKey(): void + { + $db = $this->db(['shardSize' => 4, 'shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $db->insert('mt', array_map(function ($n) { return ['n' => $n]; }, range(0, 10))); + $db->delete('mt', ['key' => [9, 10]]); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('mt')); + $this->deleteCheckpoints(); + file_put_contents($this->dbPath('mt', 'shards'), ''); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 4]); + $this->assertSame(range(0, 8), array_column($db->find('mt', 0), 'key')); + $db->insert('mt', ['n' => 'new']); + $this->assertSame(11, $db->find('mt', ['n' => 'new'])[0]['key'], 'keys 9 and 10 were used before'); + } + + /** + * @test The same rule when the key index is the body of its checkpoint (STORAGE_FORMAT.md §6.4: a process that + * can't afford the arrays keeps the body) and the highest keys were deleted after it, in a dense and in a sparse + * body: the highest live key is found below the deleted ones, and the switch keeps the highest key used. + */ + public function theSwitchToShardsKeepsTheHighestKeyWhenTheIndexIsABody(): void + { + // [records, keys deleted before the checkpoint]: 100 of 100 slots live (dense); 200 of 3000 (sparse) + foreach (['dense' => [100, []], 'sparse' => [3000, range(0, 2799)]] as $label => [$n, $early]) { + $this->cleanTestDirectory(); + $config = ['shardSize' => 1000, 'shardingEnabled' => false, 'autoCompactThreshold' => 1000]; + $db = $this->db($config); + $db->insert('mb', array_map(function ($i) { return ['i' => $i]; }, range(0, $n - 1))); + if ($early !== []) { + $this->assertSame(count($early), $db->delete('mb', ['key' => $early])['n']); + } + $this->assertSame($label, $this->readKeyIndexFile($this->dbPath('mb', 'kix'))['header']['layout'], "precondition: a $label checkpoint"); + \noneDB::clearStaticCache(); + $db = $this->db($config + ['cacheMemoryLimit' => 1]); // the body is never decoded + $this->assertSame(3, $db->delete('mb', ['key' => [$n - 3, $n - 2, $n - 1]])['n'], $label); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('mb'), $label); + $live = array_values(array_diff(range(0, $n - 4), $early)); + $this->deleteCheckpoints(); + file_put_contents($this->dbPath('mb', 'shards'), ''); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame($live, array_column($db->find('mb', 0), 'key'), $label); + $db->insert('mb', ['i' => 'new']); + $this->assertSame($n, $db->find('mb', ['i' => 'new'])[0]['key'], "$label: keys up to " . ($n - 1) . ' were used before'); + } + } + + /** + * @test STORAGE_FORMAT.md §5.2: the switch writes every shard with its key-index checkpoint (§6.4), covering the + * whole file: a database without field or spatial indexes (nothing loads its shards after the switch) has them too + */ + public function aSwitchWritesEveryShardWithItsCheckpoint(): void + { + $db = $this->db(['shardSize' => 10, 'shardingEnabled' => false]); + $db->insert('sc', array_map(function ($i) { return ['i' => $i]; }, range(0, 34))); + $db->delete('sc', ['key' => [3, 12, 34]]); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('sc')); + foreach (range(0, 3) as $shard) { + $data = $this->dbPath('sc', 'data', $shard); + $kix = $this->readKeyIndexFile($this->dbPath('sc', 'kix', $shard)); + clearstatcache(); + $this->assertSame([filesize($data), 10 * $shard, $shard, 10, $this->keyIndexOfData($data)], + [$kix['header']['s'], $kix['header']['base'], $kix['header']['sid'], $kix['header']['z'], $kix['entries']], "shard $shard"); + } + } + + /** + * @test STORAGE_FORMAT.md §5.2 PREPARE: the new generation is verified before it is published. A shard file that + * doesn't hold every record of its range (here a disk that lost its last line) stops the switch: the database + * stays in the single layout with all its records, the new generation is removed, and the switch works later. + */ + public function aSwitchWhoseShardLacksARecordIsNotPublished(): void + { + $config = ['shardSize' => 10, 'shardingEnabled' => false]; + $db = $this->db($config); + $db->insert('sv', array_map(function ($i) { return ['i' => $i]; }, range(0, 29))); + $home = $this->dbPath('sv', 'home'); + $this->setFaultHook(function ($point, $context) use ($home) { + if ($point === 'switch:shard-written' && $context['shard'] === 1) { + [$file] = glob($home . 'generations/*/segments/s1.jsonl'); + $lines = file($file); + array_pop($lines); + file_put_contents($file, implode('', $lines)); + } + }); + try { + $db->convertToSharded('sv'); + $this->fail('the switch was published'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('could not be verified', $e->getMessage()); + } finally { + $this->setFaultHook(null); + } + $this->assertFalse($db->getShardInfo('sv')['sharded']); + $this->assertSame(range(0, 29), array_column($db->find('sv', 0), 'key')); + $this->assertSame([basename($this->dbPath('sv', 'gen'))], array_map('basename', glob($home . 'generations/*')), 'only the active generation'); + $this->assertTrue($db->checkIntegrity('sv')['ok']); + $this->assertSame(['success' => true, 'status' => 'converted'], $db->convertToSharded('sv')); + $this->assertSame(range(0, 29), array_column($this->db($config)->find('sv', 0), 'key')); + } + + /** @test docs/CONFIGURATION.md shardingEnabled: "once it has assigned shardSize keys" (assigned, not live) */ + public function theSwitchToShardsStartsOnceShardSizeKeysWereAssigned(): void + { + $db = $this->db(['shardSize' => 5]); + $db->insert('am', array_map(function ($n) { return ['n' => $n]; }, range(0, 3))); + $db->delete('am', ['key' => [0, 1]]); + $this->assertFalse($db->getShardInfo('am')['sharded'], '4 keys assigned'); + $db->insert('am', ['n' => 4]); + $this->assertSame(['sharded' => true, 'shards' => 1, 'totalRecords' => 3, 'deletedCount' => 0, 'shardSize' => 5, 'nextKey' => 5], + $db->getShardInfo('am'), '5 keys assigned, 3 live'); + $this->assertSame([2, 3, 4], array_column($db->find('am', 0), 'key')); + } + + /** @test the same operations give the same records on a sharded and a non-sharded database */ + public function shardedAndNonShardedDatabasesGiveTheSameResults(): void + { + $plain = $this->db(['shardingEnabled' => false]); + $sharded = $this->db(['shardSize' => 3]); + $model = []; + $next = 0; + // Shards of 3: [0,3) [3,6) [6,9) [9,12) [12,15); step 5 empties shard 2, which compact() removes + $script = [ + ['insert', array_map(function ($n) { return ['n' => $n, 'g' => $n % 4]; }, range(0, 9))], + ['update', ['g' => 1], ['u' => 'one']], + ['delete', ['key' => [0, 6, 8]]], + ['insert', [['n' => 10, 'g' => 2], ['n' => 11, 'g' => 1]]], + ['update', ['key' => [2, 3, 7, 11, 99]], ['g' => 9]], + ['delete', ['g' => 9]], + ['compact'], + ['insert', [['n' => 12, 'g' => 0]]], + ['update', [], ['all' => true]], + ['delete', ['n' => 5]], + ]; + $matches = function (array $record, array $filter) { + foreach ($filter as $field => $value) { + if (!array_key_exists($field, $record) || $record[$field] !== $value) { + return false; + } + } + return true; + }; + foreach ($script as $step => $op) { + if ($op[0] === 'insert') { + foreach ([$plain, $sharded] as $db) { + $this->assertSame(['n' => count($op[1])], $db->insert($db === $plain ? 'eqp' : 'eqs', $op[1])); + } + foreach ($op[1] as $r) { + $model[$next++] = $r; + } + } elseif ($op[0] === 'update' || $op[0] === 'delete') { + $filter = $op[1]; + $keys = isset($filter['key']) ? array_values(array_intersect($filter['key'], array_keys($model))) : array_keys($model); + unset($filter['key']); + $hit = []; + foreach ($keys as $k) { + if ($matches($model[$k], $filter)) { + $hit[] = $k; + } + } + foreach ([$plain, $sharded] as $db) { + $name = $db === $plain ? 'eqp' : 'eqs'; + $r = $op[0] === 'update' ? $db->update($name, [$op[1], ['set' => $op[2]]]) : $db->delete($name, $op[1]); + $this->assertSame(['n' => count($hit)], $r, "step $step"); + } + foreach ($hit as $k) { + if ($op[0] === 'update') { + $model[$k] = array_merge($model[$k], $op[2]); + } else { + unset($model[$k]); + } + } + } else { + $this->assertTrue($plain->compact('eqp')['success']); + $result = $sharded->compact('eqs'); + $this->assertTrue($result['success']); + // shard 2 (keys 6..8) is empty and not the shard of the newest key (11: shard 3): removed + $this->assertSame(3, $result['newShardCount']); + } + ksort($model); + $expected = []; + foreach ($model as $k => $r) { + $expected[] = $r + ['key' => $k]; + } + $this->assertSame($expected, $plain->find('eqp', 0), "step $step: non-sharded"); + $this->assertSame($expected, $sharded->find('eqs', 0), "step $step: sharded"); + foreach ([['g' => 1], ['key' => [12, 1, 9, 40]], ['g' => 0, 'all' => true]] as $filter) { + $this->assertSame($plain->find('eqp', $filter), $sharded->find('eqs', $filter), "step $step: " . json_encode($filter)); + } + $this->assertSame(count($model), $plain->count('eqp')); + $this->assertSame(count($model), $sharded->count('eqs')); + } + $this->assertTrue($sharded->getShardInfo('eqs')['sharded']); + // live keys at the end: 1, 4, 9, 10, 12; find() by a key list returns them in the order of the list + $this->assertSame([12, 1, 9], array_column($plain->find('eqp', ['key' => [12, 1, 9, 40]]), 'key')); + $this->assertSame([12, 1, 9], array_column($sharded->find('eqs', ['key' => [12, 1, 9, 40]]), 'key')); + $this->assertSame([1, 4, 9, 10, 12], array_column($sharded->find('eqs', 0), 'key')); + } + + /** + * @test An unreadable shards.json is rebuilt from the shards. Without checkpoints and with a configured size that + * doesn't fit, the size is the database's: shard i holds keys in [i*size, (i+1)*size); keys 0..4, 5..9, + * 10..11 in shards 0, 1, 2 allow only size 5 (5/1 >= size, 9 < 2*size, 11 < 3*size), the size the database + * was created with (STORAGE_FORMAT.md 4.1) + */ + public function anUnreadableMetaIsRebuiltWithTheShardSizeTheKeysAllow(): void + { + $db = $this->db(['shardSize' => 5]); + $db->insert('rm', array_map(function ($n) { return ['n' => $n]; }, range(0, 11))); + $this->deleteCheckpoints(); + file_put_contents($this->dbPath('rm', 'shards'), ''); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 10000]); + $this->assertSame(range(0, 11), array_column($db->find('rm', 0), 'n')); + $this->assertSame(range(0, 11), array_column($db->find('rm', 0), 'key')); + $meta = json_decode(file_get_contents($this->dbPath('rm', 'shards')), true); + $this->assertSame(5, $meta['shardSize']); + $this->assertSame(12, $meta['totalRecords']); + $this->assertSame([5, 5, 2], array_column($meta['shards'], 'count')); + $this->assertSame(['meta-rebuilt'], array_column($db->getRecoveryLog('rm'), 'action')); + $db->insert('rm', ['n' => 12]); + $this->assertSame(12, $db->find('rm', ['n' => 12])[0]['key']); + } + + /** @test STORAGE_FORMAT.md 6.4: a checkpoint that doesn't fit is rebuilt from the data, together with the shard counters */ + public function aShardCheckpointThatDoesNotFitIsRebuiltAndTheCountersRecounted(): void + { + $db = $this->db(['shardSize' => 5]); + $db->insert('sc', array_map(function ($n) { return ['n' => $n]; }, range(0, 11))); + $ckpt = $this->dbPath('sc', 'kix', 1); + $file = $this->readKeyIndexFile($ckpt); + $checkpoint = $file['header']; + $checkpoint['h'] = $checkpoint['h'] === '00000000' ? '00000001' : '00000000'; + $this->writeKeyIndexFile($ckpt, $checkpoint, $file['entries'], $checkpoint['base'], $checkpoint['layout']); + $metaPath = $this->dbPath('sc', 'shards'); + $meta = json_decode(file_get_contents($metaPath), true); + $meta['totalRecords'] = 999; + $meta['shards'][1]['count'] = 999; + file_put_contents($metaPath, json_encode($meta)); + \noneDB::clearStaticCache(); + $db = $this->db(['shardSize' => 5]); + $this->assertSame(range(0, 11), array_column($db->find('sc', 0), 'n')); + $meta = json_decode(file_get_contents($metaPath), true); + $this->assertSame([5, 5, 2], array_column($meta['shards'], 'count')); + $this->assertSame(12, $meta['totalRecords']); + $this->assertSame(12, $db->getShardInfo('sc')['totalRecords']); + $this->assertNotSame($checkpoint['h'], $this->readKeyIndexFile($ckpt)['header']['h'], 'a checkpoint that fits replaced it'); + } + + // ------------------------------------------------------------------ + // Failures + // ------------------------------------------------------------------ + + /** + * @test DURABILITY.md 4: an append whose fflush() fails (disk full, I/O error) fails like a short write; + * an exception undoes the write: the tombstone is cut back, the record stays, counters unchanged + */ + public function aDeleteWhoseAppendFailsThrowsAndKeepsTheRecord(): void + { + foreach (['plain' => ['shardingEnabled' => false], 'sharded' => ['shardSize' => 3]] as $case => $config) { + $this->cleanTestDirectory(); + $db = $this->db($config); + $db->insert('fd', array_map(function ($n) { return ['n' => $n]; }, range(0, 6))); + $db->find('fd', 0); + $before = $this->snapshot('fd'); + unset($before['LOCK']); + $info = $db->getShardInfo('fd'); + $this->setStatic('flushFn', function () { return false; }); + try { + $db->delete('fd', ['n' => 4]); + $this->fail("$case: the delete must fail"); + } catch (\RuntimeException $e) { + $this->assertNotSame(\noneDB::OUTCOME_UNKNOWN, $e->getCode(), $case); + } + $this->setStatic('flushFn', null); + $after = $this->snapshot('fd'); + unset($after['LOCK']); + $this->assertSame($before, $after, "$case: no data, index or meta change"); + $this->assertSame(range(0, 6), array_column($db->find('fd', 0), 'n'), $case); + $this->assertSame($info, $db->getShardInfo('fd'), $case); + $this->assertSame(['n' => 1], $db->delete('fd', ['n' => 4]), $case); + $this->assertSame(6, $db->count('fd'), $case); + } + } + + /** + * @test docs/CONFIGURATION.md: replacing a data file (compaction) is synced; + * if that fails, compact() fails (it is not undone: "run them again") and the data is unchanged + */ + public function aCompactionThatCannotSyncItsNewFileFailsAndLosesNothing(): void + { + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $db->insert('cs', array_map(function ($n) { return ['n' => $n]; }, range(0, 9))); + $db->delete('cs', ['key' => [1, 2, 3]]); + $bytes = file_get_contents($this->dataFile('cs')); + $this->setStatic('syncFn', function ($handle, string $kind) { + return $kind !== 'file'; + }); + try { + $db->compact('cs'); + $this->fail('the compaction must fail'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('compaction', $e->getMessage()); + } + $this->setStatic('syncFn', null); + clearstatcache(); + $this->assertSame($bytes, file_get_contents($this->dataFile('cs')), 'the data file was not replaced'); + $this->assertSame([], $this->tempFiles(), 'its temp file was removed'); + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false, 'autoCompactThreshold' => 1]); + $this->assertSame([0, 4, 5, 6, 7, 8, 9], array_column($db->find('cs', 0), 'n')); + $this->assertSame(3, $db->compact('cs')['freedSlots'], 'run again: works'); + $this->assertSame(7, count($this->linesOf($this->dataFile('cs'))[0])); + } + + /** + * @test docs/CONFIGURATION.md troubleshooting: a data file changed behind a cached index (a backup restore): + * the write is refused with "changed unexpectedly" and writes nothing, the next operation rebuilds the index + * from the data and works; a process that only read it so far rebuilds at once + */ + public function aDataFileThatShrankBehindACachedIndexIsDetected(): void + { + foreach (['after a write of this process' => true, 'after a read only' => false] as $case => $wrote) { + $this->cleanTestDirectory(); + $db = $this->db(['shardingEnabled' => false]); + $db->insert('sh', [['v' => 'a'], ['v' => 'b'], ['v' => 'c']]); + if (!$wrote) { + \noneDB::clearStaticCache(); + $db = $this->db(['shardingEnabled' => false]); + $this->assertCount(3, $db->find('sh', 0)); + } + [$lines] = $this->linesOf($this->dataFile('sh')); + file_put_contents($this->dataFile('sh'), $lines[0] . "\n" . $lines[1] . "\n"); // an older copy restored + $restored = file_get_contents($this->dataFile('sh')); + if ($wrote) { + try { + $db->insert('sh', ['v' => 'd']); + $this->fail("$case: the write must be refused"); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('changed unexpectedly', $e->getMessage(), $case); + } + clearstatcache(); + $this->assertSame($restored, file_get_contents($this->dataFile('sh')), "$case: nothing written"); + } + $this->assertSame(['n' => 1], $db->insert('sh', ['v' => 'd']), $case); + $all = $db->find('sh', 0); + $this->assertSame(['a', 'b', 'd'], array_column($all, 'v'), $case); + $this->assertSame(count($all), count(array_unique(array_column($all, 'key'))), "$case: keys unique"); + \noneDB::clearStaticCache(); + $this->assertSame($all, $this->db(['shardingEnabled' => false])->find('sh', 0), "$case: the same for a new process"); + } + } +} diff --git a/tests/Regression/StorageRegressionTest.php b/tests/Integration/StorageRegressionTest.php similarity index 67% rename from tests/Regression/StorageRegressionTest.php rename to tests/Integration/StorageRegressionTest.php index bb1ee94..cd996d4 100644 --- a/tests/Regression/StorageRegressionTest.php +++ b/tests/Integration/StorageRegressionTest.php @@ -1,6 +1,8 @@ db()->insert('p', ['data' => [['title' => 'A'], ['title' => 'B']], 'owner' => 'orhan']); + $this->db()->insert('p', ['data' => [['title' => 'A'], ['title' => 'B']], 'owner' => 'alice']); $this->db()->insert('p', ['data' => 'v1', 'x' => 1]); $out = $this->runChild('echo json_encode((' . $this->childDb() . ')->find("p", 0));'); $this->assertSame( - '[{"data":[{"title":"A"},{"title":"B"}],"owner":"orhan","key":0},{"data":"v1","x":1,"key":1}]', + '[{"data":[{"title":"A"},{"title":"B"}],"owner":"alice","key":0},{"data":"v1","x":1,"key":1}]', $out ); } - /** @test K-02 */ - public function preJsonlFormatIsStillMigrated(): void - { - $base = $this->dbBasePath('legacy'); - file_put_contents($base . '.nonedb', json_encode(['data' => [['n' => 'a'], null, ['n' => 'c']]])); - file_put_contents($base . '.nonedbinfo', (string)time()); - $all = $this->db()->find('legacy', 0); - $this->assertSame([['n' => 'a', 'key' => 0], ['n' => 'c', 'key' => 2]], $all); - $this->db()->insert('legacy', ['n' => 'd']); - $this->assertSame(3, $this->db()->find('legacy', ['n' => 'd'])[0]['key'], 'next key continues after the old slots'); - } - /** @test K-03 */ public function unencodableDataIsRejectedWithoutTouchingFiles(): void { $db = $this->db(); $db->insert('c', ['city' => 'Istanbul']); $db->createFieldIndex('c', 'city'); - $files = glob($this->dbBasePath('c') . '*'); - $before = array_map('md5_file', $files); + $home = $this->dbPath('c', 'home'); + $files = $this->dbFiles('c'); + $md5 = function (array $files) use ($home) { + return array_map(function ($f) use ($home) { return md5_file($home . $f); }, $files); + }; + $before = $md5($files); $r1 = $db->insert('c', ['city' => "Bad\xB1Latin1"]); $r2 = $db->insert('c', ['v' => NAN]); $r3 = $db->update('c', [['city' => 'Istanbul'], ['set' => ['city' => "x\xB1"]]]); - foreach ([$r1, $r2, $r3] as $r) { - $this->assertSame(0, $r['n']); - $this->assertArrayHasKey('error', $r); + // docs/API.md: "insert data cannot be encoded as JSON ..." / "update data cannot be encoded as JSON ..." + foreach (['insert' => $r1, 'insert NAN' => $r2, 'update' => $r3] as $what => $r) { + $this->assertSame(['n', 'error'], array_keys($r), $what); + $this->assertSame(0, $r['n'], $what); + $this->assertStringStartsWith(strtok($what, ' ') . ' data cannot be encoded as JSON', $r['error'], $what); } clearstatcache(); - $this->assertSame($before, array_map('md5_file', $files), 'no file changed'); + $this->assertSame($before, $md5($files), 'no file changed'); + $this->assertSame($files, $this->dbFiles('c'), 'no file created'); \noneDB::clearStaticCache(); $this->assertSame([['city' => 'Istanbul', 'key' => 0]], $this->db()->find('c', ['city' => 'Istanbul'])); @@ -65,12 +62,13 @@ public function missingOrCorruptIndexIsRebuiltFromData(): void $db->update('r', [['n' => 3], ['set' => ['n' => 33]]]); $expected = $this->plain($db->find('r', 0)); + $original = (string)file_get_contents($this->dbPath('r', 'kix')); foreach (['missing', 'corrupt', 'truncated'] as $mode) { - $idx = $this->dbBasePath('r') . '.nonedb.jidx'; + $idx = $this->dbPath('r', 'kix'); if ($mode === 'missing') { unlink($idx); } elseif ($mode === 'corrupt') { - file_put_contents($idx, '{"v":3,"o":{"0":[0,'); + file_put_contents($idx, substr($original, 0, 520)); // its header, then a cut body } else { file_put_contents($idx, ''); } @@ -87,13 +85,20 @@ public function tornLastLineIsCutOff(): void { $db = $this->db(); $db->insert('t', [['n' => 1], ['n' => 2]]); - $data = $this->dbBasePath('t') . '.nonedb'; + $data = $this->dbPath('t', 'data'); + clearstatcache(); + $size = filesize($data); file_put_contents($data, '{"n":3,"ke', FILE_APPEND); // crash in the middle of an append - unlink($data . '.jidx'); + unlink($this->dbPath('t', 'kix')); \noneDB::clearStaticCache(); $this->assertSame([['n' => 1], ['n' => 2]], $this->plain($this->db()->find('t', 0))); $this->db()->insert('t', ['n' => 3]); $this->assertSame([['n' => 1], ['n' => 2], ['n' => 3]], $this->plain($this->db()->find('t', 0))); + $this->assertSame([0, 1, 2], array_column($this->db()->find('t', 0), 'key'), 'the torn record never got a key'); + // The cut bytes are kept in recovery/q.main.{offset}.{length}.{sha256}.bin (STORAGE_FORMAT.md 7.2) + $quarantine = $this->dbPath('t', 'quarantine') . 'q.main.' . $size . '.10.' . hash('sha256', '{"n":3,"ke') . '.bin'; + $this->assertFileExists($quarantine); + $this->assertSame('{"n":3,"ke', file_get_contents($quarantine)); } /** @test Y-14 O-21 */ @@ -106,13 +111,15 @@ public function crashBetweenCompactionRenamesIsDetected(): void } $db->insert('k', $rows); $db->delete('k', ['n' => 3]); - $oldIndex = file_get_contents($this->dbBasePath('k') . '.nonedb.jidx'); + $oldIndex = file_get_contents($this->dbPath('k', 'kix')); $db->compact('k'); - // Simulate: new data file renamed into place, old index still on disk - file_put_contents($this->dbBasePath('k') . '.nonedb.jidx', $oldIndex); + // Simulate: new data file renamed into place, old checkpoint still on disk + file_put_contents($this->dbPath('k', 'kix'), $oldIndex); \noneDB::clearStaticCache(); $all = $this->db()->find('k', 0); - $this->assertCount(19, $all); + $expected = array_values(array_diff(range(0, 19), [3])); + $this->assertSame($expected, array_column($all, 'n')); + $this->assertSame($expected, array_column($all, 'key'), 'keys unchanged'); $this->assertSame(19, $this->db()->find('k', ['n' => 19])[0]['n']); $this->assertSame([], $this->db()->find('k', ['n' => 3])); } @@ -154,7 +161,9 @@ public function shardedCompactFlushesBuffersFirst(): void $db->insert('sb', ['i' => 13]); $db->compact('sb'); $this->assertSame(14, $db->count('sb')); - $this->assertSame([], glob($this->dbBasePath('sb') . '*.buffer')); + $this->assertSame([], glob($this->dbPath('sb', 'gen') . 'buffer/*')); + $this->assertSame(range(0, 13), array_column($db->find('sb', 0), 'i')); + $this->assertSame(range(0, 13), array_column($db->find('sb', 0), 'key'), 'buffered records got the next keys, in order'); } /** @test O-15 a buffered database switches to shards when its flush crosses shardSize */ @@ -164,31 +173,31 @@ public function bufferedDatabaseShardsWhenFlushed(): void for ($i = 0; $i < 25; $i++) { $db->insert('bs', ['i' => $i]); } - $this->assertFileDoesNotExist($this->dbBasePath('bs') . '.nonedb.meta', 'still buffered'); + $this->assertFileDoesNotExist($this->dbPath('bs', 'shards'), 'still buffered'); $this->assertSame(range(0, 24), array_column($db->find('bs', 0), 'i'), 'the read flushes'); - $this->assertFileExists($this->dbBasePath('bs') . '.nonedb.meta', 'and the flush migrated to shards'); + $this->assertFileExists($this->dbPath('bs', 'shards'), 'and the flush switched it to shards'); $this->assertSame(range(0, 24), array_column($db->find('bs', 0), 'key')); $this->assertSame(25, $db->count('bs')); $this->assertSame(3, $db->getShardInfo('bs')['shards']); } - /** @test O-16 manual migrate() takes buffered records along */ - public function manualMigrateFlushesTheBufferFirst(): void + /** @test O-16 convertToSharded() takes buffered records along */ + public function convertToShardedFlushesTheBufferFirst(): void { $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false]); $db->insert('mb', ['i' => 0]); $db->flush('mb'); $db->insert('mb', [['i' => 1], ['i' => 2]]); - $this->assertSame('migrated', $db->migrate('mb')['status']); - $this->assertSame([], glob($this->dbBasePath('mb') . '*.buffer*')); + $this->assertSame('converted', $db->convertToSharded('mb')['status']); + $this->assertSame([], glob($this->dbPath('mb', 'gen') . 'buffer/*')); $this->assertSame([0, 1, 2], array_column($this->db()->find('mb', 0), 'i')); $this->assertSame(3, $this->db()->count('mb')); } /** @test K-06 D-10 */ - public function migrationNeverReusesKeys(): void + public function theSwitchToShardsNeverReusesKeys(): void { - $db = $this->db(['shardSize' => 10, 'autoMigrate' => false]); + $db = $this->db(['shardSize' => 10, 'shardingEnabled' => false]); $rows = []; for ($i = 0; $i < 15; $i++) { $rows[] = ['i' => $i]; @@ -197,9 +206,10 @@ public function migrationNeverReusesKeys(): void $db->delete('mg', ['i' => 3]); $db->delete('mg', ['i' => 13]); $db->delete('mg', ['i' => 14]); // trailing keys deleted too - $this->assertSame('migrated', $db->migrate('mg')['status']); - $this->assertFalse(file_exists($this->dbBasePath('mg') . '.nonedb'), 'non-sharded file removed'); - $this->assertSame([], glob($this->dbBasePath('mg') . '.nonedb.backup')); + $this->assertSame('converted', $db->convertToSharded('mg')['status']); + // The single layout's generation is removed by the switch (STORAGE_FORMAT.md 5.2 CLEANUP), no copy stays + $this->assertFalse(file_exists($this->dbPath('mg', 'data')), 'non-sharded file removed'); + $this->assertSame([rtrim($this->dbPath('mg', 'gen'), '/')], glob($this->dbPath('mg', 'home') . 'generations/*'), 'only the active generation is left'); $db->insert('mg', ['i' => 'NEW']); $new = $db->find('mg', ['i' => 'NEW'])[0]; @@ -249,7 +259,9 @@ public function keyFiltersAreNotCoerced(): void { $db = $this->db(); $db->insert('kc', [['a' => 'zero'], ['a' => 'one']]); - foreach ([null, 'abc', true, 1.9, -1, '1.0', [], ['x']] as $bad) { + // docs/API.md: an integer, a digit string or a list of them; anything else matches nothing. + // "5\n" is not key 5. + foreach ([null, 'abc', true, 1.9, 1.0, -1, '1.0', "1\n", ' 1', [], ['x']] as $bad) { $this->assertSame([], $db->find('kc', ['key' => $bad]), 'key ' . var_export($bad, true)); } $this->assertSame('one', $db->find('kc', ['key' => '1'])[0]['a'], 'digit strings are keys'); @@ -270,38 +282,13 @@ public function compactionTriggersOnGarbageRatio(): void $db->delete('g', ['i' => 0]); $db->delete('g', ['i' => 1]); $db->delete('g', ['i' => 2]); - $index = json_decode(file_get_contents($this->dbBasePath('g') . '.nonedb.jidx'), true); - $this->assertSame(3, $index['d'], '3 of 10 deleted: not above 30% yet'); + // The index in memory (the .kix file is a checkpoint, written when the tail is long) + $read = function () { + return $this->readJsonlIndex('g', null); + }; + $this->assertSame(3, $read->call(\noneDB\Tests\Support\Testing::engine($db))['d'], '3 of 10 deleted: not above 30% yet'); $db->delete('g', ['i' => 3]); - $index = json_decode(file_get_contents($this->dbBasePath('g') . '.nonedb.jidx'), true); - $this->assertSame(0, $index['d'], '4 of 10: compacted'); + $this->assertSame(0, $read->call(\noneDB\Tests\Support\Testing::engine($db))['d'], '4 of 10: compacted'); $this->assertSame(6, $db->count('g')); } - - /** @test O-23 */ - public function legacyNullSlotsAreNotCountedAsGarbage(): void - { - $base = $this->dbBasePath('lg'); - file_put_contents($base . '.nonedb', json_encode(['data' => [null, null, ['n' => 'a'], 'not-a-record']])); - file_put_contents($base . '.nonedbinfo', (string)time()); - $this->assertSame([['n' => 'a', 'key' => 2]], $this->db()->find('lg', 0)); - $index = json_decode(file_get_contents($base . '.nonedb.jidx'), true); - $this->assertSame(0, $index['d']); - } - - /** @test K-03 */ - public function zeroLengthIndexEntryDoesNotBreakTheDatabase(): void - { - $db = $this->db(); - $db->insert('z', [['n' => 1], ['n' => 2]]); - $idxPath = $this->dbBasePath('z') . '.nonedb.jidx'; - $index = json_decode(file_get_contents($idxPath), true); - $index['o'][1] = [$index['o'][1][0], 0]; // what pre-3.2 wrote after a failed json_encode - file_put_contents($idxPath, json_encode($index)); - \noneDB::clearStaticCache(); - $db = $this->db(); - $this->assertSame([['n' => 1], ['n' => 2]], $this->plain($db->find('z', 0))); - $this->assertSame(1, $db->delete('z', ['key' => 1])['n']); - $this->assertTrue($db->compact('z')['success']); - } } diff --git a/tests/Integration/StoreFormatTest.php b/tests/Integration/StoreFormatTest.php new file mode 100644 index 0000000..97d68aa --- /dev/null +++ b/tests/Integration/StoreFormatTest.php @@ -0,0 +1,359 @@ + sha256 of a file, "dir" for a directory */ + private function snapshot(string $dir): array + { + $out = []; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $full = $path . '/' . $entry; + if (is_dir($full)) { + $out[$rel . $entry] = 'dir'; + $walk($full, $rel . $entry . '/'); + } else { + $out[$rel . $entry] = hash_file('sha256', $full); + } + } + }; + if (is_dir($dir)) { + $walk(rtrim($dir, '/'), ''); + } + ksort($out); + return $out; + } + + private function storeRecord(string $file): ?array + { + $raw = @file_get_contents($this->testDbDir . $file); + if ($raw === false || substr($raw, -1) !== "\n" || !preg_match('/,"checksum":"([0-9a-f]{64})"\}\n\z/', $raw, $m, PREG_OFFSET_CAPTURE)) { + return null; + } + // §2: the checksum covers the record as written without its last member + if (hash('sha256', substr($raw, 0, $m[0][1]) . "}\n") !== $m[1][0]) { + return null; + } + return json_decode($raw, true); + } + + private function keyCheck(string $key): string + { + return substr(hash_hmac('sha256', 'nonedb4/key', $key), 0, 32); + } + + private function code(callable $fn): ?int + { + try { + $fn(); + } catch (\RuntimeException $e) { + return $e->getCode(); + } + return null; + } + + /** @test */ + public function aReadOfAnEmptyDirectoryCreatesNothing(): void + { + $before = $this->snapshot($this->testDbDir); + $db = $this->db(); + $this->assertSame([], $db->find('nothing', 0)); + $this->assertSame(0, $db->count('nothing')); + $this->assertSame([], $db->getDBs()); + $this->assertSame($before, $this->snapshot($this->testDbDir), 'reads create no file (§3.2 step 3 only for writes)'); + } + + /** @test */ + public function theFirstWriteCreatesTheStoreWithTwoIdenticalCopies(): void + { + $db = $this->db(); + $db->insert('first', ['a' => 1]); + $one = $this->storeRecord('STORE'); + $two = $this->storeRecord('STORE.2'); + $this->assertNotNull($one, 'STORE is a valid record (§2, §3.1)'); + $this->assertSame($one, $two, 'both copies identical (§3.1 invariant)'); + $this->assertSame('NONEDB4', $one['magic']); + $this->assertSame('store', $one['kind']); + $this->assertSame(4, $one['storageFormat']); + $this->assertSame(1, $one['seq']); + $this->assertSame([$this->keyCheck(self::SECRET)], $one['keys']); + $this->assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', $one['storeId']); + $this->assertFileExists($this->testDbDir . 'STORE.LOCK'); + $this->assertFileExists($this->testDbDir . '.htaccess'); + $this->assertFileExists($this->testDbDir . 'index.html'); + // The database directory is named by the locator (§1) + $locator = substr(hash_hmac('sha256', 'nonedb4/db/first', self::SECRET), 0, 32); + $this->assertDirectoryExists($this->testDbDir . $locator . '.ndb4'); + } + + /** @test */ + public function anEmptyDirectoryWithAllowedFilesBecomesAStore(): void + { + foreach (['.gitkeep', '.gitignore', '.DS_Store', '.htaccess', 'index.html', 'STORE.LOCK', 'STORE.tmp0123abcd'] as $allowed) { + file_put_contents($this->testDbDir . $allowed, ''); + } + $this->db()->insert('ok', ['a' => 1]); + $this->assertNotNull($this->storeRecord('STORE'), '§3.2: these entries keep a directory empty'); + $this->assertFileDoesNotExist($this->testDbDir . 'STORE.tmp0123abcd', 'step 3.5: stale temp files removed'); + $this->assertSame([['a' => 1, 'key' => 0]], $this->db()->find('ok', 0)); + } + + /** @test */ + public function aForeignDirectoryIsRefusedAndNothingIsCreated(): void + { + foreach (['hash-users.nonedb' => '{"a":1,"key":0}' . "\n", 'notes.txt' => 'x', 'sub/file' => 'y'] as $name => $content) { + @mkdir(dirname($this->testDbDir . $name), 0777, true); + file_put_contents($this->testDbDir . $name, $content); + } + $before = $this->snapshot($this->testDbDir); + $db = $this->db(); + foreach ([ + 'insert' => function () use ($db) { $db->insert('users', ['a' => 1]); }, + 'find' => function () use ($db) { $db->find('users', 0); }, + 'count' => function () use ($db) { $db->count('users'); }, + 'createDB' => function () use ($db) { $db->createDB('users'); }, + 'getDBs' => function () use ($db) { $db->getDBs(); }, + 'dropDB' => function () use ($db) { $db->dropDB('users'); }, + 'checkIntegrity' => function () use ($db) { $db->checkIntegrity('users'); }, + ] as $op => $fn) { + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code($fn), "$op: §3.2 step 2"); + $this->assertSame($before, $this->snapshot($this->testDbDir), "$op: no file or directory created (§3.2)"); + } + } + + /** @test */ + public function databaseDirectoriesWithoutStoreRecordAreADamagedStore(): void + { + $this->db()->insert('d', ['a' => 1]); + unlink($this->testDbDir . 'STORE'); + unlink($this->testDbDir . 'STORE.2'); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db()->find('d', 0); }), '§3.2: missing store record'); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + + /** @test */ + public function anUnregisteredKeyNeverSeesAnEmptyDatabase(): void + { + $this->db()->insert('mine', ['a' => 1]); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $other = new \noneDB(['secretKey' => self::OTHER, 'dbDir' => $this->testDbDir]); + foreach ([ + function () use ($other) { $other->find('mine', 0); }, + function () use ($other) { $other->insert('mine', ['b' => 2]); }, + function () use ($other) { $other->getDBs(); }, + ] as $i => $fn) { + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $this->code($fn), "call $i: §3.2 step 1"); + } + $this->assertSame($before, $this->snapshot($this->testDbDir), 'the wrong key created nothing'); + } + + /** @test */ + public function oneInvalidCopyIsRepairedFromTheOther(): void + { + $this->db()->insert('r', ['a' => 1]); + $expected = $this->storeRecord('STORE'); + foreach (['STORE', 'STORE.2'] as $damaged) { + foreach (['missing', 'garbage', 'checksum'] as $how) { + \noneDB::clearStaticCache(); + if ($how === 'missing') { + unlink($this->testDbDir . $damaged); + } elseif ($how === 'garbage') { + file_put_contents($this->testDbDir . $damaged, "\0\0\0"); + } else { + $raw = file_get_contents($this->testDbDir . $damaged); + file_put_contents($this->testDbDir . $damaged, str_replace('"seq":1', '"seq":2', $raw)); + } + $this->assertSame([['a' => 1, 'key' => 0]], $this->db()->find('r', 0), "$damaged $how: readable"); + $this->assertSame($expected, $this->storeRecord('STORE'), "$damaged $how: STORE"); + $this->assertSame($expected, $this->storeRecord('STORE.2'), "$damaged $how: STORE.2 (§3.1 repair)"); + } + } + $log = file_get_contents($this->testDbDir . '.nonedb-maintenance/log.jsonl'); + $this->assertSame(6, substr_count($log, '"store-copy-repaired"'), 'every repair is logged'); + } + + /** @test */ + public function aCopyDamagedWhileTheStoreIsOpenNeverBreaksTheMissingDatabasePath(): void + { + $db = $this->db(); + $db->insert('r', ['a' => 1]); + $expected = $this->storeRecord('STORE'); + foreach (['STORE', 'STORE.2'] as $damaged) { + file_put_contents($this->testDbDir . $damaged, 'garbage'); + // The store is open in this process: the missing-database path checks it again under STORE.LOCK + // shared, where the copy cannot be repaired; the answer still comes from the valid copy + $this->assertSame([], $db->find('missing', 0), "$damaged: missing database read"); + $db->insert('created' . $damaged[strlen($damaged) - 1], ['b' => 2]); + $this->assertSame([['b' => 2, 'key' => 0]], $db->find('created' . $damaged[strlen($damaged) - 1], 0), "$damaged: creation"); + \noneDB::clearStaticCache(); + $this->assertSame([['a' => 1, 'key' => 0]], $this->db()->find('r', 0), "$damaged: the next open"); + $this->assertSame($expected, $this->storeRecord('STORE'), "$damaged: repaired by the next open"); + $this->assertSame($expected, $this->storeRecord('STORE.2'), $damaged); + $db = $this->db(); + } + } + + /** @test */ + public function twoValidCopiesThatDifferAreDamaged(): void + { + $this->db()->insert('x', ['a' => 1]); + $record = $this->storeRecord('STORE'); + unset($record['checksum']); + $record['seq'] = 2; + $json = json_encode($record, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION); + $record['checksum'] = hash('sha256', $json . "\n"); + file_put_contents($this->testDbDir . 'STORE.2', json_encode($record, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION) . "\n"); + $this->assertNotNull($this->storeRecord('STORE.2'), 'a valid record'); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db()->find('x', 0); }), '§3.1: nothing is guessed'); + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + + /** @test */ + public function bothCopiesInvalidIsDamaged(): void + { + $this->db()->insert('y', ['a' => 1]); + file_put_contents($this->testDbDir . 'STORE', 'x'); + file_put_contents($this->testDbDir . 'STORE.2', ''); + \noneDB::clearStaticCache(); + $this->assertSame(\noneDB::FORMAT_DAMAGED, $this->code(function () { $this->db()->find('y', 0); })); + } + + /** @test */ + public function foreignEntriesStopMissingDatabasesButNotExistingOnes(): void + { + $db = $this->db(); + $db->insert('exists', ['a' => 1]); + file_put_contents($this->testDbDir . 'abcdef-old.nonedb', '{"a":1,"key":0}' . "\n"); + \noneDB::clearStaticCache(); + $db = $this->db(); + // §3.4: an existing database is found without a listing and works + $this->assertSame([['a' => 1, 'key' => 0]], $db->find('exists', 0)); + $db->insert('exists', ['a' => 2]); + $this->assertSame(2, $db->count('exists')); + // ... a missing one is neither "empty" nor created + $before = $this->snapshot($this->testDbDir); + foreach ([ + function () use ($db) { return $db->find('old', 0); }, + function () use ($db) { return $db->count('old'); }, + function () use ($db) { return $db->insert('old', ['b' => 1]); }, + function () use ($db) { return $db->createDB('old'); }, + ] as $i => $fn) { + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code($fn), "call $i"); + } + $this->assertSame($before, $this->snapshot($this->testDbDir)); + } + + /** @test */ + public function aForeignFileAddedInTheSameSecondIsSeen(): void + { + // §3.4: listed directly every time, no cache keyed on the directory's mtime (one-second resolution) + $db = $this->db(); + $db->insert('base', ['a' => 1]); + $this->assertSame([], $db->find('missing', 0), 'no foreign entry yet: missing means empty'); + $mtime = filemtime($this->testDbDir); + file_put_contents($this->testDbDir . 'legacy-file.nonedb', 'x'); + touch($this->testDbDir, $mtime); // same mtime as before: a cache keyed on it would not notice + clearstatcache(); + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code(function () use ($db) { $db->find('missing', 0); })); + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $this->code(function () use ($db) { $db->insert('missing', ['x' => 1]); })); + $this->assertDirectoryDoesNotExist($this->dbPath('missing', 'home')); + } + + /** @test */ + public function aForeignFileCreatedByAnotherProcessStopsEveryLaterMissingAnswer(): void + { + $db = $this->db(); + $db->insert('base', ['a' => 1]); + $flag = $this->testDbDir . '../store-foreign-created'; + @unlink($flag); + $proc = proc_open([PHP_BINARY, $this->childScript(' + usleep(200000); + file_put_contents(' . var_export($this->testDbDir . 'old-data.nonedb', true) . ', "x"); + file_put_contents(' . var_export($flag, true) . ', "1"); + ')], [], $pipes); + $created = false; + $after = 0; + $deadline = microtime(true) + 10; + while (microtime(true) < $deadline && $after < 20) { + $seenBefore = is_file($flag); + $code = $this->code(function () use ($db) { $db->find('missing', 0); }); + if ($seenBefore) { + $created = true; + $after++; + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $code, 'every read after the file exists refuses'); + } else { + $this->assertContains($code, [null, \noneDB::UNSUPPORTED_FORMAT]); + } + } + proc_close($proc); + @unlink($flag); + $this->assertTrue($created); + $this->assertDirectoryDoesNotExist($this->dbPath('missing', 'home'), 'never created'); + } + + /** @test */ + public function concurrentFirstOpensCreateOneStore(): void + { + $outputs = $this->runParallel(' + $db = ' . $this->childDb() . '; + $db->insert("p" . $argv[1], ["n" => (int)$argv[1]]); + echo "ok"; + ', 6); + $this->assertSame(array_fill(0, 6, 'ok'), array_values($outputs)); + $record = $this->storeRecord('STORE'); + $this->assertNotNull($record); + $this->assertSame($record, $this->storeRecord('STORE.2')); + $this->assertSame(1, $record['seq'], 'created once'); + \noneDB::clearStaticCache(); + $names = $this->db()->getDBs(); + $this->assertSame(['p1', 'p2', 'p3', 'p4', 'p5', 'p6'], $names); // runParallel numbers processes from 1 + } + + /** @test */ + public function aStoreCreationInterruptedAtEveryStepIsFinishedByTheNextOpen(): void + { + foreach (['store:create', 'store:write'] as $point) { + foreach ([1, 2] as $hit) { + $this->cleanTestDirectory(); + $out = $this->runChild(' + $db = ' . $this->childDb() . '; + $n = 0; + noneDB\Tests\Support\Testing::setFaultHook(function ($p) use (&$n) { + if ($p === ' . var_export($point, true) . ' && ++$n === ' . $hit . ') { exit(0); } + }); + $db->insert("s", ["a" => 1]); + echo "not reached"; + '); + if ($point === 'store:create' && $hit === 2) { + $this->assertSame('not reached', $out, 'store:create happens once'); + continue; + } + $this->assertSame('', $out, "$point #$hit"); + \noneDB::clearStaticCache(); + $db = $this->db(); + $db->insert('s', ['a' => 2]); + $this->assertSame([2], array_column($db->find('s', 0), 'a'), "$point #$hit: the interrupted insert never happened"); + $this->assertSame($this->storeRecord('STORE'), $this->storeRecord('STORE.2'), "$point #$hit: copies identical"); + $this->assertNotNull($this->storeRecord('STORE')); + } + } + } +} diff --git a/tests/Integration/StrictOptionsTest.php b/tests/Integration/StrictOptionsTest.php new file mode 100644 index 0000000..24a81fc --- /dev/null +++ b/tests/Integration/StrictOptionsTest.php @@ -0,0 +1,142 @@ +fail("$label: accepted"); + } catch (\InvalidArgumentException $e) { + $this->assertStringContainsString($message, $e->getMessage(), $label); + } + } + + /** Every file and directory under $dir with its SHA-256 */ + private function snapshot(string $dir): array + { + $out = []; + $walk = function (string $path, string $rel) use (&$walk, &$out): void { + foreach (array_diff(scandir($path) ?: [], ['.', '..']) as $entry) { + $full = $path . '/' . $entry; + $out[$rel . $entry] = is_dir($full) ? 'dir' : hash_file('sha256', $full); + if (is_dir($full)) { + $walk($full, $rel . $entry . '/'); + } + } + }; + $walk(rtrim($dir, '/'), ''); + ksort($out); + return $out; + } + + /** @test API.md "Spatial indexes": nodeSize; withinDistance() includeDistance; nearest() maxDistance, includeDistance */ + public function spatialOptionsAreTheDocumentedOnes(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('sp', [['loc' => ['type' => 'Point', 'coordinates' => [1, 1]]], ['loc' => ['type' => 'Point', 'coordinates' => [1.001, 1]]]]); + + $this->rejects(function () use ($db) { $db->createSpatialIndex('sp', 'loc', ['nodesize' => 16]); }, + "unknown option 'nodesize' of createSpatialIndex() (did you mean 'nodeSize'?); the options are: nodeSize", 'createSpatialIndex'); + foreach ([3, 0, -8, '16', 16.0, true, null] as $bad) { + $this->rejects(function () use ($db, $bad) { $db->createSpatialIndex('sp', 'loc', ['nodeSize' => $bad]); }, + "'nodeSize' must be an integer of at least 4", 'nodeSize ' . var_export($bad, true)); + } + $this->assertSame([], $db->getSpatialIndexes('sp'), 'no index made by a refused call'); + $this->assertTrue($db->createSpatialIndex('sp', 'loc', ['nodeSize' => 4])['success']); + + $this->rejects(function () use ($db) { $db->withinDistance('sp', 'loc', 1, 1, 1000, ['includeDistances' => true]); }, + "unknown option 'includeDistances' of withinDistance() (did you mean 'includeDistance'?); the options are: includeDistance", 'withinDistance'); + foreach ([1, 'true', null] as $bad) { + $this->rejects(function () use ($db, $bad) { $db->withinDistance('sp', 'loc', 1, 1, 1000, ['includeDistance' => $bad]); }, + "'includeDistance' must be true or false", 'withinDistance includeDistance ' . var_export($bad, true)); + } + // Haversine, earth radius 6371 km (SPATIAL.md): 0.001° of longitude at latitude 1° is 111.18 m + $this->assertSame([0.0, 111.18], array_column($db->withinDistance('sp', 'loc', 1, 1, 1000, ['includeDistance' => true]), '_distance')); + + $this->rejects(function () use ($db) { $db->nearest('sp', 'loc', 1, 1, 1, ['maxdistance' => 10]); }, + "unknown option 'maxdistance' of nearest() (did you mean 'maxDistance'?); the options are: maxDistance, includeDistance", 'nearest'); + $this->rejects(function () use ($db) { $db->nearest('sp', 'loc', 1, 1, 1, ['radius' => 10]); }, + "unknown option 'radius' of nearest(); the options are: maxDistance, includeDistance", 'nearest radius'); + foreach (['far', '10', true, []] as $bad) { + $this->rejects(function () use ($db, $bad) { $db->nearest('sp', 'loc', 1, 1, 1, ['maxDistance' => $bad]); }, + "'maxDistance' must be a number of meters or null", 'maxDistance ' . var_export($bad, true)); + } + $this->rejects(function () use ($db) { $db->nearest('sp', 'loc', 1, 1, 1, ['includeDistance' => 'no']); }, + "'includeDistance' must be true or false", 'nearest includeDistance'); + $near = $db->nearest('sp', 'loc', 1, 1, 5, ['maxDistance' => 50, 'includeDistance' => false]); + $this->assertSame([0], array_column($near, 'key'), 'within 50 m: the first point only'); + $this->assertArrayNotHasKey('_distance', $near[0], 'includeDistance false'); + $this->assertSame([0, 1], array_column($db->nearest('sp', 'loc', 1, 1, 5, ['maxDistance' => null]), 'key'), 'null: no maximum'); + } + + /** @test API.md "Backup and restore": restore() takes 'force', true or false */ + public function restoreTakesForce(): void + { + $db = $this->db(); + $db->insert('r', ['a' => 1]); + $dir = sys_get_temp_dir() . '/nonedb_options_backup_' . bin2hex(random_bytes(4)); + $db->backup('r', $dir); + $before = $this->snapshot($this->testDbDir); + try { + $this->rejects(function () use ($db, $dir) { $db->restore($dir, 'r', ['forse' => true]); }, + "unknown option 'forse' of restore() (did you mean 'force'?); the options are: force", 'restore'); + foreach (['yes', 1, null] as $bad) { + $this->rejects(function () use ($db, $dir, $bad) { $db->restore($dir, 'r', ['force' => $bad]); }, + "'force' must be true or false", 'force ' . var_export($bad, true)); + } + $this->assertSame($before, $this->snapshot($this->testDbDir), 'nothing changed'); + $this->assertSame('present', $db->restore($dir, 'r', ['force' => false])['target']); + } finally { + exec('rm -rf ' . escapeshellarg($dir)); + } + } + + /** + * @test API.md "Key maintenance": migrateKey() takes abort and durability, registerKey() durability, repairStore() + * manifest, confirmManualRebuild and durability; a refused call writes nothing + */ + public function keyMaintenanceOptionsAreTheDocumentedOnes(): void + { + $key = 'test_secret_key_for_unit_tests'; + $this->db()->insert('k', ['a' => 1]); + \noneDB::clearStaticCache(); + $before = $this->snapshot($this->testDbDir); + $dir = $this->testDbDir; + $cases = [ + 'migrateKey' => [function (array $o) use ($dir, $key) { return \noneDB::migrateKey($dir, $key, self::OTHER, $o); }, + ['abrot' => true], "unknown option 'abrot' of migrateKey() (did you mean 'abort'?); the options are: abort, durability", + [[['abort' => 'yes'], "'abort' must be true or false"], [['abort' => 1], "'abort' must be true or false"], + [['durability' => 'durabel'], "'durability' must be 'performance' or 'durable'"]]], + 'registerKey' => [function (array $o) use ($dir, $key) { return \noneDB::registerKey($dir, $key, self::OTHER, $o); }, + ['durabilty' => 'performance'], "unknown option 'durabilty' of registerKey() (did you mean 'durability'?); the options are: durability", + [[['durability' => null], "'durability' must be 'performance' or 'durable'"]]], + 'repairStore' => [function (array $o) use ($dir, $key) { return \noneDB::repairStore($dir, [$key], $o); }, + ['confirmManualRebuilt' => true], "unknown option 'confirmManualRebuilt' of repairStore() (did you mean 'confirmManualRebuild'?); " + . "the options are: manifest, confirmManualRebuild, durability", + [[['confirmManualRebuild' => 'yes'], "'confirmManualRebuild' must be true or false"], + [['manifest' => 5], "'manifest' must be a file path or a store record (array)"], + [['manifest' => null], "'manifest' must be a file path or a store record (array)"]]], + ]; + foreach ($cases as $method => [$call, $unknown, $message, $wrong]) { + $this->rejects(function () use ($call, $unknown) { $call($unknown); }, $message, $method); + foreach ($wrong as [$options, $wrongMessage]) { + $this->rejects(function () use ($call, $options) { $call($options); }, $wrongMessage, $method . ' ' . json_encode($options)); + } + $this->assertSame($before, $this->snapshot($this->testDbDir), "$method: nothing written"); + } + $this->assertTrue(\noneDB::registerKey($dir, $key, self::OTHER, ['durability' => 'performance'])['success'], 'the documented options work'); + } +} diff --git a/tests/Integration/TypeEdgeCasesTest.php b/tests/Integration/TypeEdgeCasesTest.php new file mode 100644 index 0000000..74a1f87 --- /dev/null +++ b/tests/Integration/TypeEdgeCasesTest.php @@ -0,0 +1,1455 @@ + 'Alice', 'age' => 25, 'score' => 85.5, 'active' => true, 'tags' => ['php', 'mysql']], + ['name' => 'Bob', 'age' => 30, 'score' => 90.0, 'active' => false, 'tags' => null], + ['name' => 'Charlie', 'age' => null, 'score' => 75, 'active' => true, 'department' => 'IT'], + ]; + + protected function setUp(): void + { + parent::setUp(); + + // Insert test data with various types + $this->noneDB->insert($this->testDbName, self::RECORDS); + } + + /** + * Names of the result records, in result order + */ + private function names(array $records): array + { + return array_column($records, 'name'); + } + + /** + * A stored record as find()/get() returns it: its fields plus "key" (fields compared regardless of + * their order, which is not part of the documented result) + */ + private function assertRecord(array $expected, array $actual, string $message = ''): void + { + ksort($expected); + ksort($actual); + $this->assertSame($expected, $actual, $message); + } + + private function storedRecord(int $key): array + { + return self::RECORDS[$key] + ['key' => $key]; + } + + /** + * The database holds exactly the records of setUp(), unchanged + */ + private function assertUnchanged(): void + { + $all = $this->noneDB->find($this->testDbName, 0); + $this->assertCount(3, $all); + foreach ($all as $i => $record) { + $this->assertRecord($this->storedRecord($i), $record); + } + } + + /** + * Groups of a groupBy() result by their _group value (group order is not documented) + * @return array 'v' . var_export(_group) => ['count' => _count, 'names' => names of _items] + */ + private function groupsByValue(array $groups): array + { + $out = []; + foreach ($groups as $group) { + $this->assertSame(['_count', '_group', '_items'], $this->sortedKeys($group)); + $this->assertSame(count($group['_items']), $group['_count']); + // "v" prefix: keys stay strings ('25' would become an int key and sort differently) + $out['v' . var_export($group['_group'], true)] = ['count' => $group['_count'], 'names' => $this->names($group['_items'])]; + } + ksort($out, SORT_STRING); + return $out; + } + + private function sortedKeys(array $array): array + { + $keys = array_keys($array); + sort($keys); + return $keys; + } + + // ========================================== + // WHERE EDGE CASES + // ========================================== + + /** + * @test + */ + public function whereWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where([]) + ->get(); + + // Empty filter should return all records, unchanged, in key order + $this->assertCount(3, $results); + foreach ($results as $i => $record) { + $this->assertRecord($this->storedRecord($i), $record); + } + } + + /** + * @test + */ + public function whereWithNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['nonexistent' => 'value']) + ->get(); + + $this->assertSame([], $results); + // Not even null matches a missing field + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['nonexistent' => null])->get()); + } + + /** + * @test + */ + public function whereWithNullValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['age' => null]) + ->get(); + + $this->assertSame(['Charlie'], $this->names($results)); + $this->assertRecord($this->storedRecord(2), $results[0]); + } + + /** + * @test + */ + public function whereWithBooleanValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['active' => true]) + ->get(); + + $this->assertSame(['Alice', 'Charlie'], $this->names($results)); + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->where(['active' => false])->get())); + } + + /** + * Floats keep their type (API.md "insert()") and plain values compare with ===: 90.0 is not 90, + * 75 is not 75.0 + * @test + */ + public function whereWithFloatValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['score' => 85.5]) + ->get(); + + $this->assertSame(['Alice'], $this->names($results)); + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->where(['score' => 90.0])->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['score' => 90])->get()); + $this->assertSame(['Charlie'], $this->names($this->noneDB->query($this->testDbName)->where(['score' => 75])->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['score' => 75.0])->get()); + } + + /** + * @test + */ + public function whereWithArrayValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->where(['tags' => ['php', 'mysql']]) + ->get(); + + $this->assertSame(['Alice'], $this->names($results)); + // Compared as a whole: another order or a subset doesn't match + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['tags' => ['mysql', 'php']])->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->where(['tags' => ['php']])->get()); + } + + // ========================================== + // WHEREIN EDGE CASES + // ========================================== + + /** + * @test + */ + public function whereInWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('name', []) + ->get(); + + // Empty array means no values match + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereInWithSingleValue(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('name', ['Alice']) + ->get(); + + $this->assertSame(['Alice'], $this->names($results)); + } + + /** + * @test + */ + public function whereInWithNullInArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('age', [25, null]) + ->get(); + + // Alice (25) and Charlie (null) + $this->assertSame(['Alice', 'Charlie'], $this->names($results)); + // null in the list doesn't match a missing field (Alice and Bob have no "department") + $this->assertSame([], $this->noneDB->query($this->testDbName)->whereIn('department', [null])->get()); + } + + /** + * @test + */ + public function whereInWithMixedTypes(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('age', [25, '30', null]) + ->get(); + + // Strict comparison: '30' won't match 30 + $this->assertSame(['Alice', 'Charlie'], $this->names($results)); // Alice (25) and Charlie (null) + } + + /** + * Lists longer than 8 values use a hash set: the result must stay strict + * @test + */ + public function whereInWithLongMixedTypeListIsStillStrict(): void + { + $values = ['25', 30.0, '30', 85.5, 'alice', 'ALICE', 0, false, true, '', 90]; + $this->assertSame([], $this->noneDB->query($this->testDbName)->whereIn('age', $values)->get()); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->whereIn('score', $values)->get())); + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->whereIn('active', $values)->where(['active' => false])->get())); + $this->assertSame(['Alice', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->whereIn('active', $values)->where(['active' => true])->get())); + $this->assertSame(['Charlie'], $this->names($this->noneDB->query($this->testDbName)->whereNotIn('score', $values)->where(['active' => true])->get())); + } + + /** + * @test + */ + public function whereInWithEmptyStringField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereIn('', ['Alice']) + ->get(); + + // Empty field name - should return 0 results + $this->assertSame([], $results); + } + + // ========================================== + // WHERENOTIN EDGE CASES + // ========================================== + + /** + * @test + */ + public function whereNotInWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('name', []) + ->get(); + + // Empty array means nothing to exclude + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function whereNotInWithAllValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->whereNotIn('name', ['Alice', 'Bob', 'Charlie']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * whereNotIn() is strict and keeps records without the field + * @test + */ + public function whereNotInIsStrictAndKeepsMissingFields(): void + { + $this->assertSame(['Bob', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->whereNotIn('age', [25, '30'])->get())); + $this->assertSame(['Alice', 'Bob'], $this->names($this->noneDB->query($this->testDbName)->whereNotIn('department', ['IT'])->get())); + } + + // ========================================== + // LIKE EDGE CASES + // ========================================== + + /** + * @test + */ + public function likeWithEmptyPattern(): void + { + $results = $this->noneDB->query($this->testDbName) + ->like('name', '') + ->get(); + + // Empty pattern matches every string + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function likeWithEmptyField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->like('', 'test') + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function likeWithSpecialRegexChars(): void + { + $this->noneDB->insert($this->testDbName, ['name' => 'Test.User+Name']); + $this->noneDB->insert($this->testDbName, ['name' => 'TestXUserrName']); + + $results = $this->noneDB->query($this->testDbName) + ->like('name', '.User+') + ->get(); + + // Special chars are literal: only the record that contains ".User+" itself + $this->assertSame(['Test.User+Name'], $this->names($results)); + $this->assertSame([3], array_column($results, 'key')); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('name', 'T.st')->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('name', 'User+?')->get()); + } + + /** + * @test + */ + public function likeOnNonStringField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->like('age', '25') + ->get(); + + // Numbers match as their text; null (Charlie) never matches + $this->assertSame(['Alice'], $this->names($results)); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->like('score', '85.5')->get())); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('age', '')->where(['name' => 'Charlie'])->get()); + } + + /** + * @test + */ + public function likeOnArrayField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->like('tags', 'php') + ->get(); + + // Array fields are never matched + $this->assertSame([], $results); + } + + /** + * Booleans never match like() (true would be "1") and notLike() keeps them + * @test + */ + public function likeOnBooleanFieldNeverMatches(): void + { + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('active', '1')->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->like('active', '')->get()); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->notLike('active', '')->get())); + } + + // ========================================== + // NOTLIKE EDGE CASES + // ========================================== + + /** + * @test + */ + public function notLikeWithEmptyPattern(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notLike('name', '') + ->get(); + + // Empty pattern matches everything, so notLike excludes all + $this->assertSame([], $results); + } + + // ========================================== + // BETWEEN EDGE CASES + // ========================================== + + /** + * @test + */ + public function betweenWithSameMinMax(): void + { + $results = $this->noneDB->query($this->testDbName) + ->between('age', 25, 25) + ->get(); + + $this->assertSame(['Alice'], $this->names($results)); + } + + /** + * @test + */ + public function betweenWithReversedRange(): void + { + $results = $this->noneDB->query($this->testDbName) + ->between('age', 30, 25) + ->get(); + + // Min > Max, no results + $this->assertSame([], $results); + } + + /** + * A null bound can't be compared ("null ... never match"): between() matches nothing and + * notBetween() returns every record (its complement) + * @test + */ + public function betweenWithNullBoundMatchesNothing(): void + { + $results = $this->noneDB->query($this->testDbName) + ->between('age', null, 30) + ->get(); + $complement = $this->noneDB->query($this->testDbName) + ->notBetween('age', null, 30) + ->get(); + + $this->assertSame([], $results); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($complement)); + } + + /** + * Strings compare lexically (byte order): "Charlie" > "C", so only Alice and Bob are in ['A', 'C'] + * @test + */ + public function betweenWithStringValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->between('name', 'A', 'C') + ->get(); + + $this->assertSame(['Alice', 'Bob'], $this->names($results)); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->between('name', 'A', 'Charlie')->get())); + // Lexical, not case-insensitive: lower case letters come after upper case ones + $this->assertSame([], $this->noneDB->query($this->testDbName)->between('name', 'a', 'z')->get()); + } + + /** + * @test + */ + public function betweenOnNullField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->between('age', 20, 35) + ->get(); + + // Charlie has null age, should be excluded + $this->assertSame(['Alice', 'Bob'], $this->names($results)); + } + + /** + * Booleans and arrays are never in a range, even when PHP's loose comparison would say so + * @test + */ + public function betweenNeverMatchesBooleansOrArrays(): void + { + $this->assertSame([], $this->noneDB->query($this->testDbName)->between('active', 0, 1)->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->between('active', false, true)->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->between('tags', 'a', 'z')->get()); + } + + // ========================================== + // NOTBETWEEN EDGE CASES + // ========================================== + + /** + * @test + */ + public function notBetweenWithSameMinMax(): void + { + $results = $this->noneDB->query($this->testDbName) + ->notBetween('age', 25, 25) + ->get(); + + // Excludes only 25, keeps 30 and null (null can't be compared) + $this->assertSame(['Bob', 'Charlie'], $this->names($results)); + } + + // ========================================== + // SEARCH EDGE CASES + // ========================================== + + /** + * @test + */ + public function searchWithEmptyTerm(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('') + ->get(); + + // Empty search term matches everything + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function searchWithEmptyFieldsArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('Alice', []) + ->get(); + + // Empty fields array = search all fields + $this->assertSame(['Alice'], $this->names($results)); + // Every field: "IT" only appears in Charlie's department + $this->assertSame(['Charlie'], $this->names($this->noneDB->query($this->testDbName)->search('it', [])->get())); + } + + /** + * @test + */ + public function searchWithNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('Alice', ['nonexistent']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function searchCaseInsensitive(): void + { + $results = $this->noneDB->query($this->testDbName) + ->search('ALICE') + ->get(); + + $this->assertSame(['Alice'], $this->names($results)); + $this->assertSame(['Alice'], $this->names($this->noneDB->query($this->testDbName)->search('aLiCe')->get())); + } + + /** + * Only scalar values are searched: the array "tags" (["php","mysql"]) isn't, booleans aren't, + * numbers are (as text); without a field list "key" isn't searched either + * @test + */ + public function searchOnlyMatchesScalarNonBooleanValues(): void + { + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('php')->get()); + $this->assertSame([], $this->noneDB->query($this->testDbName)->search('1', ['active'])->get()); + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->search('30')->get())); + // "0" is in Bob's age (30) only: Alice's key 0 is not content + $this->assertSame(['Bob'], $this->names($this->noneDB->query($this->testDbName)->search('0')->get())); + } + + // ========================================== + // GROUPBY EDGE CASES + // ========================================== + + /** + * @test + */ + public function groupByNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('nonexistent') + ->get(); + + // All records should be in one group (null) + $this->assertCount(1, $results); + $this->assertNull($results[0]['_group']); + $this->assertSame(3, $results[0]['_count']); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results[0]['_items'])); + } + + /** + * Records without the field are grouped under null + * @test + */ + public function groupByFieldWithNullValues(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('department') + ->get(); + + // One group for 'IT', one for null (Alice and Bob have no department) + $this->assertCount(2, $results); + $this->assertSame([ + "v'IT'" => ['count' => 1, 'names' => ['Charlie']], + 'vNULL' => ['count' => 2, 'names' => ['Alice', 'Bob']], + ], $this->groupsByValue($results)); + } + + /** + * A field holding null and a missing field fall into the same null group; the group value keeps + * its type (true/false are groups of their own) + * @test + */ + public function groupByKeepsValueTypes(): void + { + $this->assertSame([ + 'v25' => ['count' => 1, 'names' => ['Alice']], + 'v30' => ['count' => 1, 'names' => ['Bob']], + 'vNULL' => ['count' => 1, 'names' => ['Charlie']], + ], $this->groupsByValue($this->noneDB->query($this->testDbName)->groupBy('age')->get())); + $this->assertSame([ + 'vfalse' => ['count' => 1, 'names' => ['Bob']], + 'vtrue' => ['count' => 2, 'names' => ['Alice', 'Charlie']], + ], $this->groupsByValue($this->noneDB->query($this->testDbName)->groupBy('active')->get())); + } + + /** + * @test + */ + public function groupByEmptyField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('') + ->get(); + + // Empty field name - all in null group + $this->assertCount(1, $results); + $this->assertNull($results[0]['_group']); + $this->assertSame(3, $results[0]['_count']); + } + + // ========================================== + // HAVING EDGE CASES + // groupBy('active'): true => Alice, Charlie; false => Bob + // ========================================== + + private function activeGroups(): array + { + return [ + 'vfalse' => ['count' => 1, 'names' => ['Bob']], + 'vtrue' => ['count' => 2, 'names' => ['Alice', 'Charlie']], + ]; + } + + /** + * @test + */ + public function havingWithInvalidAggregate(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('active') + ->having('invalid', '>', 0) + ->get(); + + // Invalid aggregate is ignored: every group stays + $this->assertSame($this->activeGroups(), $this->groupsByValue($results)); + } + + /** + * @test + */ + public function havingWithInvalidOperator(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('active') + ->having('count', 'invalid', 0) + ->get(); + + // Invalid operator is ignored: every group stays + $this->assertSame($this->activeGroups(), $this->groupsByValue($results)); + // A valid condition next to it still applies + $this->assertSame(['vtrue' => ['count' => 2, 'names' => ['Alice', 'Charlie']]], $this->groupsByValue( + $this->noneDB->query($this->testDbName)->groupBy('active')->having('count', '==', 0)->having('count', '>', 1)->get())); + } + + /** + * sum:name of non-numeric values is the sum of no values, 0 (as the builder's sum() of a field + * without numbers): "> 0" keeps no group, "= 0" keeps both + * @test + */ + public function havingWithNonNumericField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->groupBy('active') + ->having('sum:name', '>', 0) + ->get(); + + $this->assertSame([], $results); + $this->assertSame($this->activeGroups(), $this->groupsByValue( + $this->noneDB->query($this->testDbName)->groupBy('active')->having('sum:name', '=', 0)->get())); + } + + /** + * having() only applies to groups: without groupBy() it is ignored + * @test + */ + public function havingWithoutGroupByIsIgnored(): void + { + $results = $this->noneDB->query($this->testDbName) + ->having('count', '>', 100) + ->get(); + + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + $this->assertSame(3, $this->noneDB->query($this->testDbName)->having('count', '>', 100)->count()); + } + + // ========================================== + // SELECT/EXCEPT EDGE CASES + // ========================================== + + /** + * @test + */ + public function selectWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select([]) + ->get(); + + // key is always kept, records stay in key order + $this->assertCount(3, $results); + $this->assertSame([0, 1, 2], array_column($results, 'key')); + } + + /** + * @test + */ + public function selectWithNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['nonexistent', 'name']) + ->get(); + + // Only the selected fields that exist, plus key + $this->assertCount(3, $results); + foreach (['Alice', 'Bob', 'Charlie'] as $i => $name) { + $this->assertRecord(['name' => $name, 'key' => $i], $results[$i]); + } + } + + /** + * select() keeps key even when it isn't listed, and a selected null value is kept + * @test + */ + public function selectKeepsKeyAndNullValues(): void + { + $results = $this->noneDB->query($this->testDbName)->select(['age', 'tags'])->get(); + + $this->assertRecord(['age' => 25, 'tags' => ['php', 'mysql'], 'key' => 0], $results[0]); + $this->assertRecord(['age' => 30, 'tags' => null, 'key' => 1], $results[1]); + $this->assertRecord(['age' => null, 'key' => 2], $results[2]); + } + + /** + * @test + */ + public function exceptWithEmptyArray(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except([]) + ->get(); + + // Empty except = all fields + $this->assertCount(3, $results); + foreach ($results as $i => $record) { + $this->assertRecord($this->storedRecord($i), $record); + } + } + + /** + * @test + */ + public function exceptWithNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->except(['nonexistent']) + ->get(); + + // Should work normally: every record complete + $this->assertCount(3, $results); + foreach ($results as $i => $record) { + $this->assertRecord($this->storedRecord($i), $record); + } + } + + /** + * except() drops the listed fields but never key; the stored records keep them + * @test + */ + public function exceptDropsTheListedFieldsKeyToo(): void + { + // QUERY.md select() / except(): except() removes key when it is listed + $results = $this->noneDB->query($this->testDbName)->except(['tags', 'score', 'key', 'department'])->get(); + + $this->assertCount(3, $results); + $this->assertRecord(['name' => 'Alice', 'age' => 25, 'active' => true], $results[0]); + $this->assertRecord(['name' => 'Bob', 'age' => 30, 'active' => false], $results[1]); + $this->assertRecord(['name' => 'Charlie', 'age' => null, 'active' => true], $results[2]); + $this->assertUnchanged(); + } + + /** + * @test + */ + public function selectAndExceptTogether(): void + { + $results = $this->noneDB->query($this->testDbName) + ->select(['name', 'age', 'score']) + ->except(['age']) + ->get(); + + // Select takes precedence; fields outside select() never appear + $this->assertCount(3, $results); + foreach ($results as $i => $record) { + $this->assertSame($i, $record['key']); + $this->assertArrayHasKey('name', $record); + $this->assertArrayHasKey('age', $record); + $this->assertArrayHasKey('score', $record); + foreach (['active', 'tags', 'department'] as $field) { + $this->assertArrayNotHasKey($field, $record); + } + } + } + + // ========================================== + // SORT EDGE CASES + // ========================================== + + /** + * @test + */ + public function sortWithEmptyField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->sort('', 'asc') + ->get(); + + // The field is missing everywhere: all values tie, a stable sort keeps key order + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function sortWithNonExistentField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->sort('nonexistent', 'asc') + ->get(); + $desc = $this->noneDB->query($this->testDbName) + ->sort('nonexistent', 'desc') + ->get(); + + // All ties: key order for asc and for desc (stable) + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($desc)); + } + + /** + * Anything but 'desc' sorts ascending (the default order) + * @test + */ + public function sortWithInvalidOrder(): void + { + $results = $this->noneDB->query($this->testDbName) + ->sort('age', 'invalid') + ->get(); + + // Ascending by age: null (Charlie) < 25 (Alice) < 30 (Bob) - not key order + $this->assertSame(['Charlie', 'Alice', 'Bob'], $this->names($results)); + } + + /** + * null sorts first ("missing / null" < numbers), last with desc + * @test + */ + public function sortOnNullableField(): void + { + $results = $this->noneDB->query($this->testDbName) + ->sort('age', 'asc') + ->get(); + $desc = $this->noneDB->query($this->testDbName) + ->sort('age', 'desc') + ->get(); + + $this->assertSame(['Charlie', 'Alice', 'Bob'], $this->names($results)); + $this->assertSame(['Bob', 'Alice', 'Charlie'], $this->names($desc)); + } + + /** + * Total order: booleans (false < true) < numbers; arrays last; a field holding null and a missing + * field are both in the first class + * @test + */ + public function sortAcrossTypesUsesTotalOrder(): void + { + $this->assertSame(['Bob', 'Alice', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->sort('active')->get())); + // tags: Bob null, Charlie missing (tie, key order) < Alice's array + $this->assertSame(['Bob', 'Charlie', 'Alice'], $this->names($this->noneDB->query($this->testDbName)->sort('tags')->get())); + // score: 75 < 85.5 < 90.0 (int and float together) + $this->assertSame(['Charlie', 'Alice', 'Bob'], $this->names($this->noneDB->query($this->testDbName)->sort('score')->get())); + } + + // ========================================== + // LIMIT/OFFSET EDGE CASES + // ========================================== + + /** + * @test + */ + public function limitWithZero(): void + { + $results = $this->noneDB->query($this->testDbName) + ->limit(0) + ->get(); + + // 0 limit = no results (also for first(), exists() and count()) + $this->assertSame([], $results); + $this->assertNull($this->noneDB->query($this->testDbName)->limit(0)->first()); + $this->assertFalse($this->noneDB->query($this->testDbName)->limit(0)->exists()); + $this->assertSame(0, $this->noneDB->query($this->testDbName)->limit(0)->count()); + } + + /** + * @test + */ + public function limitExceedingRecordCount(): void + { + $results = $this->noneDB->query($this->testDbName) + ->limit(100) + ->get(); + + // Should return all available + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + /** + * @test + */ + public function offsetExceedingRecordCount(): void + { + $results = $this->noneDB->query($this->testDbName) + ->offset(100) + ->get(); + + // Should return empty + $this->assertSame([], $results); + } + + /** + * @test + */ + public function offsetWithZero(): void + { + $results = $this->noneDB->query($this->testDbName) + ->offset(0) + ->get(); + + // 0 offset = start from beginning + $this->assertSame(['Alice', 'Bob', 'Charlie'], $this->names($results)); + } + + // ========================================== + // AGGREGATION EDGE CASES + // sum()/avg() of the builder return float; min()/max() use the sort order and skip null + // ========================================== + + /** + * @test + */ + public function sumOnEmptyResult(): void + { + $sum = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->sum('age'); + + $this->assertSame(0.0, $sum); + } + + /** + * @test + */ + public function sumOnNonNumericField(): void + { + $sum = $this->noneDB->query($this->testDbName) + ->sum('name'); + + // Non-numeric values should be skipped + $this->assertSame(0.0, $sum); + // Booleans are not numbers either (true is not 1) + $this->assertSame(0.0, $this->noneDB->query($this->testDbName)->sum('active')); + } + + /** + * @test + */ + public function sumOnMixedField(): void + { + $sum = $this->noneDB->query($this->testDbName) + ->sum('age'); + + // null should be skipped, 25 + 30 = 55 + $this->assertSame(55.0, $sum); + // int and float together: 85.5 + 90.0 + 75 + $this->assertSame(250.5, $this->noneDB->query($this->testDbName)->sum('score')); + } + + /** + * @test + */ + public function avgOnEmptyResult(): void + { + $avg = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->avg('age'); + + $this->assertSame(0.0, $avg); + } + + /** + * @test + */ + public function avgWithNullValues(): void + { + $avg = $this->noneDB->query($this->testDbName) + ->avg('age'); + + // null should be skipped, (25 + 30) / 2 = 27.5 + $this->assertSame(27.5, $avg); + } + + /** + * @test + */ + public function minOnEmptyResult(): void + { + $min = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->min('age'); + + $this->assertNull($min); + } + + /** + * @test + */ + public function minOnNonExistentField(): void + { + $min = $this->noneDB->query($this->testDbName) + ->min('nonexistent'); + + $this->assertNull($min); + } + + /** + * min()/max() of a field with a null value: the null is no value (as distinct() leaves it out), + * the extremes are 25 and 30; the int/float field keeps the stored types + * @test + */ + public function minAndMaxOnNullableAndMixedNumberFields(): void + { + $this->assertSame(25, $this->noneDB->query($this->testDbName)->min('age')); + $this->assertSame(30, $this->noneDB->query($this->testDbName)->max('age')); + $this->assertSame(75, $this->noneDB->query($this->testDbName)->min('score')); + $this->assertSame(90.0, $this->noneDB->query($this->testDbName)->max('score')); + // Booleans: false < true + $this->assertFalse($this->noneDB->query($this->testDbName)->min('active')); + $this->assertTrue($this->noneDB->query($this->testDbName)->max('active')); + } + + /** + * @test + */ + public function maxOnEmptyResult(): void + { + $max = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->max('age'); + + $this->assertNull($max); + } + + /** + * @test + */ + public function distinctOnEmptyResult(): void + { + $distinct = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->distinct('age'); + + $this->assertSame([], $distinct); + } + + /** + * @test + */ + public function distinctOnNonExistentField(): void + { + $distinct = $this->noneDB->query($this->testDbName) + ->distinct('nonexistent'); + + $this->assertSame([], $distinct); + } + + /** + * @test + */ + public function distinctWithNullValues(): void + { + $distinct = $this->noneDB->query($this->testDbName) + ->distinct('age'); + + // 25, 30 in order of first appearance; null is left out + $this->assertSame([25, 30], $distinct); + // Booleans are values; arrays too (null of Bob's tags left out) + $this->assertSame([true, false], $this->noneDB->query($this->testDbName)->distinct('active')); + $this->assertSame([['php', 'mysql']], $this->noneDB->query($this->testDbName)->distinct('tags')); + } + + // ========================================== + // UPDATE EDGE CASES + // ========================================== + + /** + * An empty set changes nothing and returns the number of matching records (API.md "update()") + * @test + */ + public function updateWithEmptySet(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'Alice']) + ->update([]); + + $this->assertSame(1, $result['n']); + $this->assertUnchanged(); + } + + /** + * @test + */ + public function updateOnEmptyResult(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->update(['age' => 100]); + + $this->assertSame(0, $result['n']); + $this->assertUnchanged(); + } + + /** + * @test + */ + public function updateWithNullValue(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'Alice']) + ->update(['age' => null]); + + $this->assertSame(1, $result['n']); + + $all = $this->noneDB->find($this->testDbName, 0); + // The field stays, with null; the other records are untouched + $this->assertRecord(['age' => null] + $this->storedRecord(0), $all[0]); + $this->assertRecord($this->storedRecord(1), $all[1]); + $this->assertRecord($this->storedRecord(2), $all[2]); + $this->assertSame(['Alice', 'Charlie'], $this->names($this->noneDB->query($this->testDbName)->where(['age' => null])->get())); + } + + // ========================================== + // DELETE EDGE CASES + // ========================================== + + /** + * @test + */ + public function deleteOnEmptyResult(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->delete(); + + $this->assertSame(0, $result['n']); + $this->assertUnchanged(); + } + + // ========================================== + // REMOVEFIELDS EDGE CASES + // ========================================== + + /** + * @test + */ + public function removeFieldsWithEmptyArray(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields([]); + + $this->assertSame(0, $result['n']); + $this->assertSame([], $result['fields_removed']); + $this->assertIsString($result['error']); + $this->assertNotSame('', $result['error']); + $this->assertUnchanged(); + } + + /** + * @test + */ + public function removeFieldsWithOnlyKey(): void + { + $result = $this->noneDB->query($this->testDbName) + ->removeFields(['key']); + + $this->assertSame(0, $result['n']); + $this->assertSame([], $result['fields_removed']); + $this->assertIsString($result['error']); + $this->assertNotSame('', $result['error']); + $this->assertUnchanged(); + } + + /** + * "key" next to other fields is skipped, the others are removed + * @test + */ + public function removeFieldsSkipsKeyNextToOtherFields(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'Bob']) + ->removeFields(['key', 'tags']); + + $this->assertSame(1, $result['n']); + $this->assertSame(['tags'], $result['fields_removed']); + $bob = $this->noneDB->find($this->testDbName, ['key' => 1]); + $expected = $this->storedRecord(1); + unset($expected['tags']); + $this->assertRecord($expected, $bob[0]); + } + + /** + * fields_removed lists the fields actually removed (QUERY.md "removeFields()": 'tmp' isn't listed in + * the example); with no matching record nothing is removed + * @test + */ + public function removeFieldsOnEmptyResult(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->removeFields(['age']); + + $this->assertSame(0, $result['n']); + $this->assertUnchanged(); + $this->assertSame([], $result['fields_removed']); + } + + /** + * @test + */ + public function removeFieldsWithNonExistentField(): void + { + $result = $this->noneDB->query($this->testDbName) + ->where(['name' => 'Alice']) + ->removeFields(['nonexistent']); + + // Should not crash, but no fields actually removed + $this->assertSame(0, $result['n']); + $this->assertSame([], $result['fields_removed']); + $this->assertUnchanged(); + } + + // ========================================== + // FIRST/LAST EDGE CASES + // ========================================== + + /** + * @test + */ + public function firstOnEmptyResult(): void + { + $first = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->first(); + + $this->assertNull($first); + } + + /** + * @test + */ + public function lastOnEmptyResult(): void + { + $last = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->last(); + + $this->assertNull($last); + } + + // ========================================== + // COUNT/EXISTS EDGE CASES + // ========================================== + + /** + * @test + */ + public function countOnEmptyResult(): void + { + $count = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->count(); + + $this->assertSame(0, $count); + } + + /** + * @test + */ + public function existsOnEmptyResult(): void + { + $exists = $this->noneDB->query($this->testDbName) + ->where(['name' => 'NonExistent']) + ->exists(); + + $this->assertFalse($exists); + } + + // ========================================== + // TYPE ERROR TESTS (PHP will throw TypeError) + // ========================================== + + /** + * @test + */ + public function whereThrowsTypeErrorWithNonArray(): void + { + $this->expectException(TypeError::class); + $this->noneDB->query($this->testDbName)->where('invalid'); + } + + /** + * @test + */ + public function whereInWithIntegerFieldName(): void + { + // A numeric field name is a string like any other: no record has field "123" + $results = $this->noneDB->query($this->testDbName) + ->whereIn('123', ['a', 'b']) + ->get(); + + $this->assertSame([], $results); + } + + /** + * @test + */ + public function whereInThrowsTypeErrorWithNull(): void + { + $this->expectException(TypeError::class); + $this->noneDB->query($this->testDbName)->whereIn(null, ['a', 'b']); + } + + /** + * @test + */ + public function whereInThrowsTypeErrorWithNonArray(): void + { + $this->expectException(TypeError::class); + $this->noneDB->query($this->testDbName)->whereIn('field', 'not-array'); + } + + /** + * @test + */ + public function limitThrowsTypeErrorWithNonInt(): void + { + $this->expectException(TypeError::class); + $this->noneDB->query($this->testDbName)->limit('invalid'); + } + + /** + * @test + */ + public function selectThrowsTypeErrorWithNonArray(): void + { + $this->expectException(TypeError::class); + $this->noneDB->query($this->testDbName)->select('invalid'); + } + + /** + * @test + */ + public function updateThrowsTypeErrorWithNonArray(): void + { + $this->expectException(TypeError::class); + try { + $this->noneDB->query($this->testDbName)->update('invalid'); + } finally { + $this->assertUnchanged(); + } + } + + /** + * @test + */ + public function removeFieldsThrowsTypeErrorWithNonArray(): void + { + $this->expectException(TypeError::class); + try { + $this->noneDB->query($this->testDbName)->removeFields('invalid'); + } finally { + $this->assertUnchanged(); + } + } + + /** + * A filter value mixing operators and plain keys throws InvalidArgumentException as soon as where() + * is called (QUERY.md "How values compare") + * @test + */ + public function whereMixingOperatorsAndPlainKeysThrowsAtCall(): void + { + $query = $this->noneDB->query($this->testDbName); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('cannot mix operators'); + $query->where(['age' => ['$gt' => 1, 'plain' => 2]]); + } +} diff --git a/tests/Feature/UpdateTest.php b/tests/Integration/UpdateTest.php similarity index 60% rename from tests/Feature/UpdateTest.php rename to tests/Integration/UpdateTest.php index ceec4bb..07141ea 100644 --- a/tests/Feature/UpdateTest.php +++ b/tests/Integration/UpdateTest.php @@ -1,6 +1,6 @@ noneDB->insert($this->testDbName, $data); } + /** Records as find() returns them after setUp: [key => record] */ + private function original(): array + { + return [ + ['username' => 'john', 'email' => 'john@test.com', 'age' => 25, 'key' => 0], + ['username' => 'jane', 'email' => 'jane@test.com', 'age' => 30, 'key' => 1], + ['username' => 'bob', 'email' => 'bob@test.com', 'age' => 25, 'key' => 2], + ]; + } + + private function all(): array + { + return $this->noneDB->find($this->testDbName, 0); + } + // ========================================== // SUCCESSFUL UPDATE TESTS // ========================================== @@ -38,7 +53,7 @@ public function updateExistingFieldReturnsN1(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); } /** @@ -52,9 +67,10 @@ public function updateExistingFieldChangesValue(): void ]; $this->noneDB->update($this->testDbName, $update); - $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals('newemail@test.com', $result[0]['email']); + $expected = $this->original(); + $expected[0]['email'] = 'newemail@test.com'; + $this->assertSame($expected, $this->all(), 'only that field of that record changed, the key stays'); } /** @@ -70,8 +86,11 @@ public function updateAddNewField(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertArrayHasKey('phone', $result[0]); - $this->assertEquals('555-1234', $result[0]['phone']); + $this->assertCount(1, $result); + $record = $result[0]; + ksort($record); // field order is not specified + $this->assertSame(['age' => 25, 'email' => 'john@test.com', 'key' => 0, 'phone' => '555-1234', 'username' => 'john'], $record, + 'a new field is added to the stored record'); } /** @@ -87,9 +106,11 @@ public function updateMultipleFields(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals('new@test.com', $result[0]['email']); - $this->assertEquals(26, $result[0]['age']); - $this->assertEquals('active', $result[0]['status']); + $this->assertSame('new@test.com', $result[0]['email']); + $this->assertSame(26, $result[0]['age']); + $this->assertSame('active', $result[0]['status']); + $this->assertSame('john', $result[0]['username']); + $this->assertSame(0, $result[0]['key']); } /** @@ -105,7 +126,8 @@ public function updateMultipleRecords(): void $result = $this->noneDB->update($this->testDbName, $update); // john and bob both have age=25 - $this->assertEquals(2, $result['n']); + $this->assertSame(['n' => 2], $result); + $this->assertSame([26, 30, 26], array_column($this->all(), 'age')); } /** @@ -120,10 +142,11 @@ public function updateByKey(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertEquals(1, $result['n']); + $this->assertSame(['n' => 1], $result); $found = $this->noneDB->find($this->testDbName, ['key' => 0]); $this->assertTrue($found[0]['updated']); + $this->assertSame([true, null, null], array_map(function ($r) { return $r['updated'] ?? null; }, $this->all())); } /** @@ -138,7 +161,19 @@ public function updateByMultipleKeys(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertEquals(2, $result['n']); + $this->assertSame(['n' => 2], $result); + $this->assertSame([true, null, true], array_map(function ($r) { return $r['marked'] ?? null; }, $this->all())); + } + + /** + * @test docs/API.md: key accepts a digit string ("5") as well + */ + public function updateByDigitStringKey(): void + { + $this->assertSame(['n' => 1], $this->noneDB->update($this->testDbName, [['key' => '1'], ['set' => ['age' => 31]]])); + $this->assertSame([25, 31, 25], array_column($this->all(), 'age')); + $this->assertSame(['n' => 0], $this->noneDB->update($this->testDbName, [['key' => '1.0'], ['set' => ['age' => 99]]]), 'not a digit string'); + $this->assertSame([25, 31, 25], array_column($this->all(), 'age')); } /** @@ -154,9 +189,7 @@ public function updatePreservesOtherFields(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals('john', $result[0]['username']); - $this->assertEquals('john@test.com', $result[0]['email']); - $this->assertEquals(26, $result[0]['age']); + $this->assertSame([['username' => 'john', 'email' => 'john@test.com', 'age' => 26, 'key' => 0]], $result); } // ========================================== @@ -170,8 +203,8 @@ public function updateNonArrayReturnsError(): void { $result = $this->noneDB->update($this->testDbName, 'not an array'); - $this->assertArrayHasKey('error', $result); - $this->assertEquals(0, $result['n']); + $this->assertSame(['n' => 0, 'error' => 'Please check your update parameters'], $result); + $this->assertSame($this->original(), $this->all()); } /** @@ -186,7 +219,8 @@ public function updateMissingSetKeyReturnsError(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'Please check your update parameters'], $result); + $this->assertSame($this->original(), $this->all()); } /** @@ -201,7 +235,8 @@ public function updateWithReservedKeyInSetReturnsError(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'Please check your update parameters'], $result); + $this->assertSame($this->original(), $this->all(), 'keys never change'); } /** @@ -214,9 +249,10 @@ public function updateSingleElementArrayReturnsError(): void // Missing second element with 'set' ]; - $result = @$this->noneDB->update($this->testDbName, $update); + $result = $this->noneDB->update($this->testDbName, $update); // no warning either - $this->assertArrayHasKey('error', $result); + $this->assertSame(['n' => 0, 'error' => 'Please check your update parameters'], $result); + $this->assertSame($this->original(), $this->all()); } /** @@ -231,8 +267,8 @@ public function updateNoMatchingRecords(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertEquals(0, $result['n']); - $this->assertArrayNotHasKey('error', $result); + $this->assertSame(['n' => 0], $result); + $this->assertSame($this->original(), $this->all()); } // ========================================== @@ -251,8 +287,9 @@ public function updateWithEmptySet(): void $result = $this->noneDB->update($this->testDbName, $update); - // Empty set = no changes, but should match 1 record - $this->assertEquals(1, $result['n']); + // docs/API.md: "An empty set changes nothing and returns the number of matching records" + $this->assertSame(['n' => 1], $result); + $this->assertSame($this->original(), $this->all()); } /** @@ -268,7 +305,9 @@ public function updateToNullValue(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); + $this->assertArrayHasKey('email', $result[0], 'the field stays, with the value null'); $this->assertNull($result[0]['email']); + $this->assertSame([['username' => 'john', 'email' => null, 'age' => 25, 'key' => 0]], $this->noneDB->find($this->testDbName, ['email' => null])); } /** @@ -284,7 +323,7 @@ public function updateToEmptyString(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals('', $result[0]['email']); + $this->assertSame('', $result[0]['email']); } /** @@ -300,7 +339,7 @@ public function updateToZero(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals(0, $result[0]['age']); + $this->assertSame(0, $result[0]['age']); } /** @@ -332,7 +371,7 @@ public function updateWithNestedArray(): void $this->noneDB->update($this->testDbName, $update); $result = $this->noneDB->find($this->testDbName, ['username' => 'john']); - $this->assertEquals('NYC', $result[0]['address']['city']); + $this->assertSame(['city' => 'NYC', 'zip' => '10001'], $result[0]['address']); } /** @@ -348,6 +387,17 @@ public function updateAllRecords(): void $result = $this->noneDB->update($this->testDbName, $update); - $this->assertEquals(3, $result['n']); + $this->assertSame(['n' => 3], $result); + $this->assertSame([true, true, true], array_column($this->all(), 'updated')); + $this->assertSame([0, 1, 2], array_column($this->all(), 'key')); + } + + /** + * @test an update of a database that doesn't exist matches nothing and creates nothing + */ + public function updateOfMissingDatabaseMatchesNothing(): void + { + $this->assertSame(['n' => 0], $this->noneDB->update('missing-db', [[], ['set' => ['a' => 1]]])); + $this->assertDatabaseNotExists('missing-db'); } } diff --git a/tests/Feature/UtilityMethodsTest.php b/tests/Integration/UtilityMethodsTest.php similarity index 54% rename from tests/Feature/UtilityMethodsTest.php rename to tests/Integration/UtilityMethodsTest.php index 961b5e5..811c7cd 100644 --- a/tests/Feature/UtilityMethodsTest.php +++ b/tests/Integration/UtilityMethodsTest.php @@ -1,6 +1,6 @@ noneDB->first($this->testDbName); - $this->assertIsArray($result); - $this->assertEquals('Alice', $result['name']); + $this->assertSame(['age' => 25, 'key' => 0, 'name' => 'Alice', 'score' => 85], $this->canon($result)); } /** @@ -47,8 +46,8 @@ public function firstWithFilter(): void { $result = $this->noneDB->first($this->testDbName, ['age' => 30]); - $this->assertIsArray($result); - $this->assertEquals('Bob', $result['name']); + $this->assertSame(['age' => 30, 'key' => 1, 'name' => 'Bob', 'score' => 90], $this->canon($result)); + $this->assertNull($this->noneDB->first($this->testDbName, ['age' => '30']), 'filters are strict'); } /** @@ -80,7 +79,7 @@ public function firstIncludesKeyField(): void $result = $this->noneDB->first($this->testDbName); $this->assertArrayHasKey('key', $result); - $this->assertEquals(0, $result['key']); + $this->assertSame(0, $result['key']); } // ========================================== @@ -94,8 +93,7 @@ public function lastReturnsLastRecord(): void { $result = $this->noneDB->last($this->testDbName); - $this->assertIsArray($result); - $this->assertEquals('Eve', $result['name']); + $this->assertSame(['age' => 22, 'key' => 4, 'name' => 'Eve', 'score' => 80], $this->canon($result)); } /** @@ -105,8 +103,7 @@ public function lastWithFilter(): void { $result = $this->noneDB->last($this->testDbName, ['age' => 30]); - $this->assertIsArray($result); - $this->assertEquals('Bob', $result['name']); + $this->assertSame(['age' => 30, 'key' => 1, 'name' => 'Bob', 'score' => 90], $this->canon($result)); } /** @@ -138,7 +135,7 @@ public function lastIncludesKeyField(): void $result = $this->noneDB->last($this->testDbName); $this->assertArrayHasKey('key', $result); - $this->assertEquals(4, $result['key']); + $this->assertSame(4, $result['key']); } /** @@ -152,7 +149,8 @@ public function lastWithMultipleMatches(): void $result = $this->noneDB->last($this->testDbName, ['age' => 25]); - $this->assertEquals('Frank', $result['name']); + $this->assertSame(['age' => 25, 'key' => 5, 'name' => 'Frank', 'score' => 70], $this->canon($result)); + $this->assertSame('Alice', $this->noneDB->first($this->testDbName, ['age' => 25])['name']); } // ========================================== @@ -233,9 +231,8 @@ public function sortAscending(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'age', 'asc'); - $this->assertIsArray($result); - $this->assertEquals(22, $result[0]['age']); - $this->assertEquals(35, $result[4]['age']); + $this->assertSame([22, 25, 28, 30, 35], array_column($result, 'age')); + $this->assertSame(['Eve', 'Alice', 'David', 'Bob', 'Charlie'], array_column($result, 'name')); } /** @@ -246,9 +243,8 @@ public function sortDescending(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'age', 'desc'); - $this->assertIsArray($result); - $this->assertEquals(35, $result[0]['age']); - $this->assertEquals(22, $result[4]['age']); + $this->assertSame([35, 30, 28, 25, 22], array_column($result, 'age')); + $this->assertSame(['Charlie', 'Bob', 'David', 'Alice', 'Eve'], array_column($result, 'name')); } /** @@ -259,16 +255,18 @@ public function sortByString(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'name', 'asc'); - $this->assertEquals('Alice', $result[0]['name']); - $this->assertEquals('Eve', $result[4]['name']); + $this->assertSame(['Alice', 'Bob', 'Charlie', 'David', 'Eve'], array_column($result, 'name')); + // Byte order (QUERY.md): upper case before lower case, "1a" before "Z" + $mixed = [['name' => 'a'], ['name' => 'Z'], ['name' => '1a'], ['name' => 'B']]; + $this->assertSame(['1a', 'B', 'Z', 'a'], array_column($this->noneDB->sort($mixed, 'name'), 'name')); } /** * @test */ - public function sortReturnsFalseForEmptyArray(): void + public function sortReturnsEmptyArrayForEmptyInput(): void { - // v3.2.0: sorting an empty result returns an empty result (it used to be false) + // Sorting an empty result returns an empty result (not false) $this->assertSame([], $this->noneDB->sort([], 'age')); $this->assertFalse($this->noneDB->sort('not an array', 'age')); } @@ -291,8 +289,9 @@ public function sortDefaultsToAsc(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'score'); - $this->assertEquals(75, $result[0]['score']); - $this->assertEquals(95, $result[4]['score']); + $this->assertSame([75, 80, 85, 90, 95], array_column($result, 'score')); + $this->assertSame($result, $this->noneDB->sort($data, 'score', 'asc')); + $this->assertSame($result, $this->noneDB->sort($data, 'score', 'other'), 'anything but desc is ascending'); } /** @@ -303,10 +302,12 @@ public function sortPreservesAllFields(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'age'); - $this->assertArrayHasKey('name', $result[0]); - $this->assertArrayHasKey('age', $result[0]); - $this->assertArrayHasKey('score', $result[0]); - $this->assertArrayHasKey('key', $result[0]); + // The same records, untouched, in the new order + $byKey = []; + foreach ($data as $record) { + $byKey[$record['key']] = $record; + } + $this->assertSame([$byKey[4], $byKey[0], $byKey[3], $byKey[1], $byKey[2]], $result); } /** @@ -317,7 +318,8 @@ public function sortIsCaseInsensitiveForOrder(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'age', 'DESC'); - $this->assertEquals(35, $result[0]['age']); + $this->assertSame([35, 30, 28, 25, 22], array_column($result, 'age')); + $this->assertSame($result, $this->noneDB->sort($data, 'age', 'Desc')); } /** @@ -332,8 +334,73 @@ public function sortWithMissingField(): void $data = $this->noneDB->find($this->testDbName, 0); $result = $this->noneDB->sort($data, 'age'); - // Should still return array, records with missing field stay in place - $this->assertIsArray($result); - $this->assertCount(6, $result); + // A missing field sorts like null: first, reversed for desc (API.md sort()) + $this->assertSame(['Frank', 'Eve', 'Alice', 'David', 'Bob', 'Charlie'], array_column($result, 'name')); + $this->assertSame(['Charlie', 'Bob', 'David', 'Alice', 'Eve', 'Frank'], array_column($this->noneDB->sort($data, 'age', 'desc'), 'name')); + } + + /** + * @test + */ + public function sortIsStableForTies(): void + { + $rows = [['n' => 'a', 'v' => 2], ['n' => 'b', 'v' => 1], ['n' => 'c', 'v' => 2], ['n' => 'd', 'v' => 1.0], ['n' => 'e', 'v' => null], ['n' => 'f']]; + // Ties keep the input order, also with desc: 1 and 1.0 are the same number, null and missing the same + $this->assertSame(['e', 'f', 'b', 'd', 'a', 'c'], array_column($this->noneDB->sort($rows, 'v'), 'n')); + $this->assertSame(['a', 'c', 'b', 'd', 'e', 'f'], array_column($this->noneDB->sort($rows, 'v', 'desc'), 'n')); + } + + /** + * @test + */ + public function firstLastExistsWithKeyFiltersKeepTheKeyListOrder(): void + { + $this->assertSame('David', $this->noneDB->first($this->testDbName, ['key' => [3, '1']])['name']); + $this->assertSame('Bob', $this->noneDB->last($this->testDbName, ['key' => [3, '1']])['name']); + $this->assertTrue($this->noneDB->exists($this->testDbName, ['key' => '2'])); + $this->assertFalse($this->noneDB->exists($this->testDbName, ['key' => '02'])); + $this->assertFalse($this->noneDB->exists($this->testDbName, ['key' => 2, 'name' => 'Bob'])); + $this->assertTrue($this->noneDB->exists($this->testDbName, ['key' => 2, 'name' => 'Charlie'])); + $this->assertNull($this->noneDB->first($this->testDbName, ['key' => 9])); + } + + /** + * @test + */ + public function invalidFiltersFindNothing(): void + { + foreach (['Alice', false, 1.5] as $invalid) { + $this->assertNull($this->noneDB->first($this->testDbName, $invalid)); + $this->assertNull($this->noneDB->last($this->testDbName, $invalid)); + $this->assertFalse($this->noneDB->exists($this->testDbName, $invalid)); + } + // 0 and null mean every record + $this->assertSame('Alice', $this->noneDB->first($this->testDbName, null)['name']); + $this->assertSame('Eve', $this->noneDB->last($this->testDbName, 0)['name']); + $this->assertTrue($this->noneDB->exists($this->testDbName, null)); + } + + /** + * @test + */ + public function utilityMethodsOnAMissingDatabaseCreateNothing(): void + { + $before = glob($this->testDbDir . '*'); + $this->assertNull($this->noneDB->first('ghostdb')); + $this->assertNull($this->noneDB->last('ghostdb')); + $this->assertFalse($this->noneDB->exists('ghostdb', ['a' => 1])); + $this->assertSame($before, glob($this->testDbDir . '*')); + } + + /** + * Fields sorted by name (field order is not part of the contract) + */ + private function canon(?array $record): ?array + { + if ($record === null) { + return null; + } + ksort($record); + return $record; } } diff --git a/tests/InternalMethod.php b/tests/InternalMethod.php new file mode 100644 index 0000000..c779413 --- /dev/null +++ b/tests/InternalMethod.php @@ -0,0 +1,31 @@ +invoke($db, ...$args) calls it for that instance, ->invoke(null, ...) calls a static one + */ +final class InternalMethod +{ + /** @var string */ + private $name; + + public function __construct(string $name) + { + $this->name = $name; + } + + /** @return mixed */ + public function invoke(?\noneDB $db, ...$args) + { + return $db === null ? \noneDB\Tests\Support\Testing::callStatic($this->name, ...$args) + : \noneDB\Tests\Support\Testing::call($db, $this->name, ...$args); + } + + /** @return mixed */ + public function invokeArgs(?\noneDB $db, array $args) + { + return $this->invoke($db, ...$args); + } +} diff --git a/tests/InternalValue.php b/tests/InternalValue.php new file mode 100644 index 0000000..c29aed7 --- /dev/null +++ b/tests/InternalValue.php @@ -0,0 +1,30 @@ +name = $name; + } + + /** @return mixed */ + public function getValue(?\noneDB $db = null) + { + return $db === null ? \noneDB\Tests\Support\Testing::get($this->name) : \noneDB\Tests\Support\Testing::getOf($db, $this->name); + } + + public function setValue(?\noneDB $db, $value): void + { + $db === null ? \noneDB\Tests\Support\Testing::set($this->name, $value) : \noneDB\Tests\Support\Testing::setOf($db, $this->name, $value); + } +} diff --git a/tests/Migration/MigrationFixtureTest.php b/tests/Migration/MigrationFixtureTest.php new file mode 100644 index 0000000..2a16f21 --- /dev/null +++ b/tests/Migration/MigrationFixtureTest.php @@ -0,0 +1,169 @@ +fixture($name); + $key = $data['sourceKey']; + $fixtureDir = self::fixturesDir() . "/$name/db"; + foreach ($this->runs($name, $data) as [$options, $status, $oracle, $registered]) { + $label = "$name " . json_encode($options); + $this->cleanTestDirectory(); + $dir = $this->stage($name); + $backup = $this->testDbDir . 'backup/'; + $before = $this->snapshot($dir); + [$exit, $out, $stdout, $stderr] = $this->tool(array_merge(["--dir=$dir", "--backup-dir=$backup", '--confirm-stopped', '--verify'], $options), $key); + $this->assertSame($status, $exit, "$label: exit status\n$stdout$stderr"); + if ($oracle === null) { + $this->assertContains($out['result'] ?? null, ['AMBIGUOUS_SOURCE', 'UNSUPPORTED_SOURCE'], "$label\n$stdout"); + $this->assertSame($before, $this->snapshot($dir), "$label: a stop writes nothing"); + $this->assertDirectoryDoesNotExist($backup, "$label: no backup either"); + continue; + } + $this->assertSame($registered, $out['migrated']['keyRegistered'], $label); + $run = $out['migrated']['run']; + $migrated = array_keys($out['migrated']['databases']); + $this->assertNotEmpty($migrated, $label); + if ($registered) { + $db = $this->v4($dir); + // The tool writes every segment's key-index checkpoint (STORAGE_FORMAT.md §6.4): the engine replays no line + $replayed = \noneDB\Tests\Support\Testing::get('indexStats')['replayedLines']; + foreach ($migrated as $db_name) { + $db->find($db_name, 0); + } + $this->assertSame($replayed, \noneDB\Tests\Support\Testing::get('indexStats')['replayedLines'], "$label: checkpoints used as written"); + $scan = $this->v4($dir, self::TARGET, ['fieldIndexEnabled' => false, 'spatialIndexEnabled' => false]); + foreach ($migrated as $db_name) { + $this->assertDatabase($label, $db, $scan, $db_name, $this->oracleRecords($data, $oracle, $db_name), $fixtureDir, $key, $name); + } + } else { + try { + $this->v4($dir)->find('users', 0); + $this->fail("$label: the key is not registered by a partial run"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $e->getCode(), $label); + } + } + // The old files of the migrated databases: in the backup and retained, byte for byte; none left in the directory + $after = $this->snapshot($dir); + $fixtureFiles = $this->snapshot($fixtureDir); + foreach ($fixtureFiles as $file => $sha) { + $owned = $this->ownedBy($file, $key, $migrated); + if ($owned) { + $this->assertArrayNotHasKey($file, $after, "$label: $file left in the directory"); + $this->assertSame($sha, $after[".nonedb-retained/$run/files/$file"] ?? null, "$label: $file retained"); + $this->assertSame($sha, hash_file('sha256', "$backup$run/files/$file"), "$label: $file in the backup"); + } elseif (!$this->isKeyLevel($file, $key, $registered)) { + $this->assertSame($sha, $after[$file] ?? null, "$label: $file is not this run's and stays as it was"); + } + } + } + } + + /** A migrated database: its records (===), count, next key, layout, indexes, integrity */ + private function assertDatabase(string $label, \noneDB $db, \noneDB $scan, string $name, array $expected, string $fixtureDir, string $key, string $fixture): void + { + $label .= " $name"; + $this->assertSame($expected, $this->v4Records($db, $name), "$label: records"); + $this->assertSame(count($expected), $db->count($name), "$label: count"); + $this->assertTrue($db->checkIntegrity($name)['ok'], "$label: checkIntegrity"); + $info = $db->getShardInfo($name); + $this->assertSame($this->expectedNextKey($fixtureDir, $key, $name, $expected), $info['nextKey'], "$label: next key (§5)"); + $meta = json_decode((string)@file_get_contents($fixtureDir . '/' . hash_pbkdf2('sha256', $name, $key, 1000, 20) . "-$name.nonedb.meta"), true); + $this->assertSame(is_array($meta), $info['sharded'], "$label: layout"); + $this->assertSame(is_array($meta) ? $meta['shardSize'] : 10000, $info['shardSize'], "$label: shard size"); + // Indexes as the scenario created them (tools/make-migration-fixtures.php), answering as the oracle's records + $fields = strpos($fixture, 'field-index') !== false || ($fixture === '3.2-two-databases' && $name === 'orders') ? ['city'] : []; + $this->assertSame($fields, $db->getFieldIndexes($name)['fields'], "$label: field indexes"); + foreach ($fields === [] ? [] : ['ist', 'ank', 'izm', 'bur', 'van', 'none'] as $city) { + $keys = array_keys(array_filter($expected, function ($r) use ($city) { return ($r['city'] ?? null) === $city; })); + $this->assertSame($keys, $this->keysOf($db->find($name, ['city' => $city])), "$label: city $city"); + } + $spatial = strpos($fixture, 'spatial') !== false; + $this->assertSame($spatial ? ['loc'] : [], $db->getSpatialIndexes($name), "$label: spatial indexes (docs/SPATIAL.md: field names)"); + if ($spatial) { + $this->assertSame(array_keys($expected), $this->keysOf($db->withinBBox($name, 'loc', -180, -90, 180, 90)), "$label: spatial"); + } + $this->assertSame($this->keysOf($scan->find($name, 0)), array_keys($expected)); + } + + private function keysOf(array $records): array + { + $keys = array_column($records, 'key'); + sort($keys); + return $keys; + } + + /** A file of the fixture that belongs to one of the migrated databases of the key */ + private function ownedBy(string $file, string $key, array $names): bool + { + foreach ($names as $name) { + $hash = hash_pbkdf2('sha256', $name, $key, 1000, 20); + if (strpos($file, "$hash-$name.") === 0 || strpos($file, "$hash-{$name}_s") === 0 || preg_match('/^' . $hash . '(_s\d+)?\.nonedb\./', $file)) { + return true; + } + } + return false; + } + + /** Files of the key or the directory the run moves when it completes the key (leftovers, hash caches) */ + private function isKeyLevel(string $file, string $key, bool $registered): bool + { + if (!$registered) { + return false; + } + $fingerprint = substr(hash_hmac('sha256', 'nonedb-key-fingerprint-v1', $key), 0, 16); + return $file === ".nonedb_hash_cache_$fingerprint" || $file === '.nonedb_hash_cache' + || (preg_match('/^([0-9a-f]{20})-([^_.]*)/', $file, $m) && hash_pbkdf2('sha256', $m[2], $key, 1000, 20) === $m[1]); + } +} diff --git a/tests/Migration/MigrationProcessTest.php b/tests/Migration/MigrationProcessTest.php new file mode 100644 index 0000000..261bf6d --- /dev/null +++ b/tests/Migration/MigrationProcessTest.php @@ -0,0 +1,753 @@ +fixture(self::FIXTURE); + $db = $this->v4($dir); + foreach (['users', 'orders'] as $name) { + $this->assertSame($this->oracleRecords($data, '3.2', $name), $this->v4Records($db, $name), "$label: $name"); + $this->assertTrue($db->checkIntegrity($name)['ok'], "$label: $name integrity"); + } + $this->assertSame(['city'], $db->getFieldIndexes('orders')['fields'], $label); + $this->assertFileDoesNotExist($dir . '.nonedb-maintenance/journal.json', "$label: the journal is closed"); + // STORAGE_FORMAT.md §9.6: the retained files come with a sealed manifest naming each with its SHA-256 + $retained = glob($dir . '.nonedb-retained/*/manifest.json'); + $this->assertCount(1, $retained, $label); + $manifest = $this->openSealedRecord((string)file_get_contents($retained[0]), 'migration-retained'); + $this->assertIsArray($manifest, "$label: a valid retained manifest"); + foreach ($manifest['files'] as $entry) { + $this->assertSame($entry['sha256'], hash_file('sha256', dirname($retained[0]) . '/files/' . $entry['name']), "$label: {$entry['name']}"); + } + $this->assertDirectoryDoesNotExist($dir . '.nonedb-maintenance/staging', "$label: no staging left"); + } + + /** The directory as it was, apart from the store's entries a stopped run may leave (an empty STORE.LOCK) */ + private function assertUnchanged(array $before, string $dir, string $label, array $allowed = []): void + { + $after = $this->snapshot($dir); + foreach ($allowed as $file => $sha) { + if (($after[$file] ?? null) === $sha) { + unset($after[$file]); + } + } + $this->assertSame($before, $after, $label); + } + + /** The id of the run a crash interrupted: its open journal's, or the closed journal's in the backup */ + private function interruptedRun(string $dir): string + { + if (is_file($dir . '.nonedb-maintenance/journal.json')) { + $journal = $this->openSealedRecord((string)file_get_contents($dir . '.nonedb-maintenance/journal.json'), 'journal'); + $this->assertIsArray($journal, 'a valid open journal'); + return $journal['id']; + } + $closed = glob($this->testDbDir . 'backup/*/journal.done.json'); + $this->assertCount(1, $closed, 'one closed journal'); + return basename(dirname($closed[0])); + } + + /** §7.8: the closed journal of the run records FINAL_VERIFIED, and its report is next to it */ + private function assertFinalVerified(string $run, string $label): void + { + $backup = $this->testDbDir . "backup/$run/"; + $done = $this->openSealedRecord((string)@file_get_contents($backup . 'journal.done.json'), 'journal'); + $this->assertIsArray($done, "$label: a valid closed journal"); + $this->assertSame('FINAL_VERIFIED', $done['state'], "$label: the final verification is recorded"); + $report = json_decode((string)@file_get_contents($backup . 'report.json'), true); + $this->assertSame($run, $report['run'] ?? null, "$label: the report of the run"); + $data = $this->fixture(self::FIXTURE); + $expected = []; + foreach (['orders', 'users'] as $name) { + $expected[$name] = count($this->oracleRecords($data, '3.2', $name)); + } + $counts = $report['databases'] ?? []; + ksort($counts); + $this->assertSame($expected, $counts, "$label: the records of each database"); + } + + private function args(string $dir, array $more = []): array + { + return array_merge(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', '--confirm-stopped'], $more); + } + + private function key(): string + { + return $this->fixture(self::FIXTURE)['sourceKey']; + } + + /** @test §7.1: a dry run reads and reports, and writes nothing */ + public function aDryRunWritesNothing(): void + { + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool(["--dir=$dir", '--dry-run'], $this->key()); + $this->assertSame(0, $status); + $this->assertSame('READY', $out['report']['result']); + $this->assertSame(['orders', 'users'], array_keys($out['report']['databases'])); + $this->assertSame(21, $out['report']['databases']['users']['records']); + $this->assertSame(23, $out['report']['databases']['users']['nextKey']); + $this->assertSame(['city'], $out['report']['databases']['orders']['fields']); + $this->assertSame($before, $this->snapshot($dir)); + $this->assertDirectoryDoesNotExist($this->testDbDir . 'backup'); + } + + /** + * @test STORAGE_FORMAT.md §3, MIGRATION.md §14: what a run writes into the store has the engine's permissions (files + * 0640, directories 0700 + group read and search: 0750), its backup 0600/0700, whatever the umask + */ + public function aRunWritesWithTheEnginesPermissions(): void + { + if (DIRECTORY_SEPARATOR !== '/') { + $this->markTestSkipped('POSIX permissions'); + } + $dir = $this->stage(self::FIXTURE); + $umask = umask(0077); + try { + [$status] = $this->tool($this->args($dir), $this->key()); + } finally { + umask($umask); + } + $this->assertSame(0, $status); + $this->assertMigrated($dir, 'run'); + $modes = function (string $root, string $prefix) use (&$modes): array { + $out = [$prefix => fileperms($root) & 0777]; + if (is_dir($root)) { + foreach (array_diff(scandir($root), ['.', '..']) as $entry) { + $out += $modes("$root/$entry", "$prefix/$entry"); + } + } + return $out; + }; + $store = []; + foreach (array_diff(scandir($dir), ['.', '..']) as $entry) { + if (preg_match('/^(STORE|STORE\.2|STORE\.LOCK|\.htaccess|index\.html|[0-9a-f]{32}\.ndb4|\.nonedb-maintenance)$/', $entry) === 1) { + $store += $modes($dir . $entry, $entry); + } + } + $this->assertGreaterThan(20, count($store)); + foreach ($store as $path => $mode) { + $this->assertSame(sprintf('%o', is_dir($dir . $path) ? 0750 : 0640), sprintf('%o', $mode), $path); + } + foreach ($modes(rtrim($this->testDbDir, '/') . '/backup', 'backup') as $path => $mode) { + $this->assertSame(sprintf('%o', is_dir($this->testDbDir . $path) ? 0700 : 0600), sprintf('%o', $mode), $path); + } + } + + /** @test §7.2: a lock another process holds stops the run before any journal, backup or flag is written */ + public function aBusyLockStopsTheRunBeforeAnythingIsWritten(): void + { + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + $lock = fopen(glob($dir . '*-orders.nonedb.lock')[0], 'c'); + $this->assertTrue(flock($lock, LOCK_EX)); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + flock($lock, LOCK_UN); + fclose($lock); + $this->assertSame(3, $status); + $this->assertSame('MAINTENANCE_BUSY', $out['result']); + $this->assertUnchanged($before, $dir, 'busy', ['STORE.LOCK' => hash('sha256', '')]); + $this->assertDirectoryDoesNotExist($this->testDbDir . 'backup'); + } + + public function crashPoints(): array + { + $points = []; + foreach (['state:DISCOVERED', 'state:BACKED_UP', 'state:STAGING', 'state:VERIFIED', 'state:PUBLISHING', 'published', + 'state:COMMITTED', 'applied:store', 'applied:retained', 'applied:unmarked', 'closed', 'reported'] as $point) { + $points[$point] = [$point]; + } + return $points; + } + + /** + * @test §10: a crash at every step. Until the journal is closed every process gets MAINTENANCE_IN_PROGRESS; the + * next run completes the migration; before the commit --abort leaves the old files as they were, after it --abort + * is refused and the next run completes + * @dataProvider crashPoints + */ + public function aCrashAtEveryStepIsCompletedOrRolledBack(string $point): void + { + $this->requireCrashPoints(); + $committed = array_search($point, array_keys($this->crashPoints()), true) >= 6; + $closed = in_array($point, ['closed', 'reported'], true); // the journal is closed: the final verification is left + foreach (['run again', 'abort'] as $then) { + $this->cleanTestDirectory(); + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + putenv("NONEDB_MIGRATE_KILL=$point"); + [$status] = $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $this->assertNotSame(0, $status, "$point: the process was killed"); + foreach (glob($dir . '*.ndb4') as $home) { + // §7.6: a published database carries m until the run's commit has been applied + $flags = explode(' ', trim(substr((string)file_get_contents("$home/LOCK"), 0, 128))); + $this->assertSame(!in_array($point, ['applied:unmarked', 'closed', 'reported'], true), in_array('m', $flags, true), "$point: m on " . basename($home)); + } + if (!$closed) { + foreach ([self::TARGET, 'some-other-key-of-this-store'] as $key) { + try { + $this->v4($dir, $key)->find('users', 0); + $this->fail("$point: a process read the store during the migration"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::MAINTENANCE_IN_PROGRESS, $e->getCode(), "$point: " . $e->getMessage()); + } + } + } + if ($then === 'run again') { + $killed = $this->interruptedRun($dir); + [$status, $out, $stdout, $stderr] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status, "$point, run again: $stdout$stderr"); + $this->assertMigrated($dir, "$point, run again"); + // §10: the next run completes the interrupted one, through its final verification (§7.8) + $this->assertSame($killed, $out['migrated']['run'], "$point: the run again reports the interrupted run"); + $this->assertFinalVerified($killed, "$point, run again"); + continue; + } + [$status, $out, $stdout] = $this->tool(["--dir=$dir", '--abort'], $this->key()); + if ($closed) { + $this->assertSame(5, $status, "$point: no open run to abort"); + $this->assertMigrated($dir, "$point, after the run"); + } elseif ($committed) { + $this->assertSame(5, $status, "$point: a committed run is not aborted\n$stdout"); + [$status] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status, "$point, run again after the refused abort"); + $this->assertMigrated($dir, "$point, completed"); + } else { + $this->assertSame(0, $status, "$point, abort: $stdout"); + $after = $this->snapshot($dir); + $this->assertSame($before, array_intersect_key($after, $before), "$point, abort: the old files are untouched"); + $this->assertSame([], array_values(array_diff(array_keys($after), array_keys($before), ['STORE.LOCK'])), + "$point, abort: nothing of the run is left but STORE.LOCK"); + } + } + } + + /** @test §7.6: a source file that changes during the run rolls the run back */ + public function aSourceChangedDuringTheRunRollsItBack(): void + { + $this->requireCrashPoints(); + $dir = $this->stage(self::FIXTURE); + putenv('NONEDB_MIGRATE_KILL=state:VERIFIED'); + $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $data = glob($dir . '*-orders_s1.nonedb')[0]; + file_put_contents($data, "{\"n\":\"late\",\"key\":7}\n", FILE_APPEND); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(2, $status); + $this->assertSame('AMBIGUOUS_SOURCE', $out['result']); + $this->assertStringContainsString(basename($data), $out['message']); + $after = $this->snapshot($dir); + $this->assertSame(array_filter($before, function ($file) { return strpos($file, '.nonedb-maintenance/') !== 0; }, ARRAY_FILTER_USE_KEY), + array_intersect_key($after, $before), 'the old files are untouched'); + $this->assertFileDoesNotExist($dir . '.nonedb-maintenance/journal.json'); + $this->assertSame([], glob($dir . '*.ndb4'), 'nothing published'); + } + + /** + * A run during which another process appends to every source file the nth time the run reaches $point: its + * result, the points it reached (the test build's trace), the files before the run (with the appended line) + * and after it + */ + private function runWithSourcesChangedAt(string $point, int $nth): array + { + $this->requireCrashPoints(); + $dir = $this->stage(self::FIXTURE); + $sources = glob($dir . '*.nonedb'); + $this->assertCount(7, $sources, 'precondition: the data files of users (5 shards) and orders (2)'); + $trace = $this->testDbDir . 'trace.txt'; + $line = "{\"n\":\"late\",\"key\":7}\n"; + putenv('NONEDB_MIGRATE_TRACE=' . $trace); + putenv('NONEDB_MIGRATE_CHANGE=' . json_encode(['at' => $point, 'nth' => $nth, 'files' => $sources, 'append' => $line])); + try { + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + } finally { + putenv('NONEDB_MIGRATE_TRACE'); + putenv('NONEDB_MIGRATE_CHANGE'); + } + foreach ($sources as $source) { + $this->assertStringEndsWith($line, (string)file_get_contents($source), 'precondition: changed during the run'); + } + $changed = $before; + foreach ($sources as $source) { + $changed[substr($source, strlen($dir))] = hash_file('sha256', $source); + } + return [$status, $out, file($trace, FILE_IGNORE_NEW_LINES), $changed, $this->snapshot($dir), $dir]; + } + + /** + * @test §7.6: a database whose source files changed after the run checked them is not published: the run stops + * at that database (the first one is published, the second never) and is rolled back + */ + public function aDatabaseWhoseSourceChangedIsNotPublished(): void + { + [$status, $out, $points, $changed, $after, $dir] = $this->runWithSourcesChangedAt('published', 1); + $this->assertSame([2, 'AMBIGUOUS_SOURCE'], [$status, $out['result']]); + $this->assertSame(1, count(array_keys($points, 'published', true)), 'one database published, the other not'); + $this->assertNotContains('state:COMMITTED', $points); + $this->assertSame(array_filter($changed, function ($file) { return strpos($file, '.nonedb-maintenance/') !== 0; }, ARRAY_FILTER_USE_KEY), + array_intersect_key($after, $changed), 'the old files are as the other process left them'); + $this->assertFileDoesNotExist($dir . '.nonedb-maintenance/journal.json'); + $this->assertSame([], glob($dir . '*.ndb4'), 'the published database was taken back'); + } + + /** + * @test §7.7: source files that changed after the last database was published stop the commit: nothing is + * committed and the run is rolled back + */ + public function aSourceChangedAfterTheLastPublishStopsTheCommit(): void + { + [$status, $out, $points, $changed, $after, $dir] = $this->runWithSourcesChangedAt('published', 2); + $this->assertSame([2, 'AMBIGUOUS_SOURCE'], [$status, $out['result']]); + $this->assertSame(2, count(array_keys($points, 'published', true)), 'precondition: both databases published'); + $this->assertNotContains('state:COMMITTED', $points); + $this->assertSame(array_filter($changed, function ($file) { return strpos($file, '.nonedb-maintenance/') !== 0; }, ARRAY_FILTER_USE_KEY), + array_intersect_key($after, $changed), 'the old files are as the other process left them'); + $this->assertFileDoesNotExist($dir . '.nonedb-maintenance/journal.json'); + $this->assertSame([], glob($dir . '*.ndb4'), 'both published databases were taken back'); + } + + /** @test §10: an interrupted run is continued only with the keys it was started with */ + public function anInterruptedRunNeedsItsOwnKeys(): void + { + $this->requireCrashPoints(); + $dir = $this->stage(self::FIXTURE); + putenv('NONEDB_MIGRATE_KILL=state:VERIFIED'); + $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key(), 'migration-test-another-target-key'); + $this->assertSame([4, 'MAINTENANCE_IN_PROGRESS'], [$status, $out['result']]); + $this->assertSame($before, $this->snapshot($dir)); + [$status] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status); + $this->assertMigrated($dir, 'continued with its keys'); + } + + /** + * @test §2 "Settings": noneDB 4 refuses a setting it doesn't know, also in a `.nonedb` file it finds (here in the + * working directory, with noneDB 3.x's autoMigrate): the pre-check stops the run, names the setting and the file, + * and nothing is written + */ + public function aSettingNoneDB4DoesNotKnowStopsTheRunFirst(): void + { + $dir = $this->stage(self::FIXTURE); + $cwd = sys_get_temp_dir() . '/nonedb_migrate_cwd_' . bin2hex(random_bytes(4)); + mkdir($cwd); + try { + file_put_contents("$cwd/.nonedb", json_encode(['secretKey' => self::TARGET, 'dbDir' => './db/', 'autoMigrate' => true])); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key(), self::TARGET, $cwd); + $this->assertSame([5, 'FAILED'], [$status, $out['result']]); + $this->assertStringContainsString('noneDB 4 refuses the key or the configuration', $out['message']); + $this->assertStringContainsString("unknown setting 'autoMigrate' in " . realpath($cwd) . '/.nonedb', $out['message']); + $this->assertSame($before, $this->snapshot($dir), 'the directory is untouched'); + $this->assertFileDoesNotExist($this->testDbDir . 'backup', 'no backup started'); + $this->assertSame(['.nonedb'], array_values(array_diff(scandir($cwd), ['.', '..'])), 'nothing created there'); + } finally { + exec('rm -rf ' . escapeshellarg($cwd)); + } + } + + /** @test §8: a backup that no longer matches its manifest is never trusted: the run does not go on */ + public function aDamagedBackupStopsTheRun(): void + { + $this->requireCrashPoints(); + $dir = $this->stage(self::FIXTURE); + putenv('NONEDB_MIGRATE_KILL=state:BACKED_UP'); + $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $copy = glob($this->testDbDir . 'backup/*/files/*-users_s0.nonedb')[0]; + file_put_contents($copy, 'x', FILE_APPEND); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame([5, 'FAILED'], [$status, $out['result']]); + $this->assertStringContainsString('does not match the manifest', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + } + + /** @test §11, §12: --restore puts the directory back byte for byte, before or after --cleanup */ + public function restorePutsTheOldFilesBack(): void + { + foreach ([false, true] as $cleanup) { + $this->cleanTestDirectory(); + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status); + $run = $out['migrated']['run']; + $backup = $this->testDbDir . 'backup/'; + if ($cleanup) { + [$status, $out] = $this->tool(["--dir=$dir", "--backup-dir=$backup", "--cleanup=$run"], $this->key()); + $this->assertSame(0, $status); + $this->assertDirectoryDoesNotExist($dir . ".nonedb-retained/$run"); + } + [$status, $out, $stdout] = $this->tool(["--dir=$dir", "--backup-dir=$backup", "--restore=$run"], $this->key()); + $this->assertSame(0, $status, $stdout); + $this->assertSame($before, $this->snapshot($dir), $cleanup ? 'restored after cleanup' : 'restored'); + $this->assertCount(2, glob("$backup$run/v4/*.ndb4"), 'the noneDB 4 databases are kept in the backup'); + } + } + + /** + * @test §7.8, §10, §12: a crash after the journal is closed, before the final verification is recorded (it runs + * after the close; with --verify it reads every record). --cleanup refuses the run, nothing removed; the next run + * verifies it (it migrates nothing again) and reports it; then --cleanup removes the retained files + */ + public function aRunClosedBeforeItsFinalVerificationIsVerifiedByTheNextRun(): void + { + $this->requireCrashPoints(); + $dir = $this->stage(self::FIXTURE); + putenv('NONEDB_MIGRATE_KILL=closed'); + [$status] = $this->tool($this->args($dir, ['--verify']), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $this->assertNotSame(0, $status, 'the process was killed'); + $run = $this->interruptedRun($dir); + $backup = $this->testDbDir . 'backup/'; + $done = $this->openSealedRecord((string)file_get_contents("$backup$run/journal.done.json"), 'journal'); + $this->assertSame('COMMITTED', $done['state'], 'closed, not verified'); + $this->assertFileDoesNotExist("$backup$run/report.json"); + + $retained = $this->snapshot($dir . ".nonedb-retained/$run"); + [$status, $out, $stdout] = $this->tool(["--dir=$dir", "--backup-dir=$backup", "--cleanup=$run"], $this->key()); + $this->assertSame(5, $status, $stdout); + $this->assertStringContainsString("the run $run has no final verification; run the tool again first", $out['message'] ?? ''); + $this->assertSame('FAILED', $out['result'] ?? null); + $this->assertSame($retained, $this->snapshot($dir . ".nonedb-retained/$run"), 'nothing removed'); + + // §10: a run with other keys never takes the run as its own: it finds nothing of its keys, the run stays unverified + foreach ([['migration-test-another-source-key', null], [$this->key(), 'migration-test-another-target-key']] as [$source, $target]) { + [$status, $out, $stdout] = $target === null ? $this->tool($this->args($dir), $source) : $this->tool($this->args($dir), $source, $target); + $this->assertSame(0, $status, $stdout); + $this->assertNull($out['migrated']['run'], "other keys: nothing to do"); + $this->assertSame('COMMITTED', $this->openSealedRecord((string)file_get_contents("$backup$run/journal.done.json"), 'journal')['state']); + } + // §7.2: the final verification is maintenance too: another holder of STORE.LOCK stops it, nothing written + $lock = fopen($dir . 'STORE.LOCK', 'c'); + $this->assertTrue(flock($lock, LOCK_EX)); + $held = $this->snapshot($this->testDbDir); + [$status, $out] = $this->tool($this->args($dir, ['--verify']), $this->key()); + flock($lock, LOCK_UN); + fclose($lock); + $this->assertSame(3, $status, 'MAINTENANCE_BUSY'); + $this->assertSame('MAINTENANCE_BUSY', $out['result']); + $this->assertSame($held, $this->snapshot($this->testDbDir), 'nothing written while busy'); + $published = $this->snapshot($dir); + [$status, $out, $stdout] = $this->tool($this->args($dir, ['--verify']), $this->key()); + $this->assertSame(0, $status, $stdout); + $this->assertSame($run, $out['migrated']['run'], 'the run again reports the interrupted run'); + $this->assertFinalVerified($run, 'run again'); + $this->assertTrue($this->openSealedRecord((string)file_get_contents("$backup$run/journal.done.json"), 'journal')['options']['verify'], + 'verified as the run was asked: every record (--verify)'); + $this->assertMigrated($dir, 'run again'); + $this->assertCount(1, glob("$backup*", GLOB_ONLYDIR), 'no other run'); + $this->assertSame(array_keys($published), array_keys($this->snapshot($dir)), 'nothing migrated again'); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status); + $this->assertNull($out['migrated']['run'], 'a verified run is not reported again: nothing left to do'); + + [$status, $out, $stdout] = $this->tool(["--dir=$dir", "--backup-dir=$backup", "--cleanup=$run"], $this->key()); + $this->assertSame(0, $status, $stdout); + $this->assertDirectoryDoesNotExist($dir . ".nonedb-retained/$run"); + } + + /** The temp files of atomic replacements the tool makes in the directory (STORAGE_FORMAT.md §2) */ + private function temps(string $dir): array + { + return array_values(array_filter(array_keys($this->snapshot($dir)), function ($path) { + return preg_match('/\.tmp[0-9a-f]{8}$/', $path) === 1; + })); + } + + /** + * @test STORAGE_FORMAT.md §2: a run killed between writing a temp and renaming it (the journal, a STORE copy) + * leaves that temp; the next run, an abort and a restore leave none: the restore gives back the old directory + * byte for byte + */ + public function aTempFileOfAKilledRunIsRemovedByTheNextOne(): void + { + $this->requireCrashPoints(); + $cases = [ + 'the first journal, run again' => ['renaming:journal.json#1', '.nonedb-maintenance/journal.json', 'run again'], + 'a later journal state, run again' => ['renaming:journal.json#3', '.nonedb-maintenance/journal.json', 'run again'], + 'a later journal state, aborted' => ['renaming:journal.json#3', '.nonedb-maintenance/journal.json', 'abort'], + 'the staging store, run again' => ['renaming:STORE.2#1', '.nonedb-maintenance/staging/STORE.2', 'run again'], + 'STORE.2, run again' => ['renaming:STORE.2#2', 'STORE.2', 'run again'], + 'STORE, run again' => ['renaming:STORE#2', 'STORE', 'run again'], + ]; + foreach ($cases as $label => [$point, $file, $then]) { + $this->cleanTestDirectory(); + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + putenv("NONEDB_MIGRATE_KILL=$point"); + [$status] = $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $this->assertNotSame(0, $status, "$label: killed"); + $temps = $this->temps($dir); + $this->assertCount(1, $temps, "$label: the temp of the file being written"); + $this->assertMatchesRegularExpression('/^' . preg_quote($file, '/') . '\.tmp[0-9a-f]{8}$/', $temps[0], $label); + if ($then === 'abort') { + [$status, $out, $stdout] = $this->tool(["--dir=$dir", '--abort'], $this->key()); + $this->assertSame(0, $status, "$label: $stdout"); + $this->assertUnchanged($before, $dir, "$label: nothing of the run is left but STORE.LOCK", ['STORE.LOCK' => hash('sha256', '')]); + continue; + } + [$status, $out, $stdout] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status, "$label: $stdout"); + $this->assertMigrated($dir, $label); + $this->assertSame([], $this->temps($dir), "$label: no temp left"); + [$status, $out, $stdout] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', '--restore=' . $out['migrated']['run']], $this->key()); + $this->assertSame(0, $status, "$label, restore: $stdout"); + $this->assertSame($before, $this->snapshot($dir), "$label: restored byte for byte"); + } + } + + /** + * @test §7.2, §11: a run stopped before its journal leaves only an empty STORE.LOCK (killed once its locks were + * taken, or stopped by a busy lock). Without a STORE record that file is a leftover of the tool (noneDB 4 creates + * none next to old files): the run that follows counts it as its own, and its restore gives back the directory the + * old release left, byte for byte + */ + public function aRestoreAfterARunStoppedBeforeItsJournalLeavesNoStoreLock(): void + { + foreach (['busy', 'killed'] as $stop) { + $this->cleanTestDirectory(); + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + $this->assertArrayNotHasKey('STORE.LOCK', $before); + if ($stop === 'killed') { + $this->requireCrashPoints(); + putenv('NONEDB_MIGRATE_KILL=locked'); + [$status] = $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_KILL'); + $this->assertNotSame(0, $status, 'the process was killed'); + } else { + $lock = fopen(glob($dir . '*-orders.nonedb.lock')[0], 'c'); + $this->assertTrue(flock($lock, LOCK_EX)); + [$status] = $this->tool($this->args($dir), $this->key()); + flock($lock, LOCK_UN); + fclose($lock); + $this->assertSame(3, $status, 'MAINTENANCE_BUSY'); + } + $this->assertUnchanged($before, $dir, "$stop: only an empty STORE.LOCK", ['STORE.LOCK' => hash('sha256', '')]); + $this->assertFileExists($dir . 'STORE.LOCK', "$stop: the stopped run left STORE.LOCK"); + [$status, $out, $stdout] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status, "$stop, run again: $stdout"); + $this->assertMigrated($dir, "$stop, run again"); + $run = $out['migrated']['run']; + [$status, $out, $stdout] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', "--restore=$run"], $this->key()); + $this->assertSame(0, $status, "$stop, restore: $stdout"); + $this->assertSame($before, $this->snapshot($dir), "$stop: restored byte for byte, STORE.LOCK gone"); + } + } + + /** + * @test §11: restore needs the run's target key. With another key it changes nothing: it could neither see the + * databases that key gained since the run nor take the right key out of the STORE record. + */ + public function restoreNeedsTheRunsTargetKey(): void + { + $dir = $this->stage(self::FIXTURE); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status); + $run = $out['migrated']['run']; + $before = $this->snapshot($dir); + $backup = $this->snapshot($this->testDbDir . 'backup/'); + [$status, $out] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', "--restore=$run"], $this->key(), 'migration-test-another-target-key'); + $this->assertSame([5, 'FAILED'], [$status, $out['result']]); + $this->assertStringContainsString('another key', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + $this->assertSame($backup, $this->snapshot($this->testDbDir . 'backup/')); + [$status] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', "--restore=$run"], $this->key()); + $this->assertSame(0, $status, 'with its key'); + } + + /** @test §11: a key that gained a database after the run is not restored: the database would lose its key */ + public function restoreRefusesWhenTheKeyGainedADatabase(): void + { + $dir = $this->stage(self::FIXTURE); + [, $out] = $this->tool($this->args($dir), $this->key()); + $run = $out['migrated']['run']; + $this->assertSame(['n' => 1], $this->v4($dir, self::TARGET, ['autoCreateDB' => true])->insert('later', ['n' => 1])); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', "--restore=$run"], $this->key()); + $this->assertSame(5, $status); + $this->assertStringContainsString('later', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + } + + /** @test §3, §9: each key is migrated by its own run into the same store; files of the other key are untouched */ + public function twoKeysMigrateIntoOneStore(): void + { + $dir = $this->stage('3.2-two-keys'); + $first = $this->fixture('3.2-two-keys')['sourceKey']; + $second = 'migration-fixture-other-key-4b2d'; + [$status, $out] = $this->tool($this->args($dir), $first); + $this->assertSame(0, $status); + $this->assertSame(['users'], array_keys($out['migrated']['databases'])); + // The other key's files are still there: the engine answers for existing databases, never "doesn't exist" + $this->assertSame([0], array_keys($this->v4Records($this->v4($dir), 'users'))); + try { + $this->v4($dir, self::TARGET, ['autoCreateDB' => true])->insert('new', ['n' => 1]); + $this->fail('a database was created next to files of another key'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::UNSUPPORTED_FORMAT, $e->getCode()); + } + [$status, $out] = $this->tool($this->args($dir), $second, 'migration-test-second-target-key'); + $this->assertSame(0, $status); + $this->assertSame(['orders'], array_keys($out['migrated']['databases'])); + $this->assertSame([0], array_keys($this->v4Records($this->v4($dir, 'migration-test-second-target-key'), 'orders'))); + $this->assertSame([0], array_keys($this->v4Records($this->v4($dir), 'users'))); + $this->assertSame(['n' => 1], $this->v4($dir, self::TARGET, ['autoCreateDB' => true])->insert('new', ['n' => 1]), 'no foreign entry is left'); + } + + /** + * @test §9: in a store another key's run made, a --db subset publishes its database but does not register its + * key while another database of the key is still in the old format; the run that completes the key does + */ + public function aSubsetRunIntoAnExistingStoreLeavesItsKeyUnregistered(): void + { + $dir = $this->stage(self::FIXTURE); + foreach (glob(self::fixturesDir() . '/3.2-two-keys/db/04b1755905ee958a7ffe-orders*') as $file) { // the other key's orders + copy($file, $dir . basename($file)); + } + [$status, $out] = $this->tool($this->args($dir), 'migration-fixture-other-key-4b2d', 'migration-test-second-target-key'); + $this->assertSame([0, true], [$status, $out['migrated']['keyRegistered']], 'precondition: the other key made the store'); + [$status, $out] = $this->tool($this->args($dir, ['--db=users']), $this->key()); + $this->assertSame(0, $status); + $this->assertSame(['users'], array_keys($out['migrated']['databases'])); + $this->assertFalse($out['migrated']['keyRegistered'], 'orders of the key is still in the old format'); + try { + $this->v4($dir)->find('users', 0); + $this->fail('the key was registered'); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $e->getCode()); + } + [$status, $out] = $this->tool($this->args($dir), $this->key()); + $this->assertSame([0, true], [$status, $out['migrated']['keyRegistered']]); + $this->assertSame(['orders'], array_keys($out['migrated']['databases'])); + $data = $this->fixture(self::FIXTURE); + $db = $this->v4($dir); + foreach (['users', 'orders'] as $name) { + $this->assertSame($this->oracleRecords($data, '3.2', $name), $this->v4Records($db, $name), $name); + $this->assertTrue($db->checkIntegrity($name)['ok'], "$name integrity"); + } + $this->assertCount(3, glob($dir . '.nonedb-retained/*/manifest.json'), 'each run retained the sources it migrated'); + $this->assertSame([0], array_keys($this->v4Records($this->v4($dir, 'migration-test-second-target-key'), 'orders')), 'the other key\'s database'); + } + + /** @test §9: a run with nothing left to migrate registers the key a partial run left unregistered */ + public function aRunWithNothingLeftRegistersTheKey(): void + { + $dir = $this->stage('2.2-empty-name'); + $key = $this->fixture('2.2-empty-name')['sourceKey']; + [$status, $out] = $this->tool($this->args($dir, ['--db=users']), $key); + $this->assertSame(0, $status); + $this->assertFalse($out['migrated']['keyRegistered']); + unlink(glob($dir . '*-.nonedb')[0]); // the database with no noneDB 4 name, moved away by its owner + [$status, $out] = $this->tool($this->args($dir), $key); + $this->assertSame(0, $status); + $this->assertTrue($out['migrated']['keyRegistered']); + $this->assertSame([0], array_keys($this->v4Records($this->v4($dir), 'users'))); + [$status, $out] = $this->tool($this->args($dir), $key); + $this->assertSame([0, null], [$status, $out['migrated']['run']], 'nothing left to do'); + } + + /** @test M1 scenario 29: a directory the run cannot write is refused, naming what failed, nothing changes; the next run succeeds */ + public function aDirectoryThatCannotBeWrittenIsRefused(): void + { + if (PHP_OS_FAMILY === 'Windows' || (function_exists('posix_geteuid') && posix_geteuid() === 0)) { + $this->markTestSkipped('a directory that cannot be written is not made with chmod on Windows or for root'); + } + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + chmod($dir, 0555); + try { + [$status, $out] = $this->tool($this->args($dir), $this->key()); + } finally { + chmod($dir, 0777); + } + $this->assertSame([5, 'FAILED'], [$status, $out['result']]); + $this->assertStringContainsString('STORE.LOCK', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + [$status] = $this->tool($this->args($dir), $this->key()); + $this->assertSame(0, $status); + $this->assertMigrated($dir, 'after the permissions were fixed'); + } + + /** @test §9: a run that finds no old file of its key registers nothing and creates nothing (a mistyped key) */ + public function aMistypedKeyIsNeverRegistered(): void + { + $dir = $this->stage('3.2-two-keys'); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool($this->args($dir), 'a-key-these-files-never-had'); + $this->assertSame([0, null, false], [$status, $out['migrated']['run'], $out['migrated']['keyRegistered']]); + $this->assertSame($before, $this->snapshot($dir)); + } + + /** @test §7.1: a run stops before writing anything when the backup and the staged databases don't fit */ + public function notEnoughSpaceStopsTheRunFirst(): void + { + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + putenv('NONEDB_MIGRATE_FREE=1000'); + [$status, $out] = $this->tool($this->args($dir), $this->key()); + putenv('NONEDB_MIGRATE_FREE'); + $this->assertSame([5, 'FAILED'], [$status, $out['result']]); + $this->assertStringContainsString('not enough free space', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + $this->assertDirectoryDoesNotExist($this->testDbDir . 'backup'); + } + + /** @test composer.json "bin": the tool runs through the PHP proxy Composer writes to vendor/bin (it includes the file) */ + public function theToolRunsThroughComposersBinProxy(): void + { + $proxy = $this->testDbDir . 'proxy.php'; + file_put_contents($proxy, " ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $err = stream_get_contents($pipes[2]); + stream_get_contents($pipes[1]); + fclose($pipes[1]); + fclose($pipes[2]); + $this->assertSame(64, proc_close($process), 'the usage of the tool'); + $this->assertStringContainsString('usage: NONEDB_SECRET=', $err); + } + + /** @test §2, §13: usage errors (64) and refused settings (5) change nothing */ + public function optionErrorsChangeNothing(): void + { + $dir = $this->stage(self::FIXTURE); + $before = $this->snapshot($dir); + $usage = [ + ["--dir=$dir"], // neither --dry-run nor --backup-dir/--confirm-stopped + ["--dir=$dir", '--backup-dir=/tmp/x'], // no --confirm-stopped + ["--dir=$dir", '--dry-run', '--abort'], // two modes + ["--dir=$dir", '--dry-run', '--flushing=maybe'], + ["--dir=$dir", '--dry-run', '--shard-size=0'], + ["--dir=$dir", '--dry-run', '--unknown'], + ["--dir=$dir", '--restore=x'], // no --backup-dir + ]; + foreach ($usage as $args) { + $this->assertSame(64, $this->tool($args, $this->key())[0], json_encode($args)); + } + $this->assertSame(5, $this->tool(["--dir=$dir", "--backup-dir={$dir}inside", '--confirm-stopped'], $this->key())[0], 'backup inside'); + $this->assertSame(5, $this->tool($this->args($dir), $this->key(), 'nonedb_123')[0], 'a public target key'); + $this->assertSame(5, $this->tool($this->args($dir), $this->key(), null)[0], 'no key'); + $this->assertSame($before, $this->snapshot($dir)); + } +} diff --git a/tests/Migration/MigrationRulesTest.php b/tests/Migration/MigrationRulesTest.php new file mode 100644 index 0000000..1727408 --- /dev/null +++ b/tests/Migration/MigrationRulesTest.php @@ -0,0 +1,259 @@ + content */ + private function old(string $dir, string $name, array $files, string $key = self::KEY): void + { + @mkdir($dir, 0777, true); + foreach ($files as $suffix => $content) { + file_put_contents($dir . $this->base($name, $key) . $suffix, $content); + } + } + + /** JSONL lines and the o map of a 3.0/3.1 index for them: [data, o] */ + private function jsonl(array $records): array + { + $data = ''; + $o = []; + foreach ($records as $record) { + $line = json_encode($record); + $o[$record['key']] = [strlen($data), strlen($line)]; + $data .= $line . "\n"; + } + return [$data, $o]; + } + + private function dryRun(string $dir, array $options = []): array + { + [$status, $out, $stdout] = $this->tool(array_merge(["--dir=$dir", '--dry-run'], $options), self::KEY); + $this->assertIsArray($out, $stdout); + return [$status, $out['report']]; + } + + private function migrate(string $dir, array $options = []): array + { + [$status, $out, $stdout, $stderr] = $this->tool(array_merge(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', '--confirm-stopped'], $options), self::KEY); + $this->assertSame(0, $status, $stdout . $stderr); + return $out['migrated']; + } + + /** @test §4.1 J30: an index entry of length 0 is not live; lines the map doesn't point at are not live */ + public function aJ30EntryOfLengthZeroIsNotLive(): void + { + $dir = $this->testDbDir . 'r/'; + [$data, $o] = $this->jsonl([['n' => 'a', 'key' => 0], ['n' => 'b', 'key' => 1], ['n' => 'c', 'key' => 2], ['n' => 'gone', 'key' => 3]]); + $o[1][1] = 0; // a failed write of 3.0/3.1 + unset($o[3]); // deleted by 3.0/3.1: no tombstone, only the map lost it + $this->old($dir, 'users', ['.nonedb' => $data, '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 4, 'd' => 1, 'o' => $o])]); + [$status, $report] = $this->dryRun($dir); + $this->assertSame([0, 'READY'], [$status, $report['databases']['users']['status']]); + $this->assertSame(['main' => 'J30'], $report['databases']['users']['segments']); + $this->assertSame(2, $report['databases']['users']['records']); + $this->assertSame(4, $report['databases']['users']['nextKey']); + $this->assertSame(1, $report['databases']['users']['notCarried']['a key index entry of length 0']); + $this->migrate($dir); + $this->assertSame([0 => ['n' => 'a', 'key' => 0], 2 => ['n' => 'c', 'key' => 2]], $this->v4Records($this->v4($dir), 'users')); + } + + /** @test §6.2: an index entry that points at another key's line stops; --rebuild-from-data replays the data */ + public function anEntryAtTheWrongLineStopsUnlessRebuilt(): void + { + $dir = $this->testDbDir . 'r/'; + [$data, $o] = $this->jsonl([['n' => 'a', 'key' => 0], ['n' => 'b', 'key' => 1]]); + $o[0] = $o[1]; + $this->old($dir, 'users', ['.nonedb' => $data, '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 2, 'd' => 0, 'o' => $o])]); + [$status, $report] = $this->dryRun($dir); + $this->assertSame([2, 'AMBIGUOUS_SOURCE'], [$status, $report['result']]); + $this->assertStringContainsString('points key 0', $report['databases']['users']['reasons'][0]); + $this->migrate($dir, ['--rebuild-from-data']); + $this->assertSame([0 => ['n' => 'a', 'key' => 0], 1 => ['n' => 'b', 'key' => 1]], $this->v4Records($this->v4($dir), 'users')); + } + + /** @test §4.1 J32: s beyond the data file stops; complete lines after s are not live */ + public function aJ32IndexMustFitItsData(): void + { + $dir = $this->testDbDir . 'r/'; + [$data, $o] = $this->jsonl([['n' => 'a', 'key' => 0]]); + $this->old($dir, 'users', ['.nonedb' => $data, '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 1, 'd' => 0, 't' => 0, + 's' => strlen($data) + 10, 'o' => $o]), '.nonedb.lock' => str_pad('3 v', 128)]); + $this->assertSame('AMBIGUOUS_SOURCE', $this->dryRun($dir)[1]['result']); + $this->cleanTestDirectory(); + $after = "{\"n\":\"after\",\"key\":1}\n"; + $this->old($dir, 'users', ['.nonedb' => $data . $after, '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 1, 'd' => 0, 't' => 0, + 's' => strlen($data), 'o' => $o]), '.nonedb.lock' => str_pad('3 v', 128)]); + [, $report] = $this->dryRun($dir); + $this->assertSame(1, $report['databases']['users']['records']); + $this->assertSame(2, $report['databases']['users']['nextKey'], 'key 1 was used by the line after s'); + $this->assertSame(1, $report['databases']['users']['notCarried']['complete lines after the key index (3.2 never showed them)']); + $this->migrate($dir); + $this->assertSame([0 => ['n' => 'a', 'key' => 0]], $this->v4Records($this->v4($dir), 'users')); + $this->assertSame(2, $this->v4($dir)->getShardInfo('users')['nextKey']); + } + + /** + * @test §4.1 J33 (3.3-dev, data authoritative): a line glued after a torn append is salvaged; a J30 entry that + * points inside such a line reads its record (M1 scenario 5) + */ + public function gluedLinesAreSalvaged(): void + { + $dir = $this->testDbDir . 'r/'; + $data = "{\"n\":\"a\",\"key\":0}\n{\"n\":\"torn{\"n\":\"b\",\"key\":1}\n{\"__nonedb_deleted\":0}\n"; + $this->old($dir, 'users', ['.nonedb' => $data, '.nonedb.lock' => str_pad('4:0a1b2c3d v', 128)]); + [, $report] = $this->dryRun($dir); + $this->assertSame(['main' => 'J33'], $report['databases']['users']['segments']); + $this->assertSame(1, $report['databases']['users']['records']); + $this->assertSame(1, $report['databases']['users']['notCarried']['bytes glued before a record in its line (the record is salvaged)']); + $this->migrate($dir); + $this->assertSame([1 => ['n' => 'b', 'key' => 1]], $this->v4Records($this->v4($dir), 'users')); + $this->cleanTestDirectory(); + $at = strpos($data, '{"n":"b"'); + $this->old($dir, 'users', ['.nonedb' => "{\"n\":\"a\",\"key\":0}\n{\"n\":\"torn{\"n\":\"b\",\"key\":1}\n", + '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 2, 'd' => 0, 'o' => ['0' => [0, 17], '1' => [$at, strlen('{"n":"b","key":1}')]]])]); + $this->migrate($dir); + $this->assertSame([0 => ['n' => 'a', 'key' => 0], 1 => ['n' => 'b', 'key' => 1]], $this->v4Records($this->v4($dir), 'users')); + } + + /** + * @test §4.1 W: entries that are arrays are records (lists included), others only count for the next key; object + * shaped data keeps its positions (M1 scenario 3) + */ + public function wholeJsonEntries(): void + { + $dir = $this->testDbDir . 'r/'; + $this->old($dir, 'users', ['.nonedb' => '{"data":[null,null,{"n":"a"},"not-a-record",["x","y"],{}]}']); + $this->old($dir, 'gaps', ['.nonedb' => '{"data":{"0":{"n":"a"},"5":{"n":"b"}}}']); + [, $report] = $this->dryRun($dir); + $this->assertSame(6, $report['databases']['users']['nextKey']); + $this->assertSame(1, $report['databases']['users']['notCarried']['an entry that is not a record']); + $this->assertSame(6, $report['databases']['gaps']['nextKey']); + $this->migrate($dir); + $db = $this->v4($dir); + $this->assertSame([2 => ['n' => 'a', 'key' => 2], 4 => [0 => 'x', 1 => 'y', 'key' => 4], 5 => ['key' => 5]], $this->v4Records($db, 'users')); + $this->assertSame([0 => ['n' => 'a', 'key' => 0], 5 => ['n' => 'b', 'key' => 5]], $this->v4Records($db, 'gaps')); + $this->assertSame(6, $db->getShardInfo('users')['nextKey']); + } + + /** @test §4.3: buffers a release never flushed stop the run; buffer lines that are not records are reported */ + public function buffersTheReleaseNeverFlushedStop(): void + { + $dir = $this->testDbDir . 'r/'; + $meta = json_encode(['version' => 1, 'shardSize' => 5, 'totalRecords' => 1, 'deletedCount' => 0, 'nextKey' => 1, + 'shards' => [['id' => 0, 'file' => '_s0', 'count' => 1, 'deleted' => 0]]]); + $base = ['.nonedb.meta' => $meta, '_s0.nonedb' => '{"data":[{"n":"a"}]}']; + $this->old($dir, 'users', $base + ['_s3.nonedb.buffer' => "{\"n\":\"x\"}\n"]); + $this->assertStringContainsString('shard 3', $this->dryRun($dir)[1]['databases']['users']['reasons'][0]); + $this->cleanTestDirectory(); + $this->old($dir, 'users', $base + ['.nonedb.buffer' => "{\"n\":\"x\"}\n"]); + $this->assertStringContainsString('next to a meta', $this->dryRun($dir)[1]['databases']['users']['reasons'][0]); + // With a 3.2 lock file the database's one buffer is flushed from the meta's next key + $this->cleanTestDirectory(); + $this->old($dir, 'users', $base + ['.nonedb.buffer' => "{\"n\":\"x\",\"key\":99}\n\n\"scalar\"\nnot json\n{\"n\":\"y\"}\n", '.nonedb.lock' => str_pad('2 v b', 128)]); + [, $report] = $this->dryRun($dir); + $this->assertSame(2, $report['databases']['users']['buffered']); + $this->assertSame(2, $report['databases']['users']['notCarried']['buffer lines that are not a record']); + $this->migrate($dir); + $this->assertSame([0 => ['n' => 'a', 'key' => 0], 1 => ['n' => 'x', 'key' => 1], 2 => ['n' => 'y', 'key' => 2]], + $this->v4Records($this->v4($dir), 'users'), 'a "key" member of a buffered record is replaced by the assigned key'); + } + + /** @test §4.2: a shard with more entries than the meta's shard size, or a meta with a repeated shard id, stops */ + public function shardsMustFitTheMeta(): void + { + $dir = $this->testDbDir . 'r/'; + $meta = function (array $ids) { + return json_encode(['version' => 1, 'shardSize' => 2, 'totalRecords' => 3, 'deletedCount' => 0, 'nextKey' => 3, + 'shards' => array_map(function ($id) { return ['id' => $id, 'file' => "_s$id", 'count' => 1, 'deleted' => 0]; }, $ids)]); + }; + $this->old($dir, 'users', ['.nonedb.meta' => $meta([0]), '_s0.nonedb' => '{"data":[{"n":1},{"n":2},{"n":3}]}']); + $this->assertStringContainsString('more entries', $this->dryRun($dir)[1]['databases']['users']['reasons'][0]); + $this->cleanTestDirectory(); + $this->old($dir, 'users', ['.nonedb.meta' => $meta([0, 0]), '_s0.nonedb' => '{"data":[{"n":1}]}']); + $this->assertStringContainsString('distinct', $this->dryRun($dir)[1]['databases']['users']['reasons'][0]); + $this->cleanTestDirectory(); + $this->old($dir, 'users', ['.nonedb.meta' => '{"version":1,', '_s0.nonedb' => '{"data":[{"n":1}]}']); + $this->assertStringContainsString('unreadable', $this->dryRun($dir)[1]['databases']['users']['reasons'][0]); + } + + /** @test §6.1 TARGET_EXISTS: nothing is merged into, or written over, a noneDB 4 database of the same name */ + public function anExistingTargetIsNeverOverwritten(): void + { + $dir = $this->testDbDir . 'r/'; + $this->v4($dir, self::TARGET, ['autoCreateDB' => true])->insert('users', ['n' => 'new']); + $this->old($dir, 'users', ['.nonedb' => '{"data":[{"n":"old"}]}']); + $before = $this->snapshot($dir); + [$status, $out] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', '--confirm-stopped'], self::KEY); + $this->assertSame([2, 'AMBIGUOUS_SOURCE'], [$status, $out['result']]); + $this->assertStringContainsString('TARGET_EXISTS', $out['message']); + $this->assertSame($before, $this->snapshot($dir)); + } + + /** + * @test §2, §3: --shard-size sets the shard size of a database that was not sharded; files of other keys and other + * entries are reported and untouched; the 3.0/3.1 hash cache leaves only with the last old file of any key + */ + public function shardSizeOtherKeysAndForeignEntries(): void + { + $dir = $this->testDbDir . 'r/'; + $this->old($dir, 'users', ['.nonedb' => '{"data":[{"n":1}]}', '.nonedbinfo' => '1700000000']); + $this->old($dir, 'theirs', ['.nonedb' => '{"data":[{"n":2}]}'], 'another-key-entirely'); + file_put_contents($dir . 'notes.txt', 'mine'); + file_put_contents($dir . '.nonedb_hash_cache', '{}'); + [, $report] = $this->dryRun($dir, ['--shard-size=7']); + $this->assertSame(['users'], array_keys($report['databases'])); + $this->assertSame([$this->base('theirs', 'another-key-entirely') . '.nonedb'], $report['otherKeys']); + $this->assertSame(['notes.txt'], $report['foreign']); + $migrated = $this->migrate($dir, ['--shard-size=7']); + $this->assertSame(7, $this->v4($dir)->getShardInfo('users')['shardSize']); + $this->assertFileExists($dir . $this->base('theirs', 'another-key-entirely') . '.nonedb'); + $this->assertFileExists($dir . 'notes.txt'); + $this->assertFileExists($dir . '.nonedb_hash_cache', 'another key still has old files: the shared hash cache stays'); + $this->assertFileExists($dir . '.nonedb-retained/' . $migrated['run'] . '/files/' . $this->base('users') . '.nonedbinfo'); + } + + /** + * @test §6: a version no release wrote is not read by its shape. A key index with `o` of another `v` is no usable + * index (§6.2, --rebuild-from-data replays the data file); a meta of another `version` is not supported (§6.1). + */ + public function aVersionNoReleaseWroteStops(): void + { + $dir = $this->testDbDir . 'r/'; + [$data, $o] = $this->jsonl([['n' => 'a', 'key' => 0], ['n' => 'b', 'key' => 1]]); + $this->old($dir, 'users', ['.nonedb' => $data, '.nonedb.jidx' => json_encode(['v' => 99, 'format' => 'jsonl', 'n' => 2, 'o' => $o])]); + [$status, $report] = $this->dryRun($dir); + $this->assertSame([2, 'AMBIGUOUS_SOURCE'], [$status, $report['result']]); + $this->assertSame('READY', $this->dryRun($dir, ['--rebuild-from-data'])[1]['result']); + + $this->cleanTestDirectory(); + $this->old($dir, 'users', ['.nonedb.meta' => json_encode(['version' => 99, 'shardSize' => 10, 'shards' => [], 'nextKey' => 0])]); + [$status, $report] = $this->dryRun($dir); + $this->assertSame([2, 'UNSUPPORTED_SOURCE'], [$status, $report['result']]); + } + + /** @test A name ending in a line feed can't be a noneDB 4 name: §6.1, the `$` of a pattern matches before it */ + public function aNameEndingInALineFeedIsNotSupported(): void + { + if (PHP_OS_FAMILY === 'Windows') { + $this->markTestSkipped('no line feed in a file name'); + } + $dir = $this->testDbDir . 'r/'; + [$data, $o] = $this->jsonl([['n' => 'a', 'key' => 0]]); + $this->old($dir, "users\n", ['.nonedb' => $data, '.nonedb.jidx' => json_encode(['v' => 3, 'format' => 'jsonl', 'n' => 1, 'o' => $o])]); + [$status, $report] = $this->dryRun($dir); + $this->assertSame([2, 'UNSUPPORTED_SOURCE'], [$status, $report['result']]); + } +} diff --git a/tests/Migration/MigrationTestCase.php b/tests/Migration/MigrationTestCase.php new file mode 100644 index 0000000..e50d505 --- /dev/null +++ b/tests/Migration/MigrationTestCase.php @@ -0,0 +1,166 @@ +markTestSkipped('the tool\'s crash points need the posix extension (SIGKILL)'); + } + } + + protected static function fixturesDir(): string + { + $dir = __DIR__ . '/fixtures'; + if (!is_dir($dir)) { + self::fail('no fixtures, they are not in git: run php tools/make-migration-fixtures.php'); + } + return $dir; + } + + /** fixture.json of a fixture */ + protected function fixture(string $name): array + { + return json_decode((string)file_get_contents(self::fixturesDir() . "/$name/fixture.json"), true); + } + + /** A copy of the fixture's files in the test directory: the --dir of a run */ + protected function stage(string $fixture): string + { + $dir = $this->testDbDir . 'm-' . preg_replace('/[^a-z0-9.-]/', '_', $fixture) . '/'; + $this->copyTree(self::fixturesDir() . "/$fixture/db", $dir); + return $dir; + } + + protected function copyTree(string $from, string $to): void + { + @mkdir($to, 0777, true); + foreach (scandir($from) as $entry) { + if ($entry !== '.' && $entry !== '..') { + is_dir("$from/$entry") ? $this->copyTree("$from/$entry", "$to/$entry") : copy("$from/$entry", "$to/$entry"); + } + } + } + + /** + * Run noneDB-migrate.php (the test build) with $args and --json; keys from the environment + * @return array [exit status, decoded output (null when it isn't JSON), stdout, stderr] + */ + protected function tool(array $args, string $sourceKey, ?string $targetKey = self::TARGET, ?string $cwd = null): array + { + $env = getenv(); + unset($env['NONEDB_SECRET'], $env['NONEDB_SOURCE_SECRET']); + if ($targetKey !== null) { + $env['NONEDB_SECRET'] = $targetKey; + } + $env['NONEDB_SOURCE_SECRET'] = $sourceKey; + $process = proc_open(array_merge([PHP_BINARY, NONEDB_TEST_TOOL], $args, ['--json']), [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $cwd, $env); + $out = stream_get_contents($pipes[1]); + $err = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($process); + return [$status, json_decode($out, true), $out, $err]; + } + + /** Every file under $dir: relative path => SHA-256 */ + protected function snapshot(string $dir): array + { + $files = []; + $walk = function (string $path, string $rel) use (&$walk, &$files) { + foreach (scandir($path) as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + is_dir("$path/$entry") ? $walk("$path/$entry", "$rel$entry/") : $files["$rel$entry"] = hash_file('sha256', "$path/$entry"); + } + }; + $walk(rtrim($dir, '/'), ''); + ksort($files); + $walk = null; + return $files; + } + + /** The records a release returned for a database: key => record */ + protected function oracleRecords(array $fixture, string $release, string $name): array + { + $records = []; + foreach ($fixture['oracles'][$release][$name]['records'] as [$key, $record]) { + $records[$key] = $record; + } + return $records; + } + + /** The records the engine returns: key => record */ + protected function v4Records(\noneDB $db, string $name): array + { + $records = []; + foreach ($db->find($name, 0) as $record) { + $records[$record['key']] = $record; + } + ksort($records); + return $records; + } + + protected function v4(string $dir, string $key = self::TARGET, array $config = []): \noneDB + { + \noneDB::clearStaticCache(); + return new \noneDB($config + ['secretKey' => $key, 'dbDir' => $dir, 'autoCreateDB' => false]); + } + + /** + * MIGRATION.md §5, from the files of the fixture (its own reading, not the tool's): the largest of the keys the + * records hold + 1, whole-JSON data counts (with the shard's base), every key of every JSONL line + 1, the key + * indexes' n (with the shard's base) and the meta's nextKey + */ + protected function expectedNextKey(string $fixtureDir, string $sourceKey, string $name, array $records): int + { + $next = $records === [] ? 0 : max(array_keys($records)) + 1; + $base = hash_pbkdf2('sha256', $name, $sourceKey, 1000, 20) . '-' . $name; + $meta = json_decode((string)@file_get_contents("$fixtureDir/$base.nonedb.meta"), true); + $size = is_array($meta) ? $meta['shardSize'] : null; + if (is_array($meta)) { + $next = max($next, (int)($meta['nextKey'] ?? 0)); + $segments = array_map(function ($shard) { return (int)$shard['id']; }, $meta['shards']); + } else { + $segments = [null]; + } + foreach ($segments as $id) { + $offset = $id === null ? 0 : $id * $size; + $data = "$fixtureDir/$base" . ($id === null ? '' : "_s$id") . '.nonedb'; + $raw = (string)@file_get_contents($data); + $whole = json_decode($raw, true); + if (is_array($whole) && array_keys($whole) === ['data']) { + $keys = array_keys($whole['data']); + $next = max($next, $keys === [] ? 0 : $offset + max($keys) + 1); + } else { + foreach (explode("\n", $raw) as $line) { + $value = json_decode($line, true); + $key = $value['key'] ?? $value['__nonedb_deleted'] ?? null; + if (is_int($key)) { + $next = max($next, $key + 1); + } + } + } + $index = json_decode((string)@file_get_contents("$data.jidx"), true); + if (is_array($index) && is_int($index['n'] ?? null)) { + $next = max($next, $offset + $index['n']); + } + } + return $next; + } +} diff --git a/tests/Performance/durability_cost.php b/tests/Performance/durability_cost.php new file mode 100644 index 0000000..7b75f41 --- /dev/null +++ b/tests/Performance/durability_cost.php @@ -0,0 +1,81 @@ + dirname(__DIR__, 2) . '/noneDB.php', 'runs' => 300, 'dir' => sys_get_temp_dir() . '/nonedb_durability_cost', 'md' => null, 'mode' => null]; +foreach (array_slice($argv, 1) as $arg) { + if (preg_match('/^--([a-z]+)=(.*)$/', $arg, $m) && array_key_exists($m[1], $options)) { + $options[$m[1]] = $m[2]; + } +} +$runs = (int)$options['runs']; + +if ($options['mode'] !== null) { + // One mode, in this process: [operation => [median ms, p95 ms, file syncs per op, directory syncs per op]] + require $options['lib']; + $dir = rtrim($options['dir'], '/') . '/' . $options['mode'] . '/'; + exec('rm -rf ' . escapeshellarg($dir)); + $config = ['secretKey' => 'durability-cost-key-0123456789abcdef', 'dbDir' => $dir, 'durability' => $options['mode'], 'shardingEnabled' => false]; + $db = new noneDB($config); + $db->insert('d', array_map(function ($i) { return ['n' => $i, 'v' => str_repeat('x', 40)]; }, range(0, 9999))); + $measure = function (callable $op, int $count) use ($db) { + $times = []; + $before = $db->getDurability()['syncs']; + for ($i = 0; $i < $count; $i++) { + $t = hrtime(true); + $op($i); + $times[] = (hrtime(true) - $t) / 1e6; + } + $after = $db->getDurability()['syncs']; + sort($times); + return [round($times[intdiv($count, 2)], 3), round($times[(int)floor(0.95 * ($count - 1))], 3), + round(($after['file'] - $before['file']) / $count, 2), round(($after['dir'] - $before['dir']) / $count, 2)]; + }; + $next = 10000; + $result = [ + 'insert one record' => $measure(function () use ($db, &$next) { $db->insert('d', ['n' => $next++, 'v' => 'y']); }, $runs), + 'insert 100 records' => $measure(function () use ($db, &$next) { + $db->insert('d', array_map(function ($i) { return ['n' => $i, 'v' => 'z']; }, range($next, $next + 99))); + $next += 100; + }, max(10, intdiv($runs, 10))), + 'update one record by key' => $measure(function ($i) use ($db) { $db->update('d', [['key' => $i], ['set' => ['v' => 'u' . $i]]]); }, $runs), + 'delete one record by key' => $measure(function ($i) use ($db) { $db->delete('d', ['key' => 5000 + $i]); }, $runs), + 'create a database' => $measure(function ($i) use ($db) { $db->createDB('c' . $i); }, max(10, intdiv($runs, 10))), + 'compact (10 000 records, after the writes above)' => $measure(function () use ($db) { $db->compact('d'); }, 1), + ]; + exec('rm -rf ' . escapeshellarg($dir)); + echo json_encode($result); + exit; +} + +if (PHP_OS_FAMILY !== 'Linux') { + fwrite(STDERR, "durable mode is supported on Linux only (docs/DURABILITY.md §1)\n"); + exit(2); +} +$results = []; +foreach (['performance', 'durable'] as $mode) { + $cmd = escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg(__FILE__) . ' --mode=' . $mode . ' --runs=' . $runs + . ' --lib=' . escapeshellarg($options['lib']) . ' --dir=' . escapeshellarg($options['dir']); + $results[$mode] = json_decode((string)shell_exec($cmd), true); + if (!is_array($results[$mode])) { + fwrite(STDERR, "the $mode run failed\n"); + exit(1); + } +} +$md = "# Durability cost: 'durable' against 'performance'\n\n" + . '- PHP ' . PHP_VERSION . ', ' . php_uname('s') . ' ' . php_uname('r') . ', library `' . basename($options['lib']) . '` (sha256 `' + . substr(hash_file('sha256', $options['lib']), 0, 12) . "…`)\n" + . "- One database of 10 000 records, not sharded; $runs runs of each single operation. Times in ms; syncs per operation.\n\n" + . "| Operation | performance median / p95 | durable median / p95 | durable ÷ performance (median) | durable syncs (file + directory) |\n|---|---|---|---|---|\n"; +foreach ($results['performance'] as $op => [$median, $p95]) { + [$dm, $dp, $df, $dd] = $results['durable'][$op]; + $md .= "| $op | $median / $p95 | $dm / $dp | " . ($median > 0 ? round($dm / $median, 1) . '×' : '-') . " | $df + $dd |\n"; +} +echo $md; +if ($options['md'] !== null) { + file_put_contents($options['md'], $md); +} diff --git a/tests/Performance/large.php b/tests/Performance/large.php new file mode 100644 index 0000000..cc89810 --- /dev/null +++ b/tests/Performance/large.php @@ -0,0 +1,337 @@ + --purge (run `sudo -n purge` before each + * OS-cold sample; without it "cold" means a new PHP process with the OS file cache warm) + * + * After the build every file of the database is read once, so each measurement starts with the OS file cache + * warm. Right after a build the new files are not all in the cache yet, and a library that reads indexes + * on demand would pay for that in its "warm" queries, while one that loads everything at its first query + * would not. + * + * Data (deterministic): record j has group = j % (total / 1000) (every group value has exactly 1000 + * records, spread over every shard), metric = j % 100000, tag "row-j" and a fixed payload; field index on + * group. Every measurement runs in its own child process (proc_open, no shell). Results: one JSON line per + * measurement in .jsonl and a Markdown summary in .md. + */ + +error_reporting(E_ALL); +ini_set('display_errors', 'stderr'); + +const SECRET = 'large-bench-0123456789abcdef0123456789abcdef'; +const BATCH = 20000; + +if (($argv[1] ?? '') === 'child') { + exit(childMain(json_decode($argv[2], true))); +} +exit(runMain(array_slice($argv, 1))); + +/** Parent: build every variant, then measure it in child processes */ +function runMain(array $args): int +{ + if (($args[0] ?? '') !== 'run') { + fwrite(STDERR, "usage: php tests/Performance/large.php run --lib=... --out=... [options]\n"); + return 2; + } + $opt = []; + foreach (array_slice($args, 1) as $arg) { + if (preg_match('/^--([a-z-]+)(?:=(.*))?$/', $arg, $m)) { + $opt[$m[1]] = $m[2] ?? true; + } + } + $lib = realpath((string)($opt['lib'] ?? '')); + if ($lib === false || !isset($opt['out'])) { + fwrite(STDERR, "--lib (an existing noneDB.php) and --out are required\n"); + return 2; + } + $total = (int)($opt['total'] ?? 10000000); + $sizes = array_map('intval', explode(',', (string)($opt['shard-sizes'] ?? '100000,10000'))); + $coldSamples = (int)($opt['cold-samples'] ?? 40); + $warmSamples = (int)($opt['warm-samples'] ?? 200); + $purge = !empty($opt['purge']); + $work = rtrim((string)($opt['work'] ?? sys_get_temp_dir()), '/') . '/nonedb-large-' . getmypid(); + $out = (string)$opt['out']; + @mkdir(dirname($out), 0777, true); + $jsonl = $out . '.jsonl'; + file_put_contents($jsonl, ''); + $meta = ['lib' => $lib, 'libSha256' => hash_file('sha256', $lib), 'php' => PHP_VERSION, 'os' => php_uname(), + 'total' => $total, 'shardSizes' => $sizes, 'coldSamples' => $coldSamples, 'warmSamples' => $warmSamples, + 'osCold' => $purge, 'started' => date('c')]; + emit($jsonl, ['meta' => $meta]); + $results = []; + foreach ($sizes as $size) { + $dir = $work . '-' . $size . '/'; + removeTree($dir); + mkdir($dir, 0777, true); + $base = ['lib' => $lib, 'dir' => $dir, 'size' => $size, 'total' => $total]; + $r = []; + $r['build'] = child($base + ['phase' => 'build'], '6000M'); + emit($jsonl, ['shardSize' => $size, 'build' => $r['build']]); + $r['disk'] = diskUsage($dir); + emit($jsonl, ['shardSize' => $size, 'disk' => $r['disk']]); + $r['cacheWarm'] = warmFileCache($dir); + emit($jsonl, ['shardSize' => $size, 'cacheWarm' => $r['cacheWarm']]); + $r['warm'] = child($base + ['phase' => 'warm', 'samples' => $warmSamples, 'limit' => 'auto'], '6000M'); + emit($jsonl, ['shardSize' => $size, 'warm' => $r['warm']]); + foreach (['cold-key', 'cold-group'] as $phase) { + $samples = []; + for ($i = 0; $i < $coldSamples; $i++) { + if ($purge) { + $p = proc_open(['sudo', '-n', 'purge'], [1 => ['file', '/dev/null', 'w'], 2 => ['file', '/dev/null', 'w']], $pipes); + if (!is_resource($p) || proc_close($p) !== 0) { + throw new RuntimeException('--purge needs `sudo -n purge` to work (passwordless sudo for purge)'); + } + } + $samples[] = child($base + ['phase' => $phase, 'i' => $i, 'limit' => 'auto'], '6000M'); + } + $r[$phase] = summarize($samples); + emit($jsonl, ['shardSize' => $size, $phase => $r[$phase]]); + } + $r['limited-384'] = child($base + ['phase' => 'limited', 'samples' => 40, 'limit' => 384 * 1048576], '6000M'); + emit($jsonl, ['shardSize' => $size, 'limited-384' => $r['limited-384']]); + $r['budget-512'] = child($base + ['phase' => 'budget', 'limit' => 'auto'], '512M', true); + emit($jsonl, ['shardSize' => $size, 'budget-512' => $r['budget-512']]); + $r['budget-128'] = child($base + ['phase' => 'budget', 'limit' => 'auto'], '128M', true); + emit($jsonl, ['shardSize' => $size, 'budget-128' => $r['budget-128']]); + $results[$size] = $r; + removeTree($dir); + } + emit($jsonl, ['finished' => date('c')]); + file_put_contents($out . '.md', markdown($meta, $results)); + echo file_get_contents($out . '.md'); + return 0; +} + +/** Run one measurement in a child process; $tolerateFailure: report a failed child instead of throwing */ +function child(array $job, string $memoryLimit, bool $tolerateFailure = false): array +{ + $cmd = [PHP_BINARY, '-d', 'memory_limit=' . $memoryLimit, __FILE__, 'child', json_encode($job)]; + $p = proc_open($cmd, [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $stdout = stream_get_contents($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $code = proc_close($p); + $line = trim((string)strrchr("\n" . trim($stdout), "\n")); + $data = json_decode($line, true); + if ($code !== 0 || !is_array($data)) { + if ($tolerateFailure) { + return ['ok' => false, 'exit' => $code, 'error' => substr(trim($stderr . "\n" . $stdout), -500)]; + } + throw new RuntimeException("child {$job['phase']} failed (exit $code): " . substr($stderr . $stdout, -2000)); + } + return $data; +} + +/** Child: one phase against an existing database */ +function childMain(array $job): int +{ + set_time_limit(0); + $t0 = hrtime(true); + require $job['lib']; + $config = ['secretKey' => SECRET, 'dbDir' => $job['dir'], 'shardSize' => $job['size'], + 'bufferEnabled' => false, 'lockTimeout' => 120]; + if (isset($job['limit'])) { + $config['cacheMemoryLimit'] = $job['limit']; + } + $total = $job['total']; + $groups = intdiv($total, 1000); + $db = new noneDB($config); + $ms = function (callable $fn) { $t = hrtime(true); $r = $fn(); return [(hrtime(true) - $t) / 1e6, $r]; }; + $out = []; + switch ($job['phase']) { + case 'build': + [$t] = $ms(function () use ($db, $total, $groups) { + for ($i = 0; $i < $total; $i += BATCH) { + $rows = []; + for ($j = $i, $u = min($total, $i + BATCH); $j < $u; $j++) { + $rows[] = ['group' => $j % $groups, 'metric' => $j % 100000, 'tag' => 'row-' . $j, 'payload' => 'noneDB-performance-2026']; + } + $r = $db->insert('large', $rows); + if (($r['n'] ?? -1) !== count($rows)) { + throw new RuntimeException('insert failed at ' . $i . ': ' . json_encode($r)); + } + } + }); + $out['insert_ms'] = round($t); + $out['insert_rows_per_s'] = round($total / ($t / 1000)); + [$t, $r] = $ms(function () use ($db) { return $db->createFieldIndex('large', 'group'); }); + if (empty($r['success'])) { + throw new RuntimeException('createFieldIndex failed: ' . json_encode($r)); + } + $out['create_index_ms'] = round($t); + $out['count'] = $db->count('large'); + if ($out['count'] !== $total) { + throw new RuntimeException("count {$out['count']} != $total"); + } + break; + case 'warm': + // First call (loads indexes) and the following calls of the same process + $ops = [ + 'key_lookup' => function ($i) use ($db, $total) { return count($db->find('large', ['key' => ($i * 15485863) % $total])); }, + 'indexed_query_1000' => function ($i) use ($db, $groups) { return count($db->find('large', ['group' => ($i * 97) % $groups])); }, + 'indexed_query_miss' => function ($i) use ($db, $groups) { return count($db->find('large', ['group' => $groups + $i])); }, + 'update_by_key' => function ($i) use ($db, $total) { return $db->update('large', [['key' => ($i * 99991) % $total], ['set' => ['metric' => -1]]])['n']; }, + ]; + foreach ($ops as $name => $fn) { + $times = []; + $first = null; + $n = $name === 'indexed_query_1000' ? max(20, intdiv($job['samples'], 5)) : $job['samples']; + for ($i = 0; $i <= $n; $i++) { + [$t, $r] = $ms(function () use ($fn, $i) { return $fn($i); }); + $expect = ['key_lookup' => 1, 'indexed_query_1000' => 1000, 'indexed_query_miss' => 0, 'update_by_key' => 1][$name]; + if ($r !== $expect) { + throw new RuntimeException("$name returned $r, expected $expect"); + } + if ($i === 0) { + $first = $t; + } else { + $times[] = $t; + } + } + $out[$name] = ['first_ms' => round($first, 3)] + stats($times); + } + break; + case 'cold-key': + [$t, $r] = $ms(function () use ($db, $total, $job) { return count($db->find('large', ['key' => (7654321 + $job['i'] * 15485863) % $total])); }); + if ($r !== 1) { + throw new RuntimeException("cold key returned $r"); + } + $out['ms'] = $t; + break; + case 'cold-group': + [$t, $r] = $ms(function () use ($db, $groups, $job) { return count($db->find('large', ['group' => (4321 + $job['i'] * 97) % $groups])); }); + if ($r !== 1000) { + throw new RuntimeException("cold indexed query returned $r"); + } + $out['ms'] = $t; + break; + case 'limited': + // One process, cacheMemoryLimit = job limit, indexed queries over different values + $times = []; + for ($i = 0; $i < $job['samples']; $i++) { + [$t, $r] = $ms(function () use ($db, $groups, $i) { return count($db->find('large', ['group' => ($i * 7919) % $groups])); }); + if ($r !== 1000) { + throw new RuntimeException("limited query returned $r"); + } + $times[] = $t; + } + $out = stats($times); + break; + case 'budget': + // The query mix of an application process under its memory_limit (512M, and PHP's default 128M) + [$t1] = $ms(function () use ($db, $total) { return $db->find('large', ['key' => 1234567 % $total]); }); + $times = []; + for ($i = 0; $i < 20; $i++) { + [$t, $r] = $ms(function () use ($db, $groups, $i) { return count($db->find('large', ['group' => ($i * 131) % $groups])); }); + if ($r !== 1000) { + throw new RuntimeException("budget query returned $r"); + } + $times[] = $t; + } + $out = ['ok' => true, 'first_key_ms' => round($t1, 3), 'indexed_query_1000' => stats($times)]; + break; + default: + throw new RuntimeException('unknown phase ' . $job['phase']); + } + $out['peak_mib'] = round(memory_get_peak_usage(true) / 1048576, 1); + $out['process_ms'] = round((hrtime(true) - $t0) / 1e6, 1); + echo json_encode($out), "\n"; + return 0; +} + +/** Percentiles (nearest rank) of a list of milliseconds */ +function stats(array $times): array +{ + sort($times); + $n = count($times); + $pick = function ($p) use ($times, $n) { return round($times[max(0, (int)ceil($p * $n) - 1)], 3); }; + return ['n' => $n, 'p50_ms' => $pick(0.50), 'p95_ms' => $pick(0.95), 'p99_ms' => $pick(0.99), + 'max_ms' => round($times[$n - 1], 3), 'mean_ms' => round(array_sum($times) / $n, 3)]; +} + +function summarize(array $samples): array +{ + return stats(array_column($samples, 'ms')) + ['peak_mib_max' => max(array_column($samples, 'peak_mib')), + 'process_ms_p50' => stats(array_column($samples, 'process_ms'))['p50_ms']]; +} + +function emit(string $file, array $row): void +{ + file_put_contents($file, json_encode($row) . "\n", FILE_APPEND); +} + +function diskUsage(string $dir): array +{ + $bytes = 0; + $files = 0; + foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS)) as $f) { + $bytes += $f->getSize(); + $files++; + } + return ['bytes' => $bytes, 'mib' => round($bytes / 1048576, 1), 'files' => $files]; +} + +/** Read every file of the database once (the OS file cache warm before the measurements) */ +function warmFileCache(string $dir): array +{ + $t = hrtime(true); + $bytes = 0; + foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS)) as $f) { + $fp = fopen($f->getPathname(), 'rb'); + while (!feof($fp)) { + $bytes += strlen((string)fread($fp, 1048576)); + } + fclose($fp); + } + return ['bytes' => $bytes, 'ms' => round((hrtime(true) - $t) / 1e6, 1)]; +} + +function removeTree(string $dir): void +{ + if (!is_dir($dir)) { + return; + } + foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::CHILD_FIRST) as $f) { + $f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname()); + } + rmdir($dir); +} + +function markdown(array $meta, array $results): string +{ + $md = "# Large-database benchmark\n\n"; + $md .= "- Library: `{$meta['lib']}` (sha256 `" . substr($meta['libSha256'], 0, 12) . "…`), PHP {$meta['php']}\n"; + $md .= "- Records: " . number_format($meta['total']) . "; cold samples: {$meta['coldSamples']} new processes; OS cache " + . ($meta['osCold'] ? 'purged before every cold sample' : 'warm (no purge)') . "\n"; + $md .= "- After each build every database file is read once (the OS file cache warm before the measurements)\n\n"; + $md .= "| | " . implode(' | ', array_map(function ($s) use ($meta) { return number_format($s) . ' per shard (' . (int)ceil($meta['total'] / $s) . ' shards)'; }, array_keys($results))) . " |\n"; + $md .= "|---|" . str_repeat('---|', count($results)) . "\n"; + $row = function ($label, callable $fn) use ($results) { + return "| $label | " . implode(' | ', array_map($fn, $results)) . " |\n"; + }; + $md .= $row('Bulk insert', function ($r) { return number_format($r['build']['insert_ms'] / 1000, 1) . ' s (' . number_format($r['build']['insert_rows_per_s']) . ' rows/s)'; }); + $md .= $row('createFieldIndex', function ($r) { return number_format($r['build']['create_index_ms'] / 1000, 1) . ' s'; }); + $md .= $row('Build peak memory', function ($r) { return $r['build']['peak_mib'] . ' MiB'; }); + $md .= $row('Disk', function ($r) { return $r['disk']['mib'] . ' MiB, ' . $r['disk']['files'] . ' files'; }); + foreach (['key_lookup' => 'Warm key lookup', 'indexed_query_1000' => 'Warm indexed query (1000 results)', 'indexed_query_miss' => 'Warm indexed miss', 'update_by_key' => 'Warm update by key'] as $k => $label) { + $md .= $row($label . ' p50 / p95 / p99', function ($r) use ($k) { $s = $r['warm'][$k]; return "{$s['p50_ms']} / {$s['p95_ms']} / {$s['p99_ms']} ms (first {$s['first_ms']} ms)"; }); + } + $md .= $row('Warm process peak', function ($r) { return $r['warm']['peak_mib'] . ' MiB'; }); + foreach (['cold-key' => 'Cold key lookup', 'cold-group' => 'Cold indexed query (1000 results)'] as $k => $label) { + $md .= $row($label . ' p50 / p95 / p99', function ($r) use ($k) { $s = $r[$k]; return "{$s['p50_ms']} / {$s['p95_ms']} / {$s['p99_ms']} ms (peak {$s['peak_mib_max']} MiB)"; }); + } + $md .= $row('384 MiB cache, indexed query p50 / p95 / p99', function ($r) { $s = $r['limited-384']; return "{$s['p50_ms']} / {$s['p95_ms']} / {$s['p99_ms']} ms (peak {$s['peak_mib']} MiB)"; }); + foreach (['budget-512' => 'memory_limit 512M query process', 'budget-128' => 'memory_limit 128M (PHP default) query process'] as $k => $label) { + $md .= $row($label, function ($r) use ($k) { + $s = $r[$k]; + return empty($s['ok']) ? 'FAILED (exit ' . $s['exit'] . '): ' . str_replace(["\n", '|'], [' ', '/'], substr($s['error'], -160)) + : "ok, peak {$s['peak_mib']} MiB, indexed p95 {$s['indexed_query_1000']['p95_ms']} ms"; + }); + } + return $md; +} diff --git a/tests/operation_benchmark.php b/tests/Performance/operation_benchmark.php similarity index 92% rename from tests/operation_benchmark.php rename to tests/Performance/operation_benchmark.php index 0c4d426..6f413fb 100644 --- a/tests/operation_benchmark.php +++ b/tests/Performance/operation_benchmark.php @@ -5,12 +5,12 @@ * Measures the cost of one call (ms per operation) on a 5,000-record database, including the * database lock every operation takes. Compare two versions of noneDB.php: * - * php tests/operation_benchmark.php # this checkout - * php tests/operation_benchmark.php /path/to/noneDB.php # another version - * php tests/operation_benchmark.php noneDB.php 3 # best of 3 runs + * php tests/Performance/operation_benchmark.php # this checkout + * php tests/Performance/operation_benchmark.php /path/to/noneDB.php # another version + * php tests/Performance/operation_benchmark.php noneDB.php 3 # best of 3 runs */ -$lib = $argv[1] ?? __DIR__ . '/../noneDB.php'; +$lib = $argv[1] ?? __DIR__ . '/../../noneDB.php'; $runs = max(1, (int)($argv[2] ?? 3)); $records = 5000; require $lib; @@ -35,7 +35,7 @@ function runOnce($cfg, $dir, $records) { noneDB::clearStaticCache(); $db = new noneDB($cfg); if (method_exists($db, 'enableBuffering')) { - $db->enableBuffering(false); // direct writes (the default since 3.2) + $db->enableBuffering(false); // direct writes (the default) } $rows = []; for ($i = 0; $i < $records; $i++) { @@ -71,6 +71,7 @@ function runOnce($cfg, $dir, $records) { } } wipe($dir); +@rmdir($dir); echo "noneDB single-operation benchmark ({$records} records, best of {$runs}, ms per operation)\n"; echo "Library: " . realpath($lib) . "\nPHP " . PHP_VERSION . "\n\n"; diff --git a/tests/operator_benchmark.php b/tests/Performance/operator_benchmark.php similarity index 95% rename from tests/operator_benchmark.php rename to tests/Performance/operator_benchmark.php index ede104f..e88df25 100644 --- a/tests/operator_benchmark.php +++ b/tests/Performance/operator_benchmark.php @@ -1,15 +1,16 @@ = 40 runs for the median) and, for + * single operations, every operation's latency (pooled over runs for the p95, >= 100 samples). + * - ratio = candidate / baseline (> 1: slower). 99% bootstrap confidence interval of the median + * ratio, one-sided Mann-Whitney U test, Hodges-Lehmann shift; p95 ratio with a cluster bootstrap + * (runs are resampled, so latencies of one process are not treated as independent). + * - regression = p < 0.01 and CI lower bound > 1 and ratio > 1.05 (median, or p95 with >= 100 + * samples); concurrent workloads: throughput ratio CI entirely below 0.95; memory: peak of the + * workload (memory_reset_peak_usage, PHP >= 8.2; approximate before) > 5% and > 64 KiB higher. + * Significant but smaller slowdowns are reported as findings. + * - Fixed costs of the fail-closed store are judged by absolute budgets (timeBudget()): a time regression + * within its budget in both runs is reported as "within budget", not counted. + * - A regression counts only when a second, independent run shows it too: "verify" compares two + * run files and exits with 1 if any workload regressed in both. + * - Baseline and candidate the same file (A/A): the noise floor; a workload whose CI is wider than + * +-2% is marked unstable (more runs or a bigger unit of work needed before deciding on it). + */ + +if (PHP_SAPI !== 'cli') { + exit(1); +} +error_reporting(E_ALL); + +const GATE_SECRET = '5f3c8a1e9b2d4f6a7c0e1b3d5f7a9c2e4b6d8f0a1c3e5b7d9f2a4c6e8b0d1f3a'; +const GATE_DB = 'bench'; + +$mode = $argv[1] ?? ''; +if ($mode === 'child') { + exit(childMain(array_slice($argv, 2))); +} +if ($mode === 'conc') { + exit(concurrentChildMain(array_slice($argv, 2))); +} +if ($mode === 'fixture') { + exit(fixtureMain(array_slice($argv, 2))); +} +if ($mode === 'run') { + exit(runMain(parseOptions(array_slice($argv, 2)))); +} +if ($mode === 'verify') { + exit(verifyMain(array_slice($argv, 2))); +} +fwrite(STDERR, "usage: php tests/Performance/perf_gate.php run --baseline=FILE --candidate=FILE --out=FILE [--md=FILE] [--runs=N] [--only=REGEX] [--quick]\n" + . " php tests/Performance/perf_gate.php verify FIRST.json SECOND.json [--md=FILE]\n"); +exit(2); + +// ====================================================================== +// Fixtures and workloads +// ====================================================================== + +/** + * Fixture specs: records inserted in batches of 1000, then the listed indexes created + */ +function fixtureSpecs(): array { + return [ + 'empty' => ['records' => 0, 'config' => ['shardingEnabled' => false]], + 'emptySh' => ['records' => 0, 'config' => ['shardSize' => 1000]], + 'plain100' => ['records' => 100, 'config' => ['shardingEnabled' => false]], + 'plain10k' => ['records' => 10000, 'config' => ['shardingEnabled' => false]], + 'indexed10k' => ['records' => 10000, 'config' => ['shardingEnabled' => false], 'fields' => ['g'], 'spatial' => ['loc']], + 'fields10k' => ['records' => 10000, 'config' => ['shardingEnabled' => false], 'fields' => ['g', 'p']], + 'sharded10k' => ['records' => 10000, 'config' => ['shardSize' => 1000], 'fields' => ['g']], + 'buffer1k' => ['records' => 1000, 'config' => ['shardingEnabled' => false, 'bufferEnabled' => true]], + // Maintenance: 2000 records updated once (old lines to compact; no automatic compaction while built) + 'garbage10k' => ['records' => 10000, 'config' => ['shardingEnabled' => false, 'autoCompactThreshold' => 5], 'supersede' => 'c1'], + 'garbageSh10k' => ['records' => 10000, 'config' => ['shardSize' => 1000, 'autoCompactThreshold' => 5], 'supersede' => 'c1'], + // Shard scale matrix: shardSize 1000, a unique field u and a field g with 32 distinct values per shard + // (below the default limit of 64 per shard); 100 slots per shard left free for the insert workload + 'matrix1' => ['records' => 900, 'config' => ['shardSize' => 1000], 'fields' => ['u', 'g'], 'matrix' => 1], + 'matrix10' => ['records' => 9900, 'config' => ['shardSize' => 1000], 'fields' => ['u', 'g'], 'matrix' => 10], + 'matrix100' => ['records' => 99900, 'config' => ['shardSize' => 1000], 'fields' => ['u', 'g'], 'matrix' => 100], + 'matrix1000' => ['records' => 999900, 'config' => ['shardSize' => 1000], 'fields' => ['u', 'g'], 'matrix' => 1000], + ]; +} + +function makeMatrixRecord(int $n, int $shards): array { + return ['u' => 'u' . $n, 'g' => 'g' . ($n % max(1, 32 * $shards)), 'n' => $n]; +} + +function makeRecord(int $n): array { + return [ + 'n' => $n, + 'c' => 'c' . ($n % 5), + 'g' => 'g' . ($n % 100), + 'p' => ($n * 7919 % 10007) / 100, + 's' => 'text-' . str_repeat(chr(97 + $n % 26), 20 + $n % 30), + 'loc' => ['type' => 'Point', 'coordinates' => [(($n * 37) % 3600) / 10 - 179.95, (($n * 53) % 1700) / 10 - 84.95]], + ]; +} + +/** + * kind: warm (k timed operations after 3 untimed ones), cold (new noneDB + first operation), + * bulk (one timed operation after the index is loaded), conc (parallel processes) + */ +function workloads(): array { + $key = function () { return mt_rand(0, 9999); }; + $box = function () { + $x = mt_rand(-170, 160); + $y = mt_rand(-80, 70); + return [$x, $y, $x + 10, $y + 10]; + }; + $w = []; + // Single operations, warm cache + $w['find_key_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 2000, + 'op' => function ($db) use ($key) { $db->find(GATE_DB, ['key' => $key()]); }]; + $w['insert_one_100_warm'] = ['fixture' => 'plain100', 'kind' => 'warm', 'k' => 150, + 'op' => function ($db, $i) { $db->insert(GATE_DB, makeRecord(100000 + $i)); }]; + $w['insert_one_10k_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 40, + 'op' => function ($db, $i) { $db->insert(GATE_DB, makeRecord(100000 + $i)); }]; + $w['update_one_10k_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 40, + 'op' => function ($db, $i) use ($key) { $db->update(GATE_DB, [['key' => $key()], ['set' => ['p' => $i + 0.5]]]); }]; + $w['delete_one_10k_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 100, + 'op' => function ($db, $i) { $db->delete(GATE_DB, ['key' => ($i * 2477) % 10000]); }]; + $w['insert_one_indexed_warm'] = ['fixture' => 'indexed10k', 'kind' => 'warm', 'k' => 40, + 'op' => function ($db, $i) { $db->insert(GATE_DB, makeRecord(100000 + $i)); }]; + // Updates of an unindexed field next to two field indexes: 400 lines reach an index checkpoint (64 KiB) + $w['update_indexed_10k_warm'] = ['fixture' => 'fields10k', 'kind' => 'warm', 'k' => 400, + 'op' => function ($db, $i) { $db->update(GATE_DB, [['key' => (($i + 10) * 2477) % 10000], ['set' => ['n2' => $i]]]); }]; + // A range on an indexed numeric field (about 5% of the records) + $w['range_indexed_warm'] = ['fixture' => 'fields10k', 'kind' => 'warm', 'k' => 20, + 'op' => function ($db) { $x = mt_rand(0, 9500) / 100; $db->between(GATE_DB, 'p', $x, $x + 5); }]; + $w['find_indexed_eq_warm'] = ['fixture' => 'indexed10k', 'kind' => 'warm', 'k' => 60, + 'op' => function ($db) { $db->find(GATE_DB, ['g' => 'g' . mt_rand(0, 99)]); }]; + $w['count_filter_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 4, + 'op' => function ($db) { $db->count(GATE_DB, ['c' => 'c' . mt_rand(0, 4)]); }]; + $w['sort_limit_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 4, + 'op' => function ($db) { $db->query(GATE_DB)->where(['c' => 'c' . mt_rand(0, 4)])->sort('p', 'desc')->limit(10)->get(); }]; + $w['within_bbox_warm'] = ['fixture' => 'indexed10k', 'kind' => 'warm', 'k' => 200, + 'op' => function ($db) use ($box) { $b = $box(); $db->withinBBox(GATE_DB, 'loc', $b[0], $b[1], $b[2], $b[3]); }]; + $w['within_distance_warm'] = ['fixture' => 'indexed10k', 'kind' => 'warm', 'k' => 200, + 'op' => function ($db) { $db->withinDistance(GATE_DB, 'loc', mt_rand(-170, 170), mt_rand(-80, 80), 300000); }]; + $w['nearest_warm'] = ['fixture' => 'indexed10k', 'kind' => 'warm', 'k' => 120, + 'op' => function ($db) { $db->nearest(GATE_DB, 'loc', mt_rand(-170, 170), mt_rand(-80, 80), 10); }]; + $w['sharded_find_key_warm'] = ['fixture' => 'sharded10k', 'kind' => 'warm', 'k' => 2000, + 'op' => function ($db) use ($key) { $db->find(GATE_DB, ['key' => $key()]); }]; + $w['sharded_find_indexed_warm'] = ['fixture' => 'sharded10k', 'kind' => 'warm', 'k' => 60, + 'op' => function ($db) { $db->find(GATE_DB, ['g' => 'g' . mt_rand(0, 99)]); }]; + $w['sharded_insert_one_warm'] = ['fixture' => 'sharded10k', 'kind' => 'warm', 'k' => 100, + 'op' => function ($db, $i) { $db->insert(GATE_DB, makeRecord(100000 + $i)); }]; + // Store rules: a read of a missing database (the dbDir is checked every time), the first operation + // after another process changed the epoch (caches dropped, layout and store checked again) + $w['find_missing_db_warm'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 400, + 'op' => function ($db) { $db->find('missing' . mt_rand(0, 9), 0); }]; + $w['find_key_after_epoch_change'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 40, + 'op' => function ($db) use ($key) { newLockEpoch($GLOBALS['gateDir']); $db->find(GATE_DB, ['key' => $key()]); }]; + // Single operations, cold (new process, empty static caches) + $w['find_key_cold'] = ['fixture' => 'plain10k', 'kind' => 'cold', + 'op' => function ($db) use ($key) { $db->find(GATE_DB, ['key' => $key()]); }]; + $w['insert_one_cold'] = ['fixture' => 'plain10k', 'kind' => 'cold', + 'op' => function ($db) { $db->insert(GATE_DB, makeRecord(100000)); }]; + $w['find_indexed_cold'] = ['fixture' => 'indexed10k', 'kind' => 'cold', + 'op' => function ($db) { $db->find(GATE_DB, ['g' => 'g' . mt_rand(0, 99)]); }]; + $w['within_bbox_cold'] = ['fixture' => 'indexed10k', 'kind' => 'cold', + 'op' => function ($db) use ($box) { $b = $box(); $db->withinBBox(GATE_DB, 'loc', $b[0], $b[1], $b[2], $b[3]); }]; + $w['sharded_find_indexed_cold'] = ['fixture' => 'sharded10k', 'kind' => 'cold', + 'op' => function ($db) { $db->find(GATE_DB, ['g' => 'g' . mt_rand(0, 99)]); }]; + // Bulk operations + foreach ([1000 => '1k', 10000 => '10k', 100000 => '100k'] as $size => $label) { + $w['bulk_insert_' . $label] = ['fixture' => 'empty', 'kind' => 'bulk', + 'prepare' => function () use ($size) { + $rows = []; + for ($n = 0; $n < $size; $n++) { + $rows[] = makeRecord($n); + } + return $rows; + }, + 'op' => function ($db, $rows) { $db->insert(GATE_DB, $rows); }]; + } + $w['sharded_bulk_insert_10k'] = ['fixture' => 'emptySh', 'kind' => 'bulk', + 'prepare' => function () { + $rows = []; + for ($n = 0; $n < 10000; $n++) { + $rows[] = makeRecord($n); + } + return $rows; + }, + 'op' => function ($db, $rows) { $db->insert(GATE_DB, $rows); }]; + $w['find_all_10k'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 3, + 'op' => function ($db) { $db->find(GATE_DB, 0); }]; + $w['filter_unindexed_10k'] = ['fixture' => 'plain10k', 'kind' => 'warm', 'k' => 3, + 'op' => function ($db) { $db->find(GATE_DB, ['c' => 'c' . mt_rand(0, 4)]); }]; + $w['bulk_update_10k'] = ['fixture' => 'plain10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->update(GATE_DB, [['c' => 'c1'], ['set' => ['flag' => true]]]); }]; + $w['bulk_delete_10k'] = ['fixture' => 'plain10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->delete(GATE_DB, ['c' => 'c1']); }]; + $w['buffer_insert_flush'] = ['fixture' => 'buffer1k', 'kind' => 'bulk', + 'op' => function ($db) { + for ($i = 0; $i < 1000; $i++) { + $db->insert(GATE_DB, makeRecord(200000 + $i)); + } + $db->flush(GATE_DB); + }]; + // Maintenance (data file replacement, index builds) + $w['compact_10k'] = ['fixture' => 'garbage10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->compact(GATE_DB); }]; + $w['sharded_compact_10k'] = ['fixture' => 'garbageSh10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->compact(GATE_DB); }]; + $w['delete_autocompact_10k'] = ['fixture' => 'plain10k', 'kind' => 'bulk', // 40%: crosses the 30% garbage threshold + 'op' => function ($db) { $db->delete(GATE_DB, ['c' => ['$in' => ['c1', 'c2']]]); }]; + $w['create_field_index_10k'] = ['fixture' => 'plain10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->createFieldIndex(GATE_DB, 'g'); }]; + $w['rebuild_index_10k'] = ['fixture' => 'plain10k', 'kind' => 'bulk', + 'op' => function ($db) { $db->rebuildIndex(GATE_DB); }]; + // Concurrent processes on one database (unit = wall time from a common start) + $w['conc_writers_4'] = ['fixture' => 'plain10k', 'kind' => 'conc', 'runs' => 30, 'roles' => ['w' => 4], 'k' => ['w' => 60]]; + $w['conc_writers_8'] = ['fixture' => 'plain10k', 'kind' => 'conc', 'runs' => 30, 'roles' => ['w' => 8], 'k' => ['w' => 60]]; + $w['conc_readers_4'] = ['fixture' => 'plain10k', 'kind' => 'conc', 'runs' => 30, 'roles' => ['r' => 4], 'k' => ['r' => 1500]]; + $w['conc_mixed_8'] = ['fixture' => 'plain10k', 'kind' => 'conc', 'runs' => 30, 'roles' => ['w' => 4, 'r' => 4], 'k' => ['w' => 60, 'r' => 1500]]; + + // Shard scale matrix (T2-7: global field index of a high-cardinality field). Fixtures are used in place + // (1M records at 1000 shards are too big to copy per run): lookups run before the inserts. + foreach ([1, 10, 100, 1000] as $shards) { + $fixture = 'matrix' . $shards; + $u = function () use ($shards) { return 'u' . mt_rand(0, $shards * 1000 - 101); }; + $w["m{$shards}_lookup_hit_warm"] = ['fixture' => $fixture, 'kind' => 'warm', 'k' => 200, 'inPlace' => true, + 'op' => function ($db) use ($u) { $db->find(GATE_DB, ['u' => $u()]); }]; + $w["m{$shards}_lookup_miss_warm"] = ['fixture' => $fixture, 'kind' => 'warm', 'k' => 200, 'inPlace' => true, + 'op' => function ($db, $i) { $db->find(GATE_DB, ['u' => 'absent' . $i]); }]; + $w["m{$shards}_g_lookup_warm"] = ['fixture' => $fixture, 'kind' => 'warm', 'k' => 100, 'inPlace' => true, + 'op' => function ($db) use ($shards) { $db->find(GATE_DB, ['g' => 'g' . mt_rand(0, max(1, 32 * $shards) - 1), 'n' => -1]); }]; + $w["m{$shards}_lookup_hit_cold"] = ['fixture' => $fixture, 'kind' => 'cold', 'inPlace' => true, + 'op' => function ($db) use ($u) { $db->find(GATE_DB, ['u' => $u()]); }]; + $w["m{$shards}_lookup_miss_cold"] = ['fixture' => $fixture, 'kind' => 'cold', 'inPlace' => true, + 'op' => function ($db) { $db->find(GATE_DB, ['u' => 'absent']); }]; + $w["m{$shards}_insert_warm"] = ['fixture' => $fixture, 'kind' => 'warm', 'k' => 25, 'inPlace' => true, + 'op' => function ($db, $i) use ($shards) { + static $next = 0; + $db->insert(GATE_DB, ['u' => 'new' . getmypid() . '-' . ($next++), 'g' => 'g0', 'n' => -2]); + }]; + } + return $w; +} + +function dbConfig(string $dir, string $fixture, array $extra): array { + $spec = fixtureSpecs()[$fixture]; + return array_merge(['secretKey' => GATE_SECRET, 'dbDir' => $dir, 'autoCreateDB' => true, 'lockTimeout' => 60], + $spec['config'], $extra); +} + +// ====================================================================== +// Child processes +// ====================================================================== + +function installErrorHandler(): void { + set_error_handler(function ($no, $str, $file, $line) { + if (!(error_reporting() & $no)) { + return false; // suppressed with @ by noneDB itself + } + throw new ErrorException($str, 0, $no, $file, $line); + }); +} + +function fixtureMain(array $args): int { + [$lib, $fixture, $dir, $extraJson] = $args; + installErrorHandler(); + require $lib; + $spec = fixtureSpecs()[$fixture]; + $db = new noneDB(dbConfig($dir, $fixture, json_decode($extraJson, true))); + $db->createDB(GATE_DB); + for ($start = 0; $start < $spec['records']; $start += 1000) { + $rows = []; + for ($n = $start; $n < min($spec['records'], $start + 1000); $n++) { + $rows[] = isset($spec['matrix']) ? makeMatrixRecord($n, $spec['matrix']) : makeRecord($n); + } + $result = $db->insert(GATE_DB, $rows); + if (($result['n'] ?? 0) !== count($rows)) { + fwrite(STDERR, 'fixture insert failed: ' . json_encode($result) . "\n"); + return 1; + } + } + if (!empty($spec['config']['bufferEnabled'])) { + $db->flush(GATE_DB); + } + if (isset($spec['supersede'])) { + $db->update(GATE_DB, [['c' => $spec['supersede']], ['set' => ['flag' => true]]]); + } + if (isset($spec['matrix'])) { + // also the 1-shard fixture is sharded (900 records < shardSize); the baseline names the method migrate() + method_exists($db, 'convertToSharded') ? $db->convertToSharded(GATE_DB) : $db->migrate(GATE_DB); + } + foreach ($spec['fields'] ?? [] as $field) { + $db->createFieldIndex(GATE_DB, $field); + } + foreach ($spec['spatial'] ?? [] as $field) { + $db->createSpatialIndex(GATE_DB, $field); + } + if ($db->count(GATE_DB) !== $spec['records'] + 0) { + fwrite(STDERR, "fixture count mismatch\n"); + return 1; + } + return 0; +} + +/** + * One measured run: prints {"unit": seconds, "ops": [seconds...], "mem": bytes} + */ +function childMain(array $args): int { + [$lib, $name, $dir, $seed, $extraJson] = $args; + $GLOBALS['gateDir'] = $dir; + installErrorHandler(); + mt_srand((int)$seed); + $w = workloads()[$name]; + $prepared = isset($w['prepare']) ? $w['prepare']() : null; + require $lib; + $config = dbConfig($dir, $w['fixture'], json_decode($extraJson, true)); + $ops = []; + + if ($w['kind'] === 'cold') { + $base = memory_get_usage(); + if (function_exists('memory_reset_peak_usage')) { + memory_reset_peak_usage(); + } + $t = hrtime(true); + $db = new noneDB($config); + $w['op']($db); + $unit = (hrtime(true) - $t) / 1e9; + $ops[] = $unit; + } else { + $db = new noneDB($config); + $db->find(GATE_DB, ['key' => 0]); // load the index (and lock handle) before measuring + if ($w['kind'] === 'warm') { + // 3 untimed operations; --warm-up=N for steady-state measurements (one-time costs excluded) + $warmUp = (int)(getenv('NONEDB_GATE_WARMUP') ?: 3); + for ($i = 0; $i < $warmUp; $i++) { + $w['op']($db, -1 - $i); + } + } + $base = memory_get_usage(); + if (function_exists('memory_reset_peak_usage')) { + memory_reset_peak_usage(); + } + if ($w['kind'] === 'warm') { + $unit = 0.0; + for ($i = 0; $i < $w['k']; $i++) { + $t = hrtime(true); + $w['op']($db, $i); + $d = (hrtime(true) - $t) / 1e9; + $ops[] = $d; + $unit += $d; + } + } else { + $t = hrtime(true); + $w['op']($db, $prepared); + $unit = (hrtime(true) - $t) / 1e9; + $ops[] = $unit; + } + } + $mem = memory_get_peak_usage() - $base; + // Keep at most 200 latencies per run (every m-th): enough for a pooled p95, cheap to bootstrap + if (count($ops) > 200) { + $step = (int)ceil(count($ops) / 200); + $kept = []; + for ($i = 0; $i < count($ops); $i += $step) { + $kept[] = $ops[$i]; + } + $ops = $kept; + } + echo json_encode(['unit' => $unit, 'ops' => $ops, 'mem' => $mem]), "\n"; + return 0; +} + +function concurrentChildMain(array $args): int { + [$lib, $name, $dir, $role, $k, $startAt, $seed, $extraJson] = $args; + installErrorHandler(); + mt_srand((int)$seed); + require $lib; + $w = workloads()[$name]; + $db = new noneDB(dbConfig($dir, $w['fixture'], json_decode($extraJson, true))); + $db->find(GATE_DB, ['key' => 0]); + $late = microtime(true) > (float)$startAt; + while (microtime(true) < (float)$startAt) { + usleep(200); + } + for ($i = 0; $i < (int)$k; $i++) { + if ($role === 'w') { + $db->insert(GATE_DB, makeRecord(300000 + (int)$seed * 1000 + $i)); + } else { + $db->find(GATE_DB, ['key' => mt_rand(0, 9999)]); + } + } + echo json_encode(['end' => microtime(true), 'late' => $late]), "\n"; + return 0; +} + +// ====================================================================== +// Parent: runs +// ====================================================================== + +function parseOptions(array $args): array { + $o = []; + foreach ($args as $a) { + if (preg_match('/^--([a-z-]+)(?:=(.*))?$/s', $a, $m)) { + $o[$m[1]] = $m[2] ?? true; + } + } + return $o; +} + +function spawn(array $cmd): array { + // Same limits for both libraries; bulk workloads need more than the default 128M + array_splice($cmd, 1, 0, ['-d', 'memory_limit=-1']); + $proc = proc_open($cmd, [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + if (!is_resource($proc)) { + throw new RuntimeException('proc_open failed'); + } + return [$proc, $pipes]; +} + +function collect(array $spawned, string $what): string { + [$proc, $pipes] = $spawned; + $out = stream_get_contents($pipes[1]); + $err = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($proc); + if ($status !== 0 || trim($err) !== '') { + throw new RuntimeException("$what failed (exit $status): " . trim($err . "\n" . $out)); + } + return $out; +} + +function runChild(array $cmd, string $what): array { + $out = collect(spawn($cmd), $what); + $lines = array_values(array_filter(explode("\n", trim($out)), 'strlen')); + $data = json_decode((string)end($lines), true); + if (!is_array($data)) { + throw new RuntimeException("$what: unexpected output: $out"); + } + return $data; +} + +/** + * Give the fixture database's lock state record a new epoch, as another process replacing a data file + * would (both storage layouts: "{gen}:{epoch}[ flags]" padded in place) + */ +function newLockEpoch(string $dir): void { + foreach (array_merge(glob($dir . '/*.ndb4/LOCK') ?: [], glob($dir . '/*.nonedb.lock') ?: []) as $lock) { + $fp = fopen($lock, 'c+'); + flock($fp, LOCK_EX); + $raw = (string)stream_get_contents($fp); + $record = preg_replace('/^(\d+):[0-9a-f]{8}/', '$1:' . bin2hex(random_bytes(4)), $raw); + fseek($fp, 0); + fwrite($fp, $record); + fflush($fp); + flock($fp, LOCK_UN); + fclose($fp); + } +} + +function removeTree(string $dir): void { + if (!is_dir($dir)) { + return; + } + $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::CHILD_FIRST); + foreach ($it as $f) { + $f->isDir() ? rmdir($f->getPathname()) : unlink($f->getPathname()); + } + rmdir($dir); +} + +function copyTree(string $from, string $to): void { + mkdir($to, 0700, true); + foreach (scandir($from) as $f) { + if ($f !== '.' && $f !== '..') { + is_dir($from . '/' . $f) ? copyTree($from . '/' . $f, $to . '/' . $f) : copy($from . '/' . $f, $to . '/' . $f); + } + } +} + +function env(): array { + $fs = trim((string)@shell_exec('df -P ' . escapeshellarg(sys_get_temp_dir()) . ' 2>/dev/null | tail -1')); + return [ + 'date' => date('c'), + 'php' => PHP_VERSION, + 'os' => php_uname(), + 'tmpFs' => $fs, + 'load' => function_exists('sys_getloadavg') ? sys_getloadavg() : null, + 'opcache' => (bool)ini_get('opcache.enable_cli'), + 'xdebug' => extension_loaded('xdebug'), + ]; +} + +function runMain(array $o): int { + foreach (['baseline', 'candidate', 'out'] as $required) { + if (!isset($o[$required]) || !is_string($o[$required])) { + fwrite(STDERR, "--$required is required\n"); + return 2; + } + } + $libs = ['A' => realpath($o['baseline']), 'B' => realpath($o['candidate'])]; + if (!$libs['A'] || !$libs['B']) { + fwrite(STDERR, "library not found\n"); + return 2; + } + $extra = ['A' => $o['baseline-config'] ?? '{}', 'B' => $o['candidate-config'] ?? '{}']; + foreach ($extra as $json) { + if (!is_array(json_decode($json, true))) { + fwrite(STDERR, "invalid --*-config JSON\n"); + return 2; + } + } + $quick = isset($o['quick']); + if (isset($o['warm-up'])) { + putenv('NONEDB_GATE_WARMUP=' . max(0, (int)$o['warm-up'])); // inherited by every child + } + $defaultRuns = $quick ? 6 : max(2, (int)($o['runs'] ?? 40)); + $work = rtrim($o['work'] ?? sys_get_temp_dir() . '/nonedb-perf-gate-' . bin2hex(random_bytes(4)), '/'); + @mkdir($work, 0700, true); + $aa = $libs['A'] === $libs['B'] && $extra['A'] === $extra['B']; + + $result = [ + 'meta' => env() + [ + 'mode' => $aa ? 'A/A' : 'A/B', + 'quick' => $quick, + 'warmUp' => (int)(getenv('NONEDB_GATE_WARMUP') ?: 3), + 'baseline' => ['path' => $libs['A'], 'sha256' => hash_file('sha256', $libs['A']), 'config' => json_decode($extra['A'], true)], + 'candidate' => ['path' => $libs['B'], 'sha256' => hash_file('sha256', $libs['B']), 'config' => json_decode($extra['B'], true)], + ], + 'workloads' => [], + ]; + $started = microtime(true); + + $fixtures = []; + $fixtureDir = function (string $variant, string $fixture) use (&$fixtures, $work, $libs, $extra) { + $key = $variant . '/' . $fixture; + if (!isset($fixtures[$key])) { + $dir = "$work/fixtures/$variant/$fixture"; + removeTree($dir); + mkdir($dir, 0700, true); + fwrite(STDERR, " building fixture $key\n"); + runChildRaw([PHP_BINARY, __FILE__, 'fixture', $libs[$variant], $fixture, $dir . '/', $extra[$variant]], "fixture $key"); + $fixtures[$key] = $dir; + } + return $fixtures[$key]; + }; + + $runCounter = 0; + foreach (workloads() as $name => $w) { + if (isset($o['only']) && !preg_match('/' . $o['only'] . '/', $name)) { + continue; + } + $runs = $quick ? $defaultRuns : max($defaultRuns, 0); + if (!$quick && isset($w['runs'])) { + $runs = min($runs, $w['runs']); + } + fwrite(STDERR, sprintf("[%s] %s (%d runs per variant)\n", date('H:i:s'), $name, $runs)); + $samples = ['A' => [], 'B' => []]; + for ($r = 0; $r < $runs; $r++) { + $order = $r % 2 === 0 ? ['A', 'B'] : ['B', 'A']; + foreach ($order as $variant) { + $template = $fixtureDir($variant, $w['fixture']); + if (!empty($w['inPlace'])) { + $runDir = $template; // big fixtures: the runs share it (each variant has its own) + } else { + $runDir = "$work/run-" . (++$runCounter); + copyTree($template, $runDir); + } + try { + $seed = 1000; // the same operations in every run: only the system varies + if ($w['kind'] === 'conc') { + $samples[$variant][] = concurrentRun($libs[$variant], $name, $w, $runDir . '/', $seed, $extra[$variant]); + } else { + $samples[$variant][] = runChild([PHP_BINARY, __FILE__, 'child', $libs[$variant], $name, $runDir . '/', (string)$seed, $extra[$variant]], "$name/$variant"); + } + } finally { + if (empty($w['inPlace'])) { + removeTree($runDir); + } + } + } + } + $result['workloads'][$name] = ['kind' => $w['kind'], 'fixture' => $w['fixture'], 'runs' => $runs] + + analyse($samples, $w['kind'], $aa, $quick); + $s = $result['workloads'][$name]; + fwrite(STDERR, sprintf(" median ratio %.3f [%.3f, %.3f] p=%.4f status=%s\n", + $s['median']['ratio'], $s['median']['ci'][0], $s['median']['ci'][1], $s['median']['p'], $s['status'])); + // Partial results survive an interrupted run + file_put_contents($o['out'] . '.partial', json_encode($result, JSON_PRETTY_PRINT | JSON_PRESERVE_ZERO_FRACTION) . "\n"); + } + @unlink($o['out'] . '.partial'); + removeTree("$work/fixtures"); + @rmdir($work); + $result['meta']['seconds'] = round(microtime(true) - $started, 1); + $result['meta']['loadEnd'] = function_exists('sys_getloadavg') ? sys_getloadavg() : null; + + file_put_contents($o['out'], json_encode($result, JSON_PRETTY_PRINT | JSON_PRESERVE_ZERO_FRACTION) . "\n"); + $md = markdownReport($result); + if (isset($o['md']) && is_string($o['md'])) { + file_put_contents($o['md'], $md); + } + echo $md; + return 0; +} + +function runChildRaw(array $cmd, string $what): void { + collect(spawn($cmd), $what); +} + +function concurrentRun(string $lib, string $name, array $w, string $dir, int $seed, string $extra): array { + for ($attempt = 0; $attempt < 3; $attempt++) { + $startAt = microtime(true) + 1.0 + $attempt; + $children = []; + $i = 0; + foreach ($w['roles'] as $role => $count) { + for ($c = 0; $c < $count; $c++) { + $children[] = spawn([PHP_BINARY, __FILE__, 'conc', $lib, $name, $dir, $role, (string)$w['k'][$role], + sprintf('%.6f', $startAt), (string)($seed * 100 + $i++), $extra]); + } + } + $end = 0.0; + $late = false; + foreach ($children as $child) { + $out = collect($child, "$name child"); + $data = json_decode(trim($out), true); + $end = max($end, (float)$data['end']); + $late = $late || $data['late']; + } + if (!$late) { + return ['unit' => $end - $startAt, 'ops' => [], 'mem' => 0]; + } + } + throw new RuntimeException("$name: children were not ready at the common start time"); +} + +// ====================================================================== +// Statistics +// ====================================================================== + +function quantile(array $sorted, float $p): float { + $n = count($sorted); + if ($n === 0) { + return NAN; + } + $h = ($n - 1) * $p; + $lo = (int)floor($h); + $hi = min($n - 1, $lo + 1); + return $sorted[$lo] + ($h - $lo) * ($sorted[$hi] - $sorted[$lo]); +} + +function median(array $values): float { + sort($values); + return quantile($values, 0.5); +} + +function p95(array $values): float { + sort($values); + return quantile($values, 0.95); +} + +function erfcApprox(float $x): float { + // Numerical Recipes erfcc, fractional error < 1.2e-7 + $z = abs($x); + $t = 1.0 / (1.0 + 0.5 * $z); + $r = $t * exp(-$z * $z - 1.26551223 + $t * (1.00002368 + $t * (0.37409196 + $t * (0.09678418 + $t * (-0.18628806 + + $t * (0.27886807 + $t * (-1.13520398 + $t * (1.48851587 + $t * (-0.82215223 + $t * 0.17087277))))))))); + return $x >= 0 ? $r : 2.0 - $r; +} + +function normalCdf(float $z): float { + return 0.5 * erfcApprox(-$z / M_SQRT2); +} + +/** + * One-sided Mann-Whitney U test, H1: values of $b tend to be larger than values of $a + * (normal approximation with tie and continuity correction) + */ +function mannWhitneyGreater(array $a, array $b): float { + $nA = count($a); + $nB = count($b); + $n = $nA + $nB; + if ($nA < 2 || $nB < 2) { + return 1.0; + } + $all = []; + foreach ($a as $v) { + $all[] = [$v, 0]; + } + foreach ($b as $v) { + $all[] = [$v, 1]; + } + usort($all, function ($x, $y) { return $x[0] <=> $y[0]; }); + $rankB = 0.0; + $ties = 0.0; + for ($i = 0; $i < $n;) { + $j = $i; + while ($j + 1 < $n && $all[$j + 1][0] == $all[$i][0]) { + $j++; + } + $rank = ($i + $j) / 2 + 1; + $t = $j - $i + 1; + $ties += $t * $t * $t - $t; + for ($k = $i; $k <= $j; $k++) { + if ($all[$k][1] === 1) { + $rankB += $rank; + } + } + $i = $j + 1; + } + $u = $rankB - $nB * ($nB + 1) / 2; + $var = $nA * $nB / 12 * (($n + 1) - $ties / ($n * ($n - 1))); + if ($var <= 0) { + return 1.0; + } + return 1.0 - normalCdf(($u - $nA * $nB / 2 - 0.5) / sqrt($var)); +} + +function hodgesLehmann(array $a, array $b): float { + $d = []; + foreach ($a as $x) { + foreach ($b as $y) { + $d[] = $y - $x; + } + } + return median($d); +} + +/** + * Percentile bootstrap CI (99%) of stat(B) / stat(A); groups are resampled independently. + * $a and $b hold whatever $stat takes (samples, or runs of latencies for a cluster bootstrap). + */ +function bootstrapRatio(array $a, array $b, callable $stat, int $iterations, int $seed): array { + mt_srand($seed); + $nA = count($a); + $nB = count($b); + $ratios = []; + for ($it = 0; $it < $iterations; $it++) { + $ra = []; + for ($i = 0; $i < $nA; $i++) { + $ra[] = $a[mt_rand(0, $nA - 1)]; + } + $rb = []; + for ($i = 0; $i < $nB; $i++) { + $rb[] = $b[mt_rand(0, $nB - 1)]; + } + $sa = $stat($ra); + $ratios[] = $sa > 0 ? $stat($rb) / $sa : INF; + } + sort($ratios); + return [quantile($ratios, 0.005), quantile($ratios, 0.995)]; +} + +function analyse(array $samples, string $kind, bool $aa, bool $quick): array { + $unitA = array_column($samples['A'], 'unit'); + $unitB = array_column($samples['B'], 'unit'); + $medA = median($unitA); + $medB = median($unitB); + $ratio = $medA > 0 ? $medB / $medA : INF; + $ci = bootstrapRatio($unitA, $unitB, 'median', 2000, 7); + $pSlower = mannWhitneyGreater($unitA, $unitB); + $pFaster = mannWhitneyGreater($unitB, $unitA); + $out = [ + 'median' => [ + 'A' => $medA, 'B' => $medB, 'ratio' => $ratio, 'ci' => $ci, 'p' => $pSlower, 'pFaster' => $pFaster, + 'hodgesLehmann' => $medA > 0 ? hodgesLehmann($unitA, $unitB) / $medA : NAN, + 'minA' => min($unitA), 'minB' => min($unitB), + ], + ]; + + // p95 of single-operation latencies, pooled over runs (cluster bootstrap over runs) + $opsA = array_column($samples['A'], 'ops'); + $opsB = array_column($samples['B'], 'ops'); + $pooledA = $opsA ? array_merge(...$opsA) : []; + $pooledB = $opsB ? array_merge(...$opsB) : []; + $p95Decides = $kind === 'warm' && count($pooledA) >= 100 && count($pooledB) >= 100; + if ($p95Decides) { + $pool = function (array $runs) { return p95(array_merge(...$runs)); }; + $p95A = p95($pooledA); + $out['p95'] = ['A' => $p95A, 'B' => p95($pooledB), 'ratio' => $p95A > 0 ? p95($pooledB) / $p95A : INF, + 'ci' => bootstrapRatio($opsA, $opsB, $pool, 1000, 11), 'samples' => [count($pooledA), count($pooledB)]]; + } + + // Peak memory of the workload (deterministic: median of the runs) + $memA = median(array_column($samples['A'], 'mem')); + $memB = median(array_column($samples['B'], 'mem')); + $out['memory'] = ['A' => $memA, 'B' => $memB, 'ratio' => $memA > 0 ? $memB / $memA : ($memB > 0 ? INF : 1.0)]; + + // Decision + $significant = $pSlower < 0.01 && $ci[0] > 1.0; + $regressions = []; + if ($kind === 'conc') { + // unit is wall time: throughput ratio = 1 / time ratio; regression when its CI is below 0.95 + if ($significant && $ci[0] > 1 / 0.95) { + $regressions[] = 'throughput'; + } + } elseif ($significant && $ratio > 1.05) { + $regressions[] = 'median'; + } + if (isset($out['p95']) && $out['p95']['ci'][0] > 1.0 && $out['p95']['ratio'] > 1.05) { + $regressions[] = 'p95'; + } + if ($kind !== 'conc' && $out['memory']['ratio'] > 1.05 && $memB - $memA > 64 * 1024) { + $regressions[] = 'memory'; + } + if ($regressions) { + $status = 'regression'; + } elseif ($significant) { + $status = 'finding'; + } elseif ($pFaster < 0.01 && $ci[1] < 1.0) { + $status = 'improvement'; + } else { + $status = 'neutral'; + } + if ($quick) { + $status .= ' (quick: not a decision)'; + } + $out['status'] = $status; + $out['regressions'] = $regressions; + if ($aa) { + $width = max(1 - $ci[0], $ci[1] - 1); + $out['noise'] = $width; + $out['stable'] = $width <= 0.02; + } + return $out; +} + +// ====================================================================== +// Reports +// ====================================================================== + +function fmtCi(array $ci): string { + return sprintf('[%.3f, %.3f]', $ci[0], $ci[1]); +} + +function markdownReport(array $r): string { + $m = $r['meta']; + $md = "# noneDB performance gate ({$m['mode']})\n\n"; + $md .= "- date: {$m['date']}, PHP {$m['php']}, " . ($m['quick'] ? 'QUICK (no decisions)' : 'full') . ", warm-up " . ($m['warmUp'] ?? 3) . ", {$m['seconds']} s\n"; + $md .= "- os: {$m['os']}\n- tmp fs: {$m['tmpFs']}\n"; + $md .= '- load at start / end: ' . json_encode($m['load']) . ' / ' . json_encode($m['loadEnd']) . "\n"; + $md .= "- baseline: {$m['baseline']['path']} (sha256 " . substr($m['baseline']['sha256'], 0, 12) . ') config ' . json_encode($m['baseline']['config']) . "\n"; + $md .= "- candidate: {$m['candidate']['path']} (sha256 " . substr($m['candidate']['sha256'], 0, 12) . ') config ' . json_encode($m['candidate']['config']) . "\n\n"; + $md .= "Ratio = candidate / baseline (> 1 slower). CI = 99% bootstrap; p = one-sided Mann-Whitney (candidate slower).\n\n"; + $md .= "| workload | runs | median A (ms) | median B (ms) | ratio [99% CI] | p | HL shift | p95 ratio [99% CI] | memory ratio | status |\n"; + $md .= "|---|---|---|---|---|---|---|---|---|---|\n"; + foreach ($r['workloads'] as $name => $s) { + $p95 = isset($s['p95']) ? sprintf('%.3f %s', $s['p95']['ratio'], fmtCi($s['p95']['ci'])) : '-'; + $status = $s['status'] . ($s['regressions'] ? ' (' . implode(', ', $s['regressions']) . ')' : ''); + if (isset($s['stable'])) { + $status .= sprintf(' noise ±%.1f%%%s', $s['noise'] * 100, $s['stable'] ? '' : ' UNSTABLE'); + } + $md .= sprintf("| %s | %d | %.3f | %.3f | %.3f %s | %.4f | %+.1f%% | %s | %.3f | %s |\n", + $name, $s['runs'], $s['median']['A'] * 1000, $s['median']['B'] * 1000, $s['median']['ratio'], fmtCi($s['median']['ci']), + $s['median']['p'], $s['median']['hodgesLehmann'] * 100, $p95, $s['memory']['ratio'], $status); + } + return $md; +} + +/** + * Absolute budgets for the fixed costs of the fail-closed store (user's decision, 2026-10-28): seconds a + * workload may add per operation before its time counts as a regression. Every cold workload (a new instance and its first operation) has the cold-open budget. + */ +function timeBudget(string $name, array $workload): ?float { + $budgets = ['find_missing_db_warm' => 0.0001, 'find_key_after_epoch_change' => 0.0001]; + if (isset($budgets[$name])) { + return $budgets[$name]; + } + return ($workload['kind'] ?? null) === 'cold' ? 0.00025 : null; +} + +/** Median seconds a run of a workload adds per operation (warm: a run is k operations) */ +function addedPerOperation(array $result, array $workload): float { + $k = ($workload['kind'] ?? null) === 'warm' ? (int)$workload['k'] : 1; + return ($result['median']['B'] - $result['median']['A']) / max(1, $k); +} + +function verifyMain(array $args): int { + $o = parseOptions($args); + $files = array_values(array_filter($args, function ($a) { return strncmp($a, '--', 2) !== 0; })); + if (count($files) !== 2) { + fwrite(STDERR, "verify needs two run files\n"); + return 2; + } + $runs = []; + foreach ($files as $f) { + $runs[] = json_decode((string)file_get_contents($f), true); + if (!is_array(end($runs))) { + fwrite(STDERR, "cannot read $f\n"); + return 2; + } + } + if ($runs[0]['meta']['candidate']['sha256'] !== $runs[1]['meta']['candidate']['sha256'] + || $runs[0]['meta']['baseline']['sha256'] !== $runs[1]['meta']['baseline']['sha256']) { + fwrite(STDERR, "the two runs compare different libraries\n"); + return 2; + } + $md = "# noneDB performance gate verdict\n\n- first: {$files[0]} ({$runs[0]['meta']['date']})\n- second: {$files[1]} ({$runs[1]['meta']['date']})\n\n"; + $md .= "| workload | first | second | verdict |\n|---|---|---|---|\n"; + $verified = 0; + foreach ($runs[0]['workloads'] as $name => $first) { + $second = $runs[1]['workloads'][$name] ?? null; + if ($second === null) { + $md .= "| $name | {$first['status']} | (missing) | not compared |\n"; + continue; + } + $both = array_values(array_intersect($first['regressions'], $second['regressions'])); + $definition = workloads()[$name] ?? []; + $budget = timeBudget($name, $definition); + $timeOnly = array_values(array_diff($both, ['median', 'p95'])); + if ($both && $budget !== null && $timeOnly === [] + && max(addedPerOperation($first, $definition), addedPerOperation($second, $definition)) <= $budget) { + // The time regression is a fixed cost within its absolute budget (both runs) + $verdict = sprintf('within budget (+%.3f ms ≤ %.2f ms per operation)', 1000 * max(addedPerOperation($first, $definition), + addedPerOperation($second, $definition)), 1000 * $budget); + } elseif ($both) { + $verdict = 'VERIFIED REGRESSION (' . implode(', ', $both) . ')'; + $verified++; + } elseif (in_array($first['status'], ['finding', 'regression'], true) && in_array($second['status'], ['finding', 'regression'], true)) { + $verdict = 'finding to investigate'; + } elseif (strpos($first['status'], 'improvement') === 0 && strpos($second['status'], 'improvement') === 0) { + $verdict = 'improvement'; + } else { + $verdict = 'no verified change'; + } + $md .= sprintf("| %s | %s %.3f | %s %.3f | %s |\n", $name, $first['status'], $first['median']['ratio'], + $second['status'], $second['median']['ratio'], $verdict); + } + $md .= "\nVerified regressions: $verified\n"; + if (isset($o['md']) && is_string($o['md'])) { + file_put_contents($o['md'], $md); + } + echo $md; + return $verified > 0 ? 1 : 0; +} diff --git a/tests/performance_benchmark.php b/tests/Performance/performance_benchmark.php similarity index 95% rename from tests/performance_benchmark.php rename to tests/Performance/performance_benchmark.php index 4999d8d..2dcda54 100644 --- a/tests/performance_benchmark.php +++ b/tests/Performance/performance_benchmark.php @@ -4,7 +4,7 @@ * Tests all operations from 100 to 500K records */ -require_once __DIR__ . '/../noneDB.php'; +require_once __DIR__ . '/../../noneDB.php'; ini_set('memory_limit', '-1'); set_time_limit(0); @@ -16,7 +16,7 @@ } $db = new noneDB(['secretKey' => 'nonedb-benchmark-key-not-for-production', 'dbDir' => $benchDir]); -// Test sizes (override: php tests/performance_benchmark.php 100,1000) +// Test sizes (override: php tests/Performance/performance_benchmark.php 100,1000) $sizes = isset($argv[1]) ? array_map('intval', explode(',', $argv[1])) : [100, 1000, 10000, 50000, 100000, 500000]; // Colors @@ -53,8 +53,8 @@ function generateRecord($i) { } echo blue("╔════════════════════════════════════════════════════════════════════╗\n"); -echo blue("║ noneDB Performance Benchmark v3.2 ║\n"); -echo blue("║ JSONL Engine + Static Cache + Batch Read + Single-Pass Filter ║\n"); +echo blue("║ noneDB Performance Benchmark ║\n"); +echo blue("║ JSONL storage + index checkpoints + query planner ║\n"); echo blue("╚════════════════════════════════════════════════════════════════════╝\n\n"); echo "PHP Version: " . PHP_VERSION . "\n"; @@ -325,7 +325,7 @@ function printTable($title, $data, $sizes) { printTable("Write Operations", $results['write'], $sizes); printTable("Read Operations", $results['read'], $sizes); printTable("Query & Aggregation", $results['query'], $sizes); -printTable("Method Chaining (v2.1+)", $results['chain'], $sizes); +printTable("Method Chaining", $results['chain'], $sizes); echo "### Key Lookup\n| Records | Cold (first access) | Warm (cached) |\n|---------|-----|-----|\n"; foreach ($sizes as $s) { @@ -337,4 +337,13 @@ function printTable($title, $data, $sizes) { } echo "\n"; +// Remove the benchmark's own directory (dot files such as the hash cache too) +\noneDB::clearStaticCache(); +foreach (array_diff(scandir($benchDir) ?: [], ['.', '..']) as $leftover) { + if (is_file($benchDir . $leftover)) { + unlink($benchDir . $leftover); + } +} +@rmdir($benchDir); + echo green("\nBenchmark completed!\n"); diff --git a/tests/sleekdb_comparison.php b/tests/Performance/sleekdb_comparison.php similarity index 95% rename from tests/sleekdb_comparison.php rename to tests/Performance/sleekdb_comparison.php index 359768b..98c0947 100644 --- a/tests/sleekdb_comparison.php +++ b/tests/Performance/sleekdb_comparison.php @@ -4,15 +4,15 @@ * Tests all operations from 100 to 100K records */ -require_once __DIR__ . '/../noneDB.php'; -require_once __DIR__ . '/../vendor/autoload.php'; +require_once __DIR__ . '/../../noneDB.php'; +require_once __DIR__ . '/../../vendor/autoload.php'; use SleekDB\Store; ini_set('memory_limit', '-1'); set_time_limit(0); -// Test sizes (override: php tests/sleekdb_comparison.php 100,1000) +// Test sizes (override: php tests/Performance/sleekdb_comparison.php 100,1000) $sizes = isset($argv[1]) ? array_map('intval', explode(',', $argv[1])) : [100, 1000, 10000, 50000, 100000]; // Own directory and key: the benchmark wipes its directory, never your databases @@ -67,11 +67,11 @@ function ratio($nonedb, $sleekdb) { } echo blue("╔══════════════════════════════════════════════════════════════════════════╗\n"); -echo blue("║ noneDB v3.2 vs SleekDB Comprehensive Benchmark ║\n"); +echo blue("║ " . str_pad('noneDB ' . noneDB::VERSION . ' vs SleekDB Comprehensive Benchmark', 63) . "║\n"); echo blue("╚══════════════════════════════════════════════════════════════════════════╝\n\n"); echo "PHP Version: " . PHP_VERSION . "\n"; -echo "noneDB: v3.2.0 (JSONL + database locks + static cache)\n"; +echo "noneDB: " . noneDB::VERSION . " (storage format " . noneDB::STORAGE_FORMAT . ")\n"; echo "SleekDB: v2.x\n\n"; $results = []; @@ -382,4 +382,13 @@ function ratio($nonedb, $sleekdb) { echo "\n"; } +// Remove the benchmark's own directory (dot files such as the hash cache too) +\noneDB::clearStaticCache(); +foreach (array_diff(scandir($benchDir) ?: [], ['.', '..']) as $leftover) { + if (is_file($benchDir . $leftover)) { + unlink($benchDir . $leftover); + } +} +@rmdir($benchDir); + echo green("\n\nBenchmark completed!\n"); diff --git a/tests/spatial_benchmark.php b/tests/Performance/spatial_benchmark.php similarity index 81% rename from tests/spatial_benchmark.php rename to tests/Performance/spatial_benchmark.php index 9130738..d7b5d86 100644 --- a/tests/spatial_benchmark.php +++ b/tests/Performance/spatial_benchmark.php @@ -1,15 +1,15 @@ 5, 'sync' => 'off', 'markdown' => false, 'child' => null]; +$sizes = [1000, 10000, 100000]; +foreach ($args as $arg) { + if (preg_match('/^--runs=(\d+)$/', $arg, $m)) { + $options['runs'] = max(1, (int)$m[1]); + } elseif (preg_match('/^--sync=(off|normal|full)$/', $arg, $m)) { + $options['sync'] = $m[1]; + } elseif ($arg === '--markdown') { + $options['markdown'] = true; + } elseif (preg_match('/^--child=(.+)$/', $arg, $m)) { + $options['child'] = $m[1]; + } elseif (preg_match('/^[\d,]+$/', $arg)) { + $sizes = array_map('intval', explode(',', $arg)); + } +} + +function record(int $i): array +{ + $cities = ['Istanbul', 'Ankara', 'Izmir', 'Bursa', 'Antalya']; + return ['name' => 'User' . $i, 'email' => "user{$i}@test.com", 'age' => 20 + ($i % 50), 'salary' => 5000 + ($i % 10000), + 'city' => $cities[$i % 5], 'active' => $i % 3 !== 0, 'uid' => $i % 977]; +} + +/** Workloads: name => [kind, callable per engine] */ +interface Engine +{ + public function setUp(int $size): void; + public function run(string $workload, int $size, int $i): array; // [count, first key] +} + +final class NoneDBEngine implements Engine +{ + private $db; + private $indexed; + + public function __construct(string $dir, bool $indexed) + { + require_once __DIR__ . '/../../noneDB.php'; + $this->indexed = $indexed; + $this->db = new noneDB(['secretKey' => 'sqlite_comparison_' . bin2hex(random_bytes(4)), 'dbDir' => $dir, 'fieldIndexEnabled' => $indexed]); + } + + public function setUp(int $size): void + { + $this->db->createFieldIndex('bench', 'city'); + $this->db->createFieldIndex('bench', 'age'); + $this->db->createFieldIndex('bench', 'email'); + $rows = []; + for ($i = 0; $i < $size; $i++) { + $rows[] = record($i); + } + $this->db->insert('bench', $rows); + $this->db->insert('owners', array_map(function ($u) { return ['uid' => $u, 'owner' => 'O' . $u]; }, range(0, 976))); + $this->db->createFieldIndex('owners', 'uid'); + } + + private static function out(array $rows): array + { + return [count($rows), $rows[0]['key'] ?? null]; + } + + public function run(string $workload, int $size, int $i): array + { + $db = $this->db; + switch ($workload) { + case 'find_key': + return self::out($db->find('bench', ['key' => ($i * 7919) % $size])); + case 'find_eq': + return self::out($db->find('bench', ['city' => 'Izmir'])); + case 'find_unique': + return self::out($db->find('bench', ['email' => 'user' . (($i * 31) % $size) . '@test.com'])); + case 'range': + return self::out($db->between('bench', 'age', 30, 33)); + case 'count_filter': + return [$db->count('bench', ['city' => 'Ankara']), null]; + case 'first': + $r = $db->query('bench')->where(['city' => 'Bursa'])->first(); + return [$r === null ? 0 : 1, $r['key'] ?? null]; + case 'exists': + return [(int)$db->query('bench')->where(['email' => 'user5@test.com'])->exists(), null]; + case 'sort_limit': + return self::out($db->query('bench')->where(['city' => 'Izmir'])->sort('salary', 'desc')->limit(10)->get()); + case 'distinct': + return [count($db->distinct('bench', 'uid')), null]; + case 'sum_filter': + return [(int)$db->sum('bench', 'salary', ['city' => 'Antalya']), null]; + case 'join_limit': + return self::out($db->query('bench')->where(['city' => 'Izmir'])->join('owners', 'uid', 'uid', 'o')->limit(50)->get()); + case 'insert_500': + for ($k = 0; $k < 500; $k++) { + $db->insert('bench', record($size + $i * 500 + $k)); + } + return [500, null]; + case 'update_300': + for ($k = 0; $k < 300; $k++) { + $db->update('bench', [['key' => ($k * 7919 + $i) % $size], ['set' => ['salary' => $k]]]); + } + return [300, null]; + } + throw new RuntimeException($workload); + } +} + +final class SQLiteEngine implements Engine +{ + private $pdo; + private $columns; + + public function __construct(string $file, bool $columns, string $sync) + { + $this->columns = $columns; + $this->pdo = new PDO('sqlite:' . $file, null, null, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $this->pdo->exec('PRAGMA journal_mode = WAL'); + $this->pdo->exec('PRAGMA synchronous = ' . strtoupper($sync)); + } + + public function setUp(int $size): void + { + if ($this->columns) { + $this->pdo->exec('CREATE TABLE bench (id INTEGER PRIMARY KEY, name TEXT, email TEXT, age INT, salary INT, city TEXT, active INT, uid INT)'); + $this->pdo->exec('CREATE INDEX b_city ON bench(city)'); + $this->pdo->exec('CREATE INDEX b_age ON bench(age)'); + $this->pdo->exec('CREATE INDEX b_email ON bench(email)'); + $insert = $this->pdo->prepare('INSERT INTO bench VALUES (?, ?, ?, ?, ?, ?, ?, ?)'); + } else { + $this->pdo->exec('CREATE TABLE bench (id INTEGER PRIMARY KEY, body TEXT)'); + $this->pdo->exec("CREATE INDEX b_city ON bench(json_extract(body, '$.city'))"); + $this->pdo->exec("CREATE INDEX b_age ON bench(json_extract(body, '$.age'))"); + $this->pdo->exec("CREATE INDEX b_email ON bench(json_extract(body, '$.email'))"); + $insert = $this->pdo->prepare('INSERT INTO bench VALUES (?, ?)'); + } + $this->pdo->exec('CREATE TABLE owners (uid INT, owner TEXT)'); + $this->pdo->exec('CREATE INDEX o_uid ON owners(uid)'); + $this->pdo->beginTransaction(); + for ($i = 0; $i < $size; $i++) { + $r = record($i); + $insert->execute($this->columns ? [$i, $r['name'], $r['email'], $r['age'], $r['salary'], $r['city'], (int)$r['active'], $r['uid']] + : [$i, json_encode($r)]); + } + $owner = $this->pdo->prepare('INSERT INTO owners VALUES (?, ?)'); + for ($u = 0; $u < 977; $u++) { + $owner->execute([$u, 'O' . $u]); + } + $this->pdo->commit(); + } + + private function col(string $field): string + { + // "b.uid": a column of table alias b + $alias = ''; + if (strpos($field, '.') !== false) { + [$alias, $field] = explode('.', $field, 2); + $alias .= '.'; + } + return $this->columns ? $alias . $field : "json_extract({$alias}body, '$.{$field}')"; + } + + private function query(string $sql, array $params = []): array + { + $statement = $this->pdo->prepare($sql); + $statement->execute($params); + $rows = $statement->fetchAll(PDO::FETCH_NUM); + return [count($rows), isset($rows[0][0]) ? (int)$rows[0][0] : null]; + } + + public function run(string $workload, int $size, int $i): array + { + $all = $this->columns ? '*' : 'id, body'; + switch ($workload) { + case 'find_key': + return $this->query("SELECT $all FROM bench WHERE id = ?", [($i * 7919) % $size]); + case 'find_eq': + return $this->query("SELECT $all FROM bench WHERE {$this->col('city')} = ? ORDER BY id", ['Izmir']); + case 'find_unique': + return $this->query("SELECT $all FROM bench WHERE {$this->col('email')} = ?", ['user' . (($i * 31) % $size) . '@test.com']); + case 'range': + return $this->query("SELECT $all FROM bench WHERE {$this->col('age')} BETWEEN 30 AND 33 ORDER BY id"); + case 'count_filter': + $r = $this->pdo->query("SELECT COUNT(*) FROM bench WHERE {$this->col('city')} = 'Ankara'")->fetchColumn(); + return [(int)$r, null]; + case 'first': + return $this->query("SELECT $all FROM bench WHERE {$this->col('city')} = 'Bursa' ORDER BY id LIMIT 1"); + case 'exists': + $r = $this->pdo->query("SELECT EXISTS(SELECT 1 FROM bench WHERE {$this->col('email')} = 'user5@test.com')")->fetchColumn(); + return [(int)$r, null]; + case 'sort_limit': + return $this->query("SELECT $all FROM bench WHERE {$this->col('city')} = 'Izmir' ORDER BY {$this->col('salary')} DESC, id LIMIT 10"); + case 'distinct': + return [(int)$this->pdo->query("SELECT COUNT(DISTINCT {$this->col('uid')}) FROM bench")->fetchColumn(), null]; + case 'sum_filter': + return [(int)$this->pdo->query("SELECT SUM({$this->col('salary')}) FROM bench WHERE {$this->col('city')} = 'Antalya'")->fetchColumn(), null]; + case 'join_limit': + return $this->query("SELECT b.id, o.owner FROM bench b LEFT JOIN owners o ON o.uid = {$this->col('b.uid')} WHERE {$this->col('b.city')} = 'Izmir' ORDER BY b.id LIMIT 50"); + case 'insert_500': + $insert = $this->columns ? $this->pdo->prepare('INSERT INTO bench VALUES (?, ?, ?, ?, ?, ?, ?, ?)') : $this->pdo->prepare('INSERT INTO bench VALUES (?, ?)'); + for ($k = 0; $k < 500; $k++) { + $id = $size + $i * 500 + $k; + $r = record($id); + $insert->execute($this->columns ? [$id, $r['name'], $r['email'], $r['age'], $r['salary'], $r['city'], (int)$r['active'], $r['uid']] : [$id, json_encode($r)]); + } + return [500, null]; + case 'update_300': + $update = $this->columns ? $this->pdo->prepare('UPDATE bench SET salary = ? WHERE id = ?') + : $this->pdo->prepare("UPDATE bench SET body = json_set(body, '$.salary', ?) WHERE id = ?"); + for ($k = 0; $k < 300; $k++) { + $update->execute([$k, ($k * 7919 + $i) % $size]); + } + return [300, null]; + } + throw new RuntimeException($workload); + } +} + +const WORKLOADS = ['find_key', 'find_eq', 'find_unique', 'range', 'count_filter', 'first', 'exists', 'sort_limit', 'distinct', + 'sum_filter', 'join_limit', 'insert_500', 'update_300']; +const ENGINES = ['noneDB-indexed', 'noneDB-scan', 'SQLite-doc', 'SQLite-columns']; + +/** Child process: one engine, one size, one run. Prints JSON [workload => [ms, count, firstKey]] */ +function child(string $spec, string $sync): void +{ + [$engineName, $size, $dir] = explode('|', $spec); + $size = (int)$size; + $engine = strpos($engineName, 'noneDB') === 0 + ? new NoneDBEngine($dir, $engineName === 'noneDB-indexed') + : new SQLiteEngine($dir . '/bench.sqlite', $engineName === 'SQLite-columns', $sync); + $engine->setUp($size); + $results = []; + foreach (WORKLOADS as $workload) { + $repeat = in_array($workload, ['find_key', 'find_unique', 'exists', 'first'], true) ? 20 : 1; + foreach ($repeat === 1 ? [0] : range(1, $repeat) as $i) { + $engine->run($workload, $size, $i); // warm: the calls measured below (a write workload: a call of its own) + } + $start = hrtime(true); + $out = [0, null]; + for ($i = 1; $i <= $repeat; $i++) { + $out = $engine->run($workload, $size, $i); + } + $results[$workload] = [(hrtime(true) - $start) / 1e6 / $repeat, $out[0], $out[1]]; + } + echo json_encode($results); +} + +if ($options['child'] !== null) { + child($options['child'], $options['sync']); + exit(0); +} + +function percentile(array $values, float $p): float +{ + sort($values); + $index = ($p / 100) * (count($values) - 1); + $lo = (int)floor($index); + $hi = (int)ceil($index); + return $values[$lo] + ($values[$hi] - $values[$lo]) * ($index - $lo); +} + +function removeDir(string $dir): void +{ + foreach (glob($dir . '/{,.}*', GLOB_BRACE) ?: [] as $file) { + if (basename($file) === '.' || basename($file) === '..') { + continue; + } + is_dir($file) ? removeDir($file) : unlink($file); + } + @rmdir($dir); +} + +echo "noneDB vs SQLite " . SQLite3::version()['versionString'] . " — PHP " . PHP_VERSION . ", " . PHP_OS_FAMILY + . ", runs {$options['runs']}, SQLite synchronous {$options['sync']}, noneDB durability performance\n"; +foreach ($sizes as $size) { + $samples = []; + foreach (ENGINES as $engineName) { + for ($run = 0; $run < $options['runs']; $run++) { + $dir = sys_get_temp_dir() . '/nonedb_sqlite_' . bin2hex(random_bytes(4)); + mkdir($dir); + // Children get this process's memory_limit (php -d ... isn't inherited) + $command = [PHP_BINARY, '-d', 'memory_limit=' . ini_get('memory_limit'), __FILE__, "--child={$engineName}|{$size}|{$dir}", "--sync={$options['sync']}"]; + $process = proc_open($command, [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $output = stream_get_contents($pipes[1]); + $errors = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + removeDir($dir); + $data = json_decode((string)$output, true); + if (!is_array($data)) { + fwrite(STDERR, "$engineName failed: " . trim($errors . "\n" . $output) . "\n"); + continue 2; + } + foreach ($data as $workload => $row) { + $samples[$workload][$engineName][] = $row; + } + } + } + echo $options['markdown'] ? "\n### {$size} records (ms per operation: median / min / p95)\n\n| Workload | " . implode(' | ', ENGINES) . " | Same result |\n|---|" . str_repeat('---|', count(ENGINES) + 1) . "\n" + : "\n{$size} records (ms per operation: median / min / p95)\n"; + foreach (WORKLOADS as $workload) { + $cells = []; + $results = []; + foreach (ENGINES as $engineName) { + $rows = $samples[$workload][$engineName] ?? []; + if (empty($rows)) { + $cells[] = '-'; + continue; + } + $times = array_column($rows, 0); + $cells[] = sprintf('%.3f / %.3f / %.3f', percentile($times, 50), min($times), percentile($times, 95)); + $results[$engineName] = [$rows[0][1], $rows[0][2]]; + } + $same = count(array_unique(array_map('json_encode', $results))) <= 1 ? 'yes' : 'semantics differ: ' . json_encode($results); + echo $options['markdown'] ? "| {$workload} | " . implode(' | ', $cells) . " | {$same} |\n" + : sprintf(" %-13s %s (%s)\n", $workload, implode(' ', array_map(function ($e, $c) { return "$e $c"; }, ENGINES, $cells)), $same); + } +} diff --git a/tests/Regression/LockStateRegressionTest.php b/tests/Regression/LockStateRegressionTest.php deleted file mode 100644 index 5d7b76f..0000000 --- a/tests/Regression/LockStateRegressionTest.php +++ /dev/null @@ -1,193 +0,0 @@ -db(); - $db->insert('st', ['a' => 1]); - $state = $this->lockState('st'); - $this->assertSame(['v'], $state['flags'], 'a new database is verified, nothing pending'); - $this->assertSame(128, filesize($this->dbBasePath('st') . '.nonedb.lock'), 'fixed size record'); - - // Buffered records: flag set while they wait, cleared once they are flushed - $buffered = $this->db(['bufferEnabled' => true]); - $buffered->insert('st', ['a' => 2]); - $this->assertContains('b', $this->lockState('st')['flags']); - $this->assertSame([1, 2], array_column($this->db()->find('st', 0), 'a'), 'a reader without buffering flushes them'); - $this->assertSame(['v'], $this->lockState('st')['flags']); - - // Field index: the journal entry exists only during the write - $db->createFieldIndex('st', 'a'); - $db->insert('st', ['a' => 3]); - $this->assertSame(['v'], $this->lockState('st')['flags']); - - $gen = $this->lockState('st')['gen']; - $this->assertTrue($db->dropDB('st')['success']); - $this->assertFileExists($this->dbBasePath('st') . '.nonedb.lock', 'lock file survives a drop'); - $this->assertSame(['v'], $this->lockState('st')['flags']); - $this->assertGreaterThan($gen, $this->lockState('st')['gen']); - $this->assertFalse($db->checkDB('st') && $db->count('st') > 0); - $this->assertSame([], $this->db()->find('st', 0) ?: []); - } - - /** @test */ - public function generationMovesOncePerWriteAndNeverOnReads(): void - { - $db = $this->db(); - $db->insert('g', [['a' => 1], ['a' => 2]]); - $db->createFieldIndex('g', 'a'); - $gen = $this->lockState('g')['gen']; - - $db->find('g', 0); - $db->find('g', ['a' => 1]); - $db->count('g'); - $db->query('g')->where(['a' => ['$gt' => 0]])->get(); - $this->assertSame($gen, $this->lockState('g')['gen'], 'reads leave the generation alone'); - - $db->insert('g', ['a' => 3]); - $this->assertSame($gen + 1, $this->lockState('g')['gen'], 'one write session, one bump'); - $db->update('g', [['a' => 3], ['set' => ['b' => 1]]]); - $this->assertSame($gen + 2, $this->lockState('g')['gen']); - $db->delete('g', ['a' => 3]); - $this->assertSame($gen + 3, $this->lockState('g')['gen']); - - // Another process sees each change (its caches are dropped on the generation change) - $out = $this->runChild('$d = ' . $this->childDb() . '; echo json_encode(array_column($d->find("g", 0), "a"));'); - $this->assertSame('[1,2]', $out); - } - - /** @test the path is locked, not a deleted file a kept handle still points to */ - public function replacedLockFileIsNotTrusted(): void - { - $db = $this->db(['lockTimeout' => 1]); - $db->insert('rl', ['a' => 1]); // the lock handle stays open in this process - $lock = $this->dbBasePath('rl') . '.nonedb.lock'; - $this->assertTrue(unlink($lock)); // e.g. a cleanup script - - $holder = $this->childScript('$fp = fopen(' . var_export($lock, true) . ', "c+"); flock($fp, LOCK_EX); echo "locked\n"; fflush(STDOUT); sleep(3);'); - $proc = proc_open([PHP_BINARY, $holder], [1 => ['pipe', 'w']], $pipes); // no shell: proc_terminate() reaches PHP - $this->assertSame('locked', trim(fgets($pipes[1]))); - try { - $db->insert('rl', ['a' => 2]); - $this->fail('the insert must wait for the process holding the new lock file'); - } catch (\RuntimeException $e) { - $this->assertStringContainsString('timed out', $e->getMessage()); - } finally { - fclose($pipes[1]); - proc_terminate($proc); - proc_close($proc); - } - $db->insert('rl', ['a' => 3]); - $this->assertSame([1, 3], array_column($db->find('rl', 0), 'a')); - $this->assertSame(['v'], $this->lockState('rl')['flags'], 'state record rewritten in the new file'); - } - - /** @test a forked child must not share the parent's lock handle (flock locks belong to the open file) */ - public function forkedProcessesStillExcludeEachOther(): void - { - if (!function_exists('pcntl_fork')) { - $this->markTestSkipped('needs pcntl'); - } - $out = $this->runChild(' - $cfg = ' . var_export(['secretKey' => self::SECRET, 'dbDir' => $this->testDbDir], true) . '; - $db = new \noneDB($cfg); - $db->insert("fk", ["w" => "init"]); // lock handle opened before the fork - $pid = pcntl_fork(); - $who = $pid === 0 ? "c" : "p"; - for ($i = 0; $i < 300; $i++) { - $db->insert("fk", ["w" => $who, "i" => $i]); - $db->update("fk", [["w" => "init"], ["set" => ["n" => $i]]]); - } - if ($pid === 0) { exit(0); } - pcntl_waitpid($pid, $status); - \noneDB::clearStaticCache(); - $all = (new \noneDB($cfg))->find("fk", 0); - $c = count(array_filter($all, function ($r) { return $r["w"] === "c"; })); - echo count($all), " ", count(array_unique(array_column($all, "key"))), " ", $c; - '); - $this->assertSame('601 601 300', $out); - } - - /** - * @test kept lock handles are close-on-exec: a program this process starts must not inherit one - * (an inherited copy would keep a lock held after this process died in the middle of a write) - */ - public function keptLockHandlesAreNotInherited(): void - { - if (DIRECTORY_SEPARATOR === '\\') { - $this->markTestSkipped('POSIX file descriptors'); - } - $db = $this->db(); - $db->insert('ih', ['a' => 1]); // the lock handle stays open in this process - $lock = $this->dbBasePath('ih') . '.nonedb.lock'; - $out = $this->runChild('$ino = stat(' . var_export($lock, true) . ')["ino"]; $n = 0; - for ($i = 3; $i < 256; $i++) { - $f = @fopen("php://fd/$i", "r"); - if ($f) { $s = @fstat($f); if ($s && $s["ino"] === $ino) { $n++; } fclose($f); } - } - echo "inherited:", $n;'); - $this->assertSame('inherited:0', $out); - } - - /** @test */ - public function keptLockHandlesAreLimited(): void - { - $db = $this->db(); - for ($i = 0; $i < 100; $i++) { - $db->insert('h' . $i, ['i' => $i]); - } - $handles = new \ReflectionProperty(\noneDB::class, 'lockHandles'); - if (PHP_VERSION_ID < 80100) { - $handles->setAccessible(true); // no effect since 8.1, deprecated in 8.5 - } - $this->assertLessThanOrEqual(64, count($handles->getValue())); - for ($i = 0; $i < 100; $i++) { - $this->assertSame([$i], array_column($db->find('h' . $i, 0), 'i')); - } - \noneDB::clearStaticCache(); - $this->assertSame([], $handles->getValue(), 'clearStaticCache closes them'); - } - - /** @test */ - public function readingAMissingDatabaseCreatesNothing(): void - { - $db = $this->db(); - $before = glob($this->testDbDir . '*'); - $this->assertSame([], $db->find('nothere', 0) ?: []); - $this->assertSame(0, $db->count('nothere')); - $this->assertSame([], $db->query('nothere')->get()); - $this->assertSame($before, glob($this->testDbDir . '*')); - } - - /** - * @test a pre-3.2 database is checked once (when it has no state record yet), not per operation; - * deleting the lock files makes the check run again - */ - public function oldVersionLeftoversAreCheckedOnce(): void - { - $db = $this->db(); - $db->insert('ov', [['a' => 1]]); - file_put_contents($this->dbBasePath('ov') . '.nonedb.buffer', json_encode(['a' => 2]) . "\n"); - $this->removeLockFile('ov'); - - $db = $this->db(); - $this->assertSame([1, 2], array_column($db->find('ov', 0), 'a'), 'found by the one-time check'); - $this->assertSame(['v'], $this->lockState('ov')['flags']); - - // Verified: an old-version file appearing now is not looked for on every operation... - file_put_contents($this->dbBasePath('ov') . '.nonedb.buffer.flushing', json_encode(['a' => 3]) . "\n"); - $this->assertSame([1, 2], array_column($db->find('ov', 0), 'a')); - // ...until the state record is removed (documented for going back and forth between versions) - $this->removeLockFile('ov'); - $this->assertSame([1, 2, 3], array_column($this->db()->find('ov', 0), 'a')); - } -} diff --git a/tests/Regression/RegressionTestCase.php b/tests/RegressionTestCase.php similarity index 59% rename from tests/Regression/RegressionTestCase.php rename to tests/RegressionTestCase.php index 9aad4a3..44f6067 100644 --- a/tests/Regression/RegressionTestCase.php +++ b/tests/RegressionTestCase.php @@ -1,8 +1,7 @@ setFaultHook(null); + // Process-wide setting a test may have changed (the last instance's value counts) + \noneDB\Tests\Support\Testing::set('cacheMemoryLimit', null); // automatic foreach ($this->tempFiles as $file) { @unlink($file); } @@ -27,6 +29,35 @@ protected function tearDown(): void parent::tearDown(); } + /** + * Install (or remove) noneDB's test hook: callable($point, array $context), called at named points + * of the write path ("append:after", "index:written", "rollback:truncate", "job:deferred", ...) + */ + protected function setFaultHook(?callable $hook): void + { + \noneDB\Tests\Support\Testing::setFaultHook($hook); + } + + /** + * Set a private property of a given noneDB instance + */ + protected function setInstanceProperty(\noneDB $db, string $name, $value): void + { + \noneDB\Tests\Support\Testing::setOf($db, $name, $value); + } + + /** + * Lines of a data file (complete lines only) + */ + protected function dataLines(string $file): array + { + clearstatcache(true, $file); + if (!is_file($file)) { + return []; + } + return array_values(array_filter(explode("\n", (string)file_get_contents($file)), 'strlen')); + } + /** * New instance on the test directory with extra config */ @@ -49,13 +80,15 @@ protected function childDb(array $config = []): string } /** - * Write a child script (body runs after noneDB.php is loaded) and return its path + * Write a child script (body runs after the test build of noneDB.php and the test interface are loaded) and + * return its path */ protected function childScript(string $body): string { $path = sys_get_temp_dir() . '/nonedb_regression_' . bin2hex(random_bytes(6)) . '.php'; - $repo = realpath(__DIR__ . '/../../noneDB.php'); - file_put_contents($path, "tempFiles[] = $path; return $path; } @@ -103,31 +136,32 @@ protected function runParallel(string $body, int $count): array } /** - * Base path {dbDir}{hash}-{name} of a database for the test secret + * State record of a database's lock file: ['gen' => int, 'epoch' => ?string, 'flags' => string[]] */ - protected function dbBasePath(string $dbName): string + protected function lockState(string $dbName): array { - $hash = hash_pbkdf2('sha256', $dbName, self::SECRET, 1000, 20); - return $this->testDbDir . $hash . '-' . $dbName; + $tokens = explode(' ', trim((string)file_get_contents($this->dbPath($dbName, 'lock')), " \0")); + $state = (string)array_shift($tokens); + $epoch = strpos($state, ':') !== false ? substr($state, strpos($state, ':') + 1) : null; + return ['gen' => (int)$state, 'epoch' => $epoch, 'flags' => array_values(array_filter($tokens, 'strlen'))]; } /** - * State record of a database's lock file: ['gen' => int, 'flags' => string[]] + * Write a database's state record as it is (space padded to 128 bytes) */ - protected function lockState(string $dbName): array + protected function writeLockState(string $dbName, string $record): void { - $tokens = explode(' ', trim((string)file_get_contents($this->dbBasePath($dbName) . '.nonedb.lock'))); - return ['gen' => (int)array_shift($tokens), 'flags' => array_values(array_filter($tokens, 'strlen'))]; + \noneDB::clearStaticCache(); // closes the kept lock handles + file_put_contents($this->dbPath($dbName, 'lock'), str_pad($record, 128)); } /** - * Turn a database into what a pre-3.2 version leaves behind as far as the lock is concerned: - * those versions had no lock file (and no state record) + * Delete a database's lock file (a new, empty lock record is created on the next access) */ protected function removeLockFile(string $dbName): void { \noneDB::clearStaticCache(); // closes the kept lock handles - $this->assertTrue(unlink($this->dbBasePath($dbName) . '.nonedb.lock')); + $this->assertTrue(unlink($this->dbPath($dbName, 'lock'))); } /** diff --git a/tests/Smoke/noneDB_critical_scope_regression.php b/tests/Smoke/noneDB_critical_scope_regression.php new file mode 100644 index 0000000..d603f7b --- /dev/null +++ b/tests/Smoke/noneDB_critical_scope_regression.php @@ -0,0 +1,68 @@ +isDir()) @rmdir($f->getPathname()); + else @unlink($f->getPathname()); + } + @rmdir($path); +} +function auditCase($name, $destructiveMethod, $sharding = false) { + $dir = sys_get_temp_dir().'/nonedb-scope-'.bin2hex(random_bytes(8)); + mkdir($dir,0700,true); + try { + $db = new noneDB([ + 'secretKey' => bin2hex(random_bytes(32)), 'dbDir' => $dir, + 'shardingEnabled' => $sharding, 'shardSize' => 5, 'bufferEnabled' => false, + ]); + auditAssert(($db->insert('victim',[['v'=>'original'],['v'=>'second']])['n'] ?? 0)===2,'setup insert failed'); + $thrown = null; + try { + noneDB\Tests\Support\Testing::call($db,'withLock','victim',true,function() use ($db,$destructiveMethod) { + $db->insert('victim',['v'=>'uncommitted']); + // A storage-file replacement/removal must not execute while an append rollback mark exists. + $db->$destructiveMethod('victim'); + throw new RuntimeException('abort outer write'); + }); + } catch(Throwable $e) { $thrown = $e; } + auditAssert($thrown!==null,'test failed to trigger an outer exception'); + $rows = $db->find('victim'); + $values = array_column($rows, 'v'); + auditAssert($values === ['original','second'], + "Rollback must preserve previously committed records and remove uncommitted inserts; actual=".json_encode($values). + "; exception=".get_class($thrown).'/'.$thrown->getCode()); + echo "PASS: {$name}\n"; + return true; + } catch(Throwable $e) { + echo "FAIL: {$name} :: ".get_class($e).": ".$e->getMessage()."\n"; + return false; + } finally { + auditClean($dir); + } +} + +$ok = 0; +$ok += auditCase('Dirty-scope dropDB is rejected before any deletion','dropDB') ? 1 : 0; +$ok += auditCase('Dirty-scope compact is rejected or safely deferred','compact') ? 1 : 0; +$ok += auditCase('Dirty-scope convertToSharded is rejected or safely deferred','convertToSharded',true) ? 1 : 0; +echo "Result: {$ok} passed, ".(3-$ok)." failed\n"; +exit($ok===3?0:1); diff --git a/tests/Smoke/noneDB_regression_smoke.php b/tests/Smoke/noneDB_regression_smoke.php new file mode 100644 index 0000000..5ce3205 --- /dev/null +++ b/tests/Smoke/noneDB_regression_smoke.php @@ -0,0 +1,204 @@ + $key, + 'dbDir' => $dir, + 'shardingEnabled' => false, + 'bufferEnabled' => false, + 'autoCreateDB' => true, + 'lockTimeout' => 2, + ]; +} + +// Run the read in a fresh process to rule out any in-process index cache. +if (isset($argv[1]) && $argv[1] === '--child-find') { + if (count($argv) !== 5) { exit(3); } + [, , $dir, $key, $dbname] = $argv; + $db = new noneDB(configFor($dir, $key)); + echo json_encode($db->find($dbname), JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION); + exit(0); +} + +$dir = sys_get_temp_dir() . '/nonedb-smoke-' . bin2hex(random_bytes(8)); +if (!mkdir($dir, 0700, true) && !is_dir($dir)) { + throw new RuntimeException("Cannot create temporary directory: $dir"); +} +$key = bin2hex(random_bytes(32)); + +function cleanup(string $path): void { + if (!is_dir($path)) { return; } + $it = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($it as $f) { + if ($f->isDir()) { @rmdir($f->getPathname()); } + else { @unlink($f->getPathname()); } + } + @rmdir($path); +} +register_shutdown_function(static function() use ($dir): void { cleanup($dir); }); + +$passed = 0; +$failed = 0; +function check(string $name, callable $test): void { + global $passed, $failed; + try { + $test(); + ++$passed; + echo "PASS: $name\n"; + } catch (Throwable $t) { + ++$failed; + echo "FAIL: $name :: " . get_class($t) . ': ' . $t->getMessage() . "\n"; + } +} +function ensure(bool $condition, string $message): void { + if (!$condition) { throw new RuntimeException($message); } +} +// Data file of a single-layout database (docs/STORAGE_FORMAT.md §1, §4.3, §5.1): +// {dbDir}/{locator}.ndb4/generations/{generation named by CURRENT}/segments/main.jsonl +function dbFileFor(string $dir, string $key, string $name): string { + $home = rtrim($dir, '/') . '/' . substr(hash_hmac('sha256', 'nonedb4/db/' . $name, $key), 0, 32) . '.ndb4/'; + $current = json_decode((string)@file_get_contents($home . 'CURRENT'), true); + ensure(is_array($current) && is_int($current['generation'] ?? null), 'Expected a CURRENT record of ' . $name); + $file = $home . 'generations/' . $current['generation'] . '/segments/main.jsonl'; + ensure(is_file($file), 'Expected the segment file of ' . $name); + return $file; +} +// Key-index checkpoint of a segment: {seg}.kix next to {seg}.jsonl (§6.4) +function checkpointFor(string $dataFile): string { + return substr($dataFile, 0, -strlen('.jsonl')) . '.kix'; +} +function childRead(string $dir, string $key, string $dbname): array { + $cmd = [PHP_BINARY, __FILE__, '--child-find', $dir, $key, $dbname]; + $pipes = []; + $process = proc_open($cmd, [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + ensure(is_resource($process), 'proc_open unavailable'); + $stdout = stream_get_contents($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($process); + ensure($status === 0, 'child failed: ' . $stderr); + $result = json_decode($stdout, true, 512, JSON_THROW_ON_ERROR); + ensure(is_array($result), 'child returned non-array JSON'); + return $result; +} + +check('A1: complete unindexed JSONL line is recovered', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('stale', ['v' => 'initial'])['n'] ?? 0) === 1, 'first insert failed'); + $file = dbFileFor($dir, $key, 'stale'); + $bytes = file_put_contents($file, "{\"v\":\"after-crash\",\"key\":1}\n", FILE_APPEND); + ensure($bytes !== false, 'could not emulate crash window'); + $rows = childRead($dir, $key, 'stale'); + ensure(count($rows) === 2, 'stale index hid a complete JSONL record; count=' . count($rows)); + ensure($rows[1]['v'] === 'after-crash', 'replayed record content was not preserved'); +}); + +check('A1: incomplete tail never swallows the next committed insert', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('torn', ['v' => 'first'])['n'] ?? 0) === 1, 'initial insert failed'); + $file = dbFileFor($dir, $key, 'torn'); + ensure(file_put_contents($file, '{"uncommitted":"fragment"', FILE_APPEND) !== false, 'tail injection failed'); + ensure(($db->insert('torn', ['v' => 'second'])['n'] ?? 0) === 1, 'subsequent insert failed'); + $index = checkpointFor($file); + ensure(is_file($index), 'index not found'); + ensure(unlink($index), 'cannot force index reconstruction'); + $rows = childRead($dir, $key, 'torn'); + ensure(count($rows) === 2, 'committed record lost after index rebuild'); + ensure($rows[0]['v'] === 'first' && $rows[1]['v'] === 'second', 'record order/content changed'); +}); + +check('A1: complete JSON without newline is not committed', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('tornfull', ['v' => 'first'])['n'] ?? 0) === 1, 'first insert failed'); + $file = dbFileFor($dir, $key, 'tornfull'); + ensure(file_put_contents($file, '{"v":"should-disappear","key":99}', FILE_APPEND) !== false, 'tail injection failed'); + ensure(($db->insert('tornfull', ['v' => 'second'])['n'] ?? 0) === 1, 'subsequent insert failed'); + ensure(unlink(checkpointFor($file)), 'could not delete checkpoint'); + $rows = childRead($dir, $key, 'tornfull'); + ensure(count($rows) === 2, 'uncommitted JSON tail was accepted or valid record was lost'); + ensure($rows[1]['v'] === 'second', 'committed record incorrect'); +}); + +check('A10: fractional lockTimeout is preserved as a float', static function() use ($dir, $key): void { + $config = configFor($dir, $key); + $config['lockTimeout'] = 0.5; + $db = new noneDB($config); + ensure(abs((float)noneDB\Tests\Support\Testing::getOf($db, 'lockTimeout') - 0.5) < 1e-9, '0.5 seconds was truncated'); +}); + +check('A12: JSON serialization preserves float precision', static function() use ($dir, $key): void { + $before = ini_get('serialize_precision'); + try { + ini_set('serialize_precision', '5'); + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('precision', ['p' => 28.980345])['n'] ?? 0) === 1, 'insert failed'); + $rows = $db->find('precision'); + ensure($rows[0]['p'] === 28.980345, 'precision lost: ' . var_export($rows[0]['p'], true)); + } finally { + if ($before !== false) { ini_set('serialize_precision', $before); } + } +}); + +check('C1: distinct keeps type-sensitive identities and original order', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + $values = [1, '1', 1.0, 0, -0.0, 0.0, true, false, [1, 'x'], [1, 'x'], null]; + $items = array_map(static function($v): array { return ['v' => $v]; }, $values); + ensure(($db->insert('distinct', $items)['n'] ?? 0) === count($items), 'insert failed'); + $expected = []; + foreach ($values as $v) { + if ($v !== null && !in_array($v, $expected, true)) { $expected[] = $v; } + } + ensure($db->distinct('distinct', 'v') === $expected, 'distinct semantics differ from strict reference'); + ensure($db->query('distinct')->distinct('v') === $expected, 'builder distinct differs'); +}); + +check('C3: mixed numeric/non-numeric string sorting is deterministic', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + $values = ['9', '10', '1a', '2', '20', 'alpha']; + for ($shift = 0; $shift < count($values); ++$shift) { + $permutation = array_merge(array_slice($values, $shift), array_slice($values, 0, $shift)); + $rows = array_map(static function($v): array { return ['v' => $v]; }, $permutation); + $sorted = array_column($db->sort($rows, 'v'), 'v'); + ensure($sorted === ['2', '9', '10', '20', '1a', 'alpha'], 'non-total or unexpected sort order: ' . json_encode($sorted)); + } +}); + +check('C6: builder key filter consistent in multiple clauses', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('keyfilter', ['v' => 'match'])['n'] ?? 0) === 1, 'insert failed'); + $rows = $db->query('keyfilter')->where(['key' => '0'])->where(['v' => 'match'])->get(); + ensure(count($rows) === 1 && $rows[0]['key'] === 0, 'builder changed key semantics'); +}); + +check('C7: whereNot accepts and negates compound operators', static function() use ($dir, $key): void { + $db = new noneDB(configFor($dir, $key)); + ensure(($db->insert('notrange', [['age' => 10], ['age' => 20], ['age' => 70]])['n'] ?? 0) === 3, 'insert failed'); + $rows = $db->query('notrange')->whereNot(['age' => ['$gte' => 18, '$lt' => 65]])->get(); + ensure(array_column($rows, 'age') === [10, 70], 'whereNot operator did not exclude age 20'); +}); + +echo "\nResult: {$passed} passed, {$failed} failed\n"; +exit($failed === 0 ? 0 : 1); diff --git a/tests/Support/Testing.php b/tests/Support/Testing.php new file mode 100644 index 0000000..7b33be4 --- /dev/null +++ b/tests/Support/Testing.php @@ -0,0 +1,180 @@ + method name => kernel service property (built on first use) */ + private static $owners = null; + + /** + * Set the fault hook: callable(string $point, array $context), called at the engine's fault points; it may + * throw or exit to simulate a failure there. Null removes it. + */ + public static function setFaultHook(?callable $hook): void { + Runtime::get()->faultHook = $hook; + } + + /** + * A process-wide value: hooks (faultHook, syncFn, flushFn, renameFn, mbstringFn, sweepBudget), counters + * (directoryListings, segmentReads, indexStats, metrics, ...), limits and caches + */ + public static function get(string $name) { + $runtime = Runtime::get(); + self::requireProperty($runtime, $name); + return $runtime->$name; + } + + public static function set(string $name, $value): void { + $runtime = Runtime::get(); + self::requireProperty($runtime, $name); + $runtime->$name = $value; + } + + /** A value of one instance: its settings (shardSize, lockTimeout, ...) and caches */ + public static function getOf(\noneDB $db, string $name) { + $kernel = Kernel::of($db); + self::requireProperty($kernel, $name); + return $kernel->$name; + } + + public static function setOf(\noneDB $db, string $name, $value): void { + $kernel = Kernel::of($db); + self::requireProperty($kernel, $name); + $kernel->$name = $value; + } + + /** Number of live noneDB instances (their kernels) */ + public static function liveInstances(): int { + return count(Kernel::$instances); + } + + /** The kernel of an instance */ + public static function kernel(\noneDB $db): Kernel { + return Kernel::of($db); + } + + /** The service of an instance that owns $method (bind a closure to it to call several of its methods) */ + public static function service(\noneDB $db, string $method) { + return self::serviceOf(Kernel::of($db), $method); + } + + /** + * An object standing for the whole engine of an instance: $engine->method(...) calls the internal method + * of whichever service owns it, $engine->name reads or writes the kernel's value (by reference). + * Bind a test closure to it with ->call($engine). By-reference arguments don't pass through: use + * service() for those. + */ + public static function engine(\noneDB $db) { + $kernel = Kernel::of($db); + return new class($kernel) { + /** @var Kernel */ + private $kernel; + + public function __construct(Kernel $kernel) { + $this->kernel = $kernel; + } + + public function __call(string $method, array $args) { + $service = Testing::serviceOfKernel($this->kernel, $method); + return (function (string $method, array $args) { + return $this->$method(...$args); + })->call($service, $method, $args); + } + + public function &__get(string $name) { + if (!property_exists($this->kernel, $name)) { + throw new \InvalidArgumentException('noneDB: no internal value ' . $name); + } + return $this->kernel->$name; + } + + public function __set(string $name, $value) { + if (!property_exists($this->kernel, $name)) { + throw new \InvalidArgumentException('noneDB: no internal value ' . $name); + } + $this->kernel->$name = $value; + } + + public function __isset(string $name) { + return isset($this->kernel->$name); + } + }; + } + + /** @internal for engine() */ + public static function serviceOfKernel(Kernel $kernel, string $method) { + return self::serviceOf($kernel, $method); + } + + /** + * Call an internal method of an instance, whatever its visibility + * @return mixed + */ + public static function call(\noneDB $db, string $method, ...$args) { + $service = self::serviceOf(Kernel::of($db), $method); + return (function (string $method, array $args) { + return $this->$method(...$args); + })->call($service, $method, $args); + } + + /** + * Call an internal static method, whatever its visibility: of a service by its name, of any other engine class + * as 'Class::method' (the class by its short name, e.g. 'ShardFilters::filter') + * @return mixed + */ + public static function callStatic(string $method, ...$args) { + $classes = self::serviceClasses(); + if (strpos($method, '::') !== false) { + [$short, $method] = explode('::', $method, 2); + $classes = array_filter(get_declared_classes(), function (string $class) use ($short) { + return strpos($class, 'noneDB\\Internal\\') === 0 && substr($class, strrpos($class, '\\') + 1) === $short; + }); + } + foreach ($classes as $class) { + if (method_exists($class, $method)) { + $call = \Closure::bind(function (string $method, array $args) { + return static::$method(...$args); + }, null, $class); + return $call($method, $args); + } + } + throw new \InvalidArgumentException("noneDB: no internal method $method"); + } + + private static function serviceOf(Kernel $kernel, string $method) { + if (self::$owners === null) { + self::$owners = []; + foreach (Kernel::SERVICES as $property => $class) { + foreach ((new \ReflectionClass($class))->getMethods() as $reflection) { + self::$owners[$reflection->getName()] = self::$owners[$reflection->getName()] ?? $property; + } + } + } + if (!isset(self::$owners[$method])) { + throw new \InvalidArgumentException("noneDB: no internal method $method"); + } + return $kernel->{self::$owners[$method]}; + } + + /** @return string[] the classes of the kernel's services */ + private static function serviceClasses(): array { + return array_values(Kernel::SERVICES); + } + + private static function requireProperty($object, string $name): void { + if (!property_exists($object, $name)) { + throw new \InvalidArgumentException('noneDB: no internal value ' . $name . ' in ' . get_class($object)); + } + } +} diff --git a/tests/Unit/ArchitectureTest.php b/tests/Unit/ArchitectureTest.php new file mode 100644 index 0000000..9e1fca8 --- /dev/null +++ b/tests/Unit/ArchitectureTest.php @@ -0,0 +1,223 @@ + significant tokens of a file */ + private static function tokens(string $file): array + { + $out = []; + foreach (token_get_all((string)file_get_contents($file)) as $token) { + $token = is_array($token) ? $token : [null, $token, 0]; + if (!in_array($token[0], [T_WHITESPACE, T_COMMENT, T_DOC_COMMENT], true)) { + $out[] = $token; + } + } + return $out; + } + + private static function relative(string $file): string + { + return substr($file, strlen(self::root()) + 1); + } + + /** + * @test the Query layer works on records and plans; it has no file-system access and no knowledge of the + * storage format (it calls the storage services) + */ + public function theQueryLayerKnowsNoFilesAndNoFormat(): void + { + $files = array_merge(self::files('src/Internal/Query'), self::files('src/noneDBQuery')); + $this->assertGreaterThan(10, count($files)); + $found = []; + foreach ($files as $file) { + $tokens = self::tokens($file); + foreach ($tokens as $i => [$id, $text, $line]) { + $next = $tokens[$i + 1][1] ?? null; + $previous = $tokens[$i - 1][0] ?? null; + if ($id === T_STRING && $next === '(' && in_array(strtolower($text), self::FILE_FUNCTIONS, true) + && !in_array($previous, [T_OBJECT_OPERATOR, T_DOUBLE_COLON, T_FUNCTION, T_NULLSAFE_OBJECT_OPERATOR], true)) { + $found[] = self::relative($file) . ":$line $text()"; + } + if ($id === T_CONSTANT_ENCAPSED_STRING || $id === T_ENCAPSED_AND_WHITESPACE) { + foreach (self::FORMAT_NAMES as $name) { + if (strpos($text, $name) !== false) { + $found[] = self::relative($file) . ":$line $text"; + } + } + } + } + } + $this->assertSame([], $found); + } + + /** + * @test dependencies point inwards: the engine (src/Internal) never uses the query builder, which is part + * of the public API (noneDBQuery calls the engine's Query services, not the other way round) + */ + public function theEngineDoesNotDependOnTheQueryBuilder(): void + { + $found = []; + foreach (self::files('src/Internal') as $file) { + foreach (self::tokens($file) as [$id, $text, $line]) { + if (in_array($id, [T_STRING, T_NAME_FULLY_QUALIFIED], true) && ltrim($text, '\\') === 'noneDBQuery') { + $found[] = self::relative($file) . ":$line"; + } + } + } + $this->assertSame([], $found); + } + + /** @test the migration tool is separate: the engine artifact has none of its symbols */ + public function theEngineHasNoMigrationToolSymbols(): void + { + $found = []; + foreach (self::tokens(self::root() . '/noneDB.php') as [$id, $text, $line]) { + if (in_array($id, [T_NAME_QUALIFIED, T_NAME_FULLY_QUALIFIED, T_STRING], true) && preg_match('/\bMigrate\b|\\\\Migrate\\\\|noneDBMigrate/', $text)) { + $found[] = "$line $text"; + } + } + $this->assertSame([], $found); + $this->assertDoesNotMatchRegularExpression('/^namespace noneDB\\\\Migrate/m', (string)file_get_contents(self::root() . '/noneDB.php')); + } + + /** + * @test the engine knows no earlier format: no old file names, no old encodings, no version-dated wording + * (the M2 audit, kept as a test), no pointer to the migration tool, no + * names of earlier releases (the code audit of 2026-10-29) + */ + public function theEngineHasNoTraceOfEarlierVersions(): void + { + $code = (string)file_get_contents(self::root() . '/noneDB.php'); + $patterns = [ + 'old file names' => '/jidx|fidx|gfidx|\bsidx|gsidx|nonedbinfo|\.nonedb\.(meta|buffer|indexes|recovery|lock)\b|_s[0-9]+\.nonedb|hash_cache|pbkdf2/', + 'old conversions' => '/migrateToJsonl|convertSegmentIndexes|upgradeIndexes|"data":\[|legacy/i', + 'version-dated wording' => '/\bv[0-3]\.[0-9]|\bsince [0-3]\.[0-9]|\bbefore [0-3]\.[0-9]|\bpre-[0-3]\.[0-9]|\bas in [0-3]\.[0-9]|kept for compatib|backward compatib/i', + 'PHP version branches' => '/PHP_VERSION_ID|function_exists\(.fsync|func_overload/', + // the engine sends no one to the converter of old files (its documentation does) + 'the migration tool' => '/noneDB-migrate|MIGRATION\.md/', + // domain strings of hashes are noneDB 4's ("nonedb4/..."), never a labelled earlier one ("...-v1") + 'version-labelled names' => '/[a-z]-v[0-9]+[\'"]/', + 'names of earlier releases' => '/\bROLLBACK_FAILED\b|\bautoMigrate\b|function migrate\(|jsonlGarbageThreshold/', + ]; + foreach ($patterns as $label => $pattern) { + preg_match_all($pattern, $code, $m); + $this->assertSame([], $m[0], $label); + } + } + + /** @test every internal class is in the namespace of its directory, one class per file */ + public function internalClassesFollowTheirDirectories(): void + { + foreach (self::files('src/Internal') as $file) { + $relative = substr($file, strlen(self::root() . '/src/'), -4); + $expected = 'noneDB\\' . str_replace('/', '\\', dirname($relative)); + $content = (string)file_get_contents($file); + $this->assertMatchesRegularExpression('/^namespace ' . preg_quote($expected, '/') . ';$/m', $content, $relative); + $this->assertSame(1, preg_match_all('/^(?:final |abstract )?class \w+/m', $content), "$relative: one class"); + $this->assertMatchesRegularExpression('/^(?:final |abstract )?class ' . preg_quote(basename($relative), '/') . '\b/m', $content, $relative); + } + } + + /** + * @test noneDB.php holds only the engine: no test interface, no test seam (fault points, test hooks and + * counters exist only in the test build, tools/build.php), no seam marker + */ + public function theEngineHoldsNoTestCode(): void + { + $code = (string)file_get_contents(self::root() . '/noneDB.php'); + foreach (['class Testing', '@test', '@prod', 'faultHook', '->fault(', 'syncFn', 'flushFn', 'writeFn', 'renameFn', 'mbstringFn', + 'sweepBudget', 'segmentPairTests', 'pointEdgeTests', 'geometryValidations', 'segmentReads', 'directoryListings', + 'indexStats', 'cacheEvictions', '$counter'] as $name) { + $this->assertStringNotContainsString($name, $code, $name); + } + $this->assertFileDoesNotExist(self::root() . '/src/Internal/Testing.php'); + } + + /** @test the migration tool's classes are in the namespace of their directory under migrate/, one class per file */ + public function migrationToolClassesFollowTheirDirectories(): void + { + $files = array_filter(self::files('migrate'), function ($file) { + return !in_array(basename($file), ['header.php', 'footer.php', 'manifest.php'], true); + }); + $this->assertGreaterThan(8, count($files)); + $manifest = require self::root() . '/migrate/manifest.php'; + $this->assertSame(count($files), count($manifest['files']), 'every class file is built'); + foreach ($files as $file) { + $relative = substr($file, strlen(self::root() . '/migrate/'), -4); + $directory = dirname($relative); + $expected = 'noneDB\\Migrate' . ($directory === '.' ? '' : '\\' . str_replace('/', '\\', $directory)); + $content = (string)file_get_contents($file); + $this->assertMatchesRegularExpression('/^namespace ' . preg_quote($expected, '/') . ';$/m', $content, $relative); + $this->assertSame(1, preg_match_all('/^(?:final |abstract )?class \w+/m', $content), "$relative: one class"); + $this->assertMatchesRegularExpression('/^(?:final |abstract )?class ' . preg_quote(basename($relative), '/') . '\b/m', $content, $relative); + } + } + + /** @test noneDB-migrate.php holds only the tool: its crash points exist in the test build only */ + public function theMigrationToolHoldsNoTestCode(): void + { + $code = (string)file_get_contents(self::root() . '/noneDB-migrate.php'); + foreach (['@test', '@prod', 'NONEDB_MIGRATE_KILL', 'posix_kill', 'self::point('] as $name) { + $this->assertStringNotContainsString($name, $code, $name); + } + $this->assertStringContainsString('NONEDB_MIGRATE_KILL', (string)file_get_contents(NONEDB_TEST_TOOL), 'the test build has them'); + } + + /** @test tests reach the engine through noneDB\Tests\Support\Testing only (plan D8), never through Reflection */ + public function testsUseTheTestingInterface(): void + { + $found = []; + foreach (self::files('tests') as $file) { + // The interface itself, this test (its patterns) and the generated test build + if (in_array(self::relative($file), ['tests/Unit/ArchitectureTest.php', 'tests/Support/Testing.php'], true) + || strncmp(self::relative($file), 'tests/build/', 12) === 0) { + continue; + } + foreach (self::tokens($file) as [$id, $text, $line]) { + if (in_array($id, [T_NAME_QUALIFIED, T_NAME_FULLY_QUALIFIED], true) && preg_match('/^\\\\?noneDB\\\\Internal\\\\/', $text)) { + $found[] = self::relative($file) . ":$line $text"; + } + if (in_array($id, [T_STRING, T_NAME_FULLY_QUALIFIED], true) && in_array(ltrim($text, '\\'), ['ReflectionProperty', 'ReflectionMethod', 'ReflectionClass'], true)) { + $found[] = self::relative($file) . ":$line $text"; + } + } + } + $this->assertSame([], $found); + } +} diff --git a/tests/Unit/BuildTest.php b/tests/Unit/BuildTest.php new file mode 100644 index 0000000..9649f08 --- /dev/null +++ b/tests/Unit/BuildTest.php @@ -0,0 +1,228 @@ +markTestSkipped('no src/ (installed package)'); + } + require_once self::root() . '/tools/build.php'; + } + + /** @var string[] */ + private $tempDirs = []; + + protected function tearDown(): void + { + foreach ($this->tempDirs as $dir) { + $this->removeDir($dir); + } + } + + private function removeDir(string $dir): void + { + foreach (scandir($dir) as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + $path = "$dir/$entry"; + is_dir($path) ? $this->removeDir($path) : unlink($path); + } + rmdir($dir); + } + + /** A copy of src/ in a temporary directory, for broken variants */ + private function copyOfSrc(): string + { + $dir = sys_get_temp_dir() . '/nonedb_build_' . bin2hex(random_bytes(4)); + $this->tempDirs[] = $dir; + $copy = function (string $from, string $to) use (&$copy) { + mkdir($to, 0777, true); + foreach (scandir($from) as $entry) { + if ($entry !== '.' && $entry !== '..') { + is_dir("$from/$entry") ? $copy("$from/$entry", "$to/$entry") : copy("$from/$entry", "$to/$entry"); + } + } + }; + $copy(self::root() . '/src', "$dir/src"); + return $dir; + } + + /** + * @test test seams (tools/build.php): a line ending in "// @test" and the lines of a "// @test-begin" … + * "// @test-end" block exist only in the test build; a line ending in "// @prod" only in noneDB.php, + * without its marker. Every other line is in both, in the same order. + */ + public function testSeamsSplitTheTwoBuilds(): void + { + $source = implode("\n", [ + 'a();', + '$x = plain(); // @prod', + '$x = hooked(); // @test', + 'b(); // a comment', + ' // @test-begin', + ' function seam() {}', + ' // @test-end', + 'c(); // @test ', + 'd();', + ]); + $this->assertSame(implode("\n", ['a();', '$x = plain();', 'b(); // a comment', 'd();']), nonedb_build_variant($source, false, 'f')); + $this->assertSame(implode("\n", ['a();', '$x = hooked(); // @test', 'b(); // a comment', ' // @test-begin', ' function seam() {}', + ' // @test-end', 'c(); // @test ', 'd();']), nonedb_build_variant($source, true, 'f')); + foreach (['nested' => "// @test-begin\n// @test-begin\n// @test-end\n// @test-end", 'end alone' => "x();\n// @test-end", + 'never ended' => "// @test-begin\nx();"] as $label => $broken) { + foreach ([false, true] as $test) { + try { + nonedb_build_variant($broken, $test, 'f'); + $this->fail("$label: accepted"); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('@test-', $e->getMessage(), $label); + } + } + } + } + + /** @test the test build of src/ is valid PHP declaring the same classes, with the seams noneDB.php leaves out */ + public function theTestBuildHoldsTheSeams(): void + { + $test = nonedb_build(self::root(), true)['code']; // verified as the production build is + $production = (string)file_get_contents(self::root() . '/noneDB.php'); + foreach (['faultHook', 'function fault(', 'writeFn', 'indexStats'] as $seam) { + $this->assertStringContainsString($seam, $test, $seam); + $this->assertStringNotContainsString($seam, $production, $seam); + } + $this->assertGreaterThan(substr_count($production, "\n"), substr_count($test, "\n")); + } + + /** @test */ + public function noneDBphpIsTheBuildOfSrc(): void + { + $build = nonedb_build(self::root()); + $this->assertTrue($build['code'] === file_get_contents(self::root() . '/noneDB.php'), + 'noneDB.php differs from the build of src/: run php tools/build.php'); + } + + /** @test */ + public function everyModuleIsListedAndTheMapCoversTheClasses(): void + { + $build = nonedb_build(self::root()); + $listed = []; + $previous = null; + foreach ($build['map'] as [$class, $module, $first, $last]) { + $listed[] = "$class/$module.php"; + $this->assertLessThanOrEqual($last, $first); + if ($previous !== null && $previous[0] === $class) { + $this->assertSame($previous[1] + 1, $first, "$class/$module follows the previous module"); + } + $previous = [$class, $last]; + } + $modules = []; + foreach (['noneDB', 'noneDBQuery'] as $class) { + foreach (glob(self::root() . "/src/$class/*.php") as $file) { + $modules[] = "$class/" . basename($file); + } + } + $classFiles = []; + foreach (new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator(self::root() . '/src/Internal', \FilesystemIterator::SKIP_DOTS)) as $file) { + $classFiles[] = substr((string)$file, strlen(self::root() . '/src/')); + } + $manifest = require self::root() . '/src/manifest.php'; + $listedFiles = $manifest['files']; + $listedModules = []; + foreach ($build['map'] as [$class, $module]) { + if (in_array($class, ['noneDB', 'noneDBQuery'], true)) { + $listedModules[] = "$class/$module.php"; + } + } + sort($modules); + sort($listedModules); + sort($classFiles); + sort($listedFiles); + $this->assertSame($modules, $listedModules, 'every file in src// is in the manifest, once'); + $this->assertSame($classFiles, $listedFiles, 'every file in src/Internal/ is in the manifest, once'); + $this->assertSame(count($listedFiles), count(array_unique($listedFiles))); + } + + /** @test */ + public function checkModeAgreesWithTheFile(): void + { + $process = proc_open([PHP_BINARY, self::root() . '/tools/build.php', '--check'], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + $output = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $this->assertSame(0, proc_close($process), $output); + } + + /** @test */ + public function malformedModulesAreRejected(): void + { + $cases = [ + 'CR line ends' => function (string $s) { return str_replace("\n", "\r\n", $s); }, + 'wrong trait name' => function (string $s) { return str_replace('trait noneDBQuery_Chain {', 'trait noneDBQuery_Other {', $s); }, + 'text before the trait' => function (string $s) { return str_replace("trait noneDBQuery_Chain {", "\$x = 1;\ntrait noneDBQuery_Chain {", $s); }, + 'unbalanced braces' => function (string $s) { return preg_replace('/\}\n\z/', "}\n}\n", $s); }, + 'syntax error' => function (string $s) { return str_replace('public function where(', 'public function where(,', $s); }, + 'a class in a body' => function (string $s) { return preg_replace('/\}\n\z/', "}\n}\nclass Extra {\n}\n", $s); }, + ]; + foreach ($cases as $name => $mutate) { + $dir = $this->copyOfSrc(); + $file = "$dir/src/noneDBQuery/Chain.php"; + file_put_contents($file, $mutate((string)file_get_contents($file))); + try { + nonedb_build($dir); + $this->fail("$name: accepted"); + } catch (\RuntimeException $e) { + $this->assertNotSame('', $e->getMessage(), $name); + } + } + $dir = $this->copyOfSrc(); + unlink("$dir/src/noneDBQuery/Chain.php"); + $this->expectException(\RuntimeException::class); + nonedb_build($dir); + } + + /** @test class files: one class named like the file, in a noneDB\Internal namespace, valid PHP */ + public function malformedClassFilesAreRejected(): void + { + $cases = [ + 'CR line ends' => function (string $s) { return str_replace("\n", "\r\n", $s); }, + 'another namespace' => function (string $s) { return str_replace('namespace noneDB\\Internal\\Query;', 'namespace Other;', $s); }, + 'no namespace' => function (string $s) { return str_replace("namespace noneDB\\Internal\\Query;\n", '', $s); }, + 'class named otherwise' => function (string $s) { return str_replace('final class Planner {', 'final class Planned {', $s); }, + 'a second class' => function (string $s) { return $s . "class Extra {\n}\n"; }, + 'a trait' => function (string $s) { return $s . "trait Extra {\n}\n"; }, + 'syntax error' => function (string $s) { return preg_replace('/public function (\w+)\(/', 'public function $1(,', $s, 1); }, + ]; + foreach ($cases as $name => $mutate) { + $dir = $this->copyOfSrc(); + $file = "$dir/src/Internal/Query/Planner.php"; + $before = (string)file_get_contents($file); + $after = $mutate($before); + $this->assertNotSame($before, $after, "$name: the mutation applies"); + file_put_contents($file, $after); + try { + nonedb_build($dir); + $this->fail("$name: accepted"); + } catch (\RuntimeException $e) { + $this->assertNotSame('', $e->getMessage(), $name); + } + } + $dir = $this->copyOfSrc(); + unlink("$dir/src/Internal/Query/Planner.php"); + $this->expectException(\RuntimeException::class); + nonedb_build($dir); + } +} diff --git a/tests/Unit/DatabaseLocatorTest.php b/tests/Unit/DatabaseLocatorTest.php new file mode 100644 index 0000000..0798632 --- /dev/null +++ b/tests/Unit/DatabaseLocatorTest.php @@ -0,0 +1,230 @@ +getPrivateMethod('locator')->invoke($db ?? $this->noneDB, $name); + } + + /** + * @test + */ + public function locatorIs32Characters(): void + { + $locator = $this->locator('testdb'); + + $this->assertSame(32, strlen($locator), 'Locator should be exactly 32 characters'); + $this->assertSame($this->spec('testdb'), $locator); + } + + /** + * @test + */ + public function locatorIsConsistentForSameInput(): void + { + $first = $this->locator('mydb'); + $this->assertSame($first, $this->locator('mydb'), 'cached'); + // A new process state (no in-memory cache) computes the same value + \noneDB::clearStaticCache(); + $this->assertSame($first, $this->locator('mydb', $this->createTestInstance())); + $this->assertSame($this->spec('mydb'), $first); + } + + /** + * @test + */ + public function differentInputsProduceDifferentLocators(): void + { + $this->assertSame($this->spec('database1'), $this->locator('database1')); + $this->assertSame($this->spec('database2'), $this->locator('database2')); + $this->assertNotSame($this->locator('database1'), $this->locator('database2')); + } + + /** + * @test + */ + public function locatorWorksWithEmptyString(): void + { + $this->assertSame($this->spec(''), $this->locator('')); + } + + /** + * @test + */ + public function locatorWorksWithSpecialCharacters(): void + { + $this->assertSame($this->spec("test'db-name 123"), $this->locator("test'db-name 123")); + } + + /** + * @test + */ + public function locatorWorksWithUnicodeCharacters(): void + { + $this->assertSame($this->spec('veritabanı'), $this->locator('veritabanı')); + } + + /** + * @test + */ + public function locatorWorksWithVeryLongString(): void + { + $longString = str_repeat('a', 10000); + $this->assertSame($this->spec($longString), $this->locator($longString)); + } + + /** + * @test + */ + public function locatorOnlyContainsHexCharacters(): void + { + $this->assertMatchesRegularExpression('/^[0-9a-f]{32}$/', $this->locator('testdb'), 'exactly 32 lower-case hex digits'); + } + + /** + * @test + */ + public function locatorIsCaseSensitive(): void + { + $this->assertSame($this->spec('TestDB'), $this->locator('TestDB')); + $this->assertNotSame($this->locator('TestDB'), $this->locator('testdb'), 'Locator should be case sensitive'); + } + + /** + * @test + */ + public function locatorWithWhitespace(): void + { + $this->assertSame($this->spec('test db'), $this->locator('test db')); + $this->assertNotSame($this->locator('test db'), $this->locator('testdb'), 'Whitespace should affect the locator'); + } + + /** + * @test the key binds the locator: the same name under another key has another directory (STORAGE_FORMAT.md §1 + * "unpredictable without the key, and changes when the key is rotated"), also within one process (caches are per key) + */ + public function locatorDependsOnTheSecretKey(): void + { + $other = new \noneDB(['secretKey' => 'a-completely-different-key', 'dbDir' => $this->testDbDir]); + $this->assertSame($this->spec('users'), $this->locator('users')); + $this->assertSame($this->spec('users', 'a-completely-different-key'), $this->locator('users', $other)); + $this->assertNotSame($this->locator('users'), $this->locator('users', $other)); + $this->assertSame($this->spec('users'), $this->locator('users'), 'the other key cached nothing for this one'); + } + + /** + * @test a database is the directory {locator}.ndb4/ in dbDir with FORMAT, LOCK, CURRENT and its generation + * (STORAGE_FORMAT.md §3, §4); no entry of dbDir names the database in clear text + */ + public function databaseDirectoryIsNamedAfterTheLocator(): void + { + $this->noneDB->insert('named', ['a' => 1]); + $home = $this->testDbDir . $this->spec('named') . '.ndb4/'; + $this->assertDirectoryExists($home); + $this->assertSame($this->dbPath('named', 'home'), $home); + $this->assertFileExists($home . 'FORMAT'); + $this->assertFileExists($home . 'LOCK'); + $this->assertFileExists($home . 'CURRENT'); + $this->assertFileExists($home . 'generations/1/metadata'); + $this->assertFileExists($home . 'generations/1/segments/main.jsonl'); + + $entries = array_values(array_diff(scandir($this->testDbDir), ['.', '..'])); + sort($entries); + $expected = ['.htaccess', 'STORE', 'STORE.2', 'STORE.LOCK', $this->spec('named') . '.ndb4', 'index.html']; + sort($expected); + $this->assertSame($expected, $entries); + foreach ($entries as $entry) { + $this->assertStringNotContainsString('named', $entry, 'the name is not readable from dbDir'); + } + + // FORMAT carries the name but no key material (§4.1) + $format = json_decode((string)file_get_contents($home . 'FORMAT'), true); + $this->assertSame('named', $format['name']); + $raw = (string)file_get_contents($home . 'FORMAT'); + $this->assertStringNotContainsString(self::KEY, $raw); + $this->assertStringNotContainsString($this->specKeyCheck(self::KEY), $raw); + } + + /** + * @test STORE lists the keyCheck of the key that created the store, never the key itself (STORAGE_FORMAT.md §1, §3.1) + */ + public function storeListsTheKeyCheckOfItsKey(): void + { + $this->noneDB->insert('named', ['a' => 1]); + foreach (['STORE', 'STORE.2'] as $copy) { + $raw = (string)file_get_contents($this->testDbDir . $copy); + $store = json_decode($raw, true); + $this->assertSame([$this->specKeyCheck(self::KEY)], $store['keys'], $copy); + $this->assertStringNotContainsString(self::KEY, $raw, $copy); + } + } + + /** + * @test key separation: a key that is not registered in STORE can't use the store (KEY_NOT_REGISTERED, nothing + * created, §3.2); once registered, the same name under each key is its own database in its own directory + */ + public function keysAreSeparated(): void + { + $otherKey = 'another-registered-key-0123'; + $this->noneDB->insert('users', ['owner' => 'first']); + $before = scandir($this->testDbDir); + + $other = new \noneDB(['secretKey' => $otherKey, 'dbDir' => $this->testDbDir]); + foreach (['find' => function () use ($other) { return $other->find('users', 0); }, + 'insert' => function () use ($other) { return $other->insert('users', ['owner' => 'second']); }] as $op => $call) { + try { + $call(); + $this->fail("$op with an unregistered key must fail"); + } catch (\RuntimeException $e) { + $this->assertSame(\noneDB::KEY_NOT_REGISTERED, $e->getCode(), $op); + } + } + clearstatcache(); + $this->assertSame($before, scandir($this->testDbDir), 'nothing was created'); + $this->assertDirectoryDoesNotExist($this->testDbDir . $this->spec('users', $otherKey) . '.ndb4'); + + \noneDB::clearStaticCache(); + $result = \noneDB::registerKey($this->testDbDir, self::KEY, $otherKey); + $this->assertTrue($result['success'], (string)$result['error']); + $expectedKeys = [$this->specKeyCheck(self::KEY), $this->specKeyCheck($otherKey)]; + sort($expectedKeys); + $this->assertSame($expectedKeys, $result['storeManifest']['keys']); + + $other = new \noneDB(['secretKey' => $otherKey, 'dbDir' => $this->testDbDir]); + $this->assertSame([], $other->find('users', 0), 'the other key does not see the first key\'s database'); + $this->assertSame(1, $other->insert('users', ['owner' => 'second'])['n']); + $this->assertDirectoryExists($this->testDbDir . $this->spec('users', $otherKey) . '.ndb4'); + $this->assertDirectoryExists($this->testDbDir . $this->spec('users') . '.ndb4'); + + $this->assertSame([['owner' => 'second', 'key' => 0]], $other->find('users', 0)); + $first = $this->createTestInstance(); + $this->assertSame([['owner' => 'first', 'key' => 0]], $first->find('users', 0)); + $this->assertSame(['users'], $first->getDBs(false)); + $this->assertSame(['users'], $other->getDBs(false)); + } +} diff --git a/tests/Unit/FileSizeConvertTest.php b/tests/Unit/FileSizeConvertTest.php index 18eed4e..a5f3b26 100644 --- a/tests/Unit/FileSizeConvertTest.php +++ b/tests/Unit/FileSizeConvertTest.php @@ -182,9 +182,25 @@ public function roundsToTwoDecimalPlaces(): void public function floatInput(): void { $method = $this->getPrivateMethod('fileSizeConvert'); - $result = $method->invoke($this->noneDB, 1024.5); - $this->assertIsString($result, 'Should handle float input'); + // 1024.5 / 1024 = 1.00049 -> rounded to 2 decimals: 1 + $this->assertSame('1 KB', $method->invoke($this->noneDB, 1024.5)); + // 1536.5 / 1024 = 1.50049 -> 1.5 + $this->assertSame('1,5 KB', $method->invoke($this->noneDB, 1536.5)); + // 512.25 bytes stay bytes: 512.25 + $this->assertSame('512,25 B', $method->invoke($this->noneDB, 512.25)); + } + + /** + * @test TB is the largest unit: larger sizes stay in TB + */ + public function largerThanTerabytesStaysInTerabytes(): void + { + $method = $this->getPrivateMethod('fileSizeConvert'); + + $this->assertSame('2048 TB', $method->invoke($this->noneDB, pow(1024, 4) * 2048)); + // 1.5 TB + $this->assertSame('1,5 TB', $method->invoke($this->noneDB, pow(1024, 4) * 1.5)); } /** @@ -210,7 +226,7 @@ public function includesUnitSuffix(): void foreach ($values as $i => $value) { $result = $method->invoke($this->noneDB, $value); - $this->assertStringEndsWith(' ' . $units[$i], $result, "Should end with {$units[$i]}"); + $this->assertSame('1 ' . $units[$i], $result, "exactly one {$units[$i]}"); } } } diff --git a/tests/Unit/GeoJSONValidationTest.php b/tests/Unit/GeoJSONValidationTest.php index f3cd429..04e1d61 100644 --- a/tests/Unit/GeoJSONValidationTest.php +++ b/tests/Unit/GeoJSONValidationTest.php @@ -2,7 +2,6 @@ /** * GeoJSON Validation Unit Tests * Tests all GeoJSON geometry type validations - * @version 3.1.0 */ namespace noneDB\Tests\Unit; @@ -24,6 +23,7 @@ public function testValidPoint(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); $this->assertNull($result['error'] ?? null); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'Point'], $result); } /** @@ -39,6 +39,7 @@ public function testPointInvalidCoordinatesType(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('array', $result['error']); + $this->assertSame(['valid' => false, 'error' => 'Coordinates must be an array', 'type' => null], $result); // docs/SPATIAL.md error table } /** @@ -52,6 +53,9 @@ public function testPointMissingCoordinates(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); + $this->assertIsString($result['error']); + $this->assertNotSame('', $result['error']); + $this->assertNull($result['type']); } /** @@ -66,6 +70,8 @@ public function testPointWrongCoordinateCount(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); + $this->assertIsString($result['error']); + $this->assertNull($result['type']); } /** @@ -81,6 +87,7 @@ public function testPointInvalidLongitude(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('longitude', strtolower($result['error'])); + $this->assertSame('Longitude must be between -180 and 180', $result['error']); // docs/SPATIAL.md error table } /** @@ -96,6 +103,7 @@ public function testPointInvalidLatitude(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('latitude', strtolower($result['error'])); + $this->assertSame('Latitude must be between -90 and 90', $result['error']); // docs/SPATIAL.md error table } /** @@ -113,6 +121,7 @@ public function testValidLineString(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'LineString'], $result); } /** @@ -130,6 +139,8 @@ public function testLineStringTooFewPoints(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('2', $result['error']); + $this->assertStringContainsString('at least 2', $result['error']); + $this->assertNull($result['type']); } /** @@ -150,6 +161,7 @@ public function testValidPolygon(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'Polygon'], $result); } /** @@ -171,6 +183,7 @@ public function testPolygonUnclosedRing(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('closed', strtolower($result['error'])); + $this->assertSame('Ring 0 must be closed (first and last positions must be identical)', $result['error']); // docs } /** @@ -189,6 +202,7 @@ public function testPolygonTooFewPoints(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); + $this->assertSame('Ring 0 must have at least 4 positions', $result['error']); // docs/SPATIAL.md error table } /** @@ -208,6 +222,7 @@ public function testPolygonWithHole(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'Polygon'], $result); } /** @@ -226,6 +241,7 @@ public function testValidMultiPoint(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'MultiPoint'], $result); } /** @@ -243,6 +259,7 @@ public function testValidMultiLineString(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'MultiLineString'], $result); } /** @@ -260,6 +277,7 @@ public function testValidMultiPolygon(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'MultiPolygon'], $result); } /** @@ -277,6 +295,7 @@ public function testValidGeometryCollection(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'GeometryCollection'], $result); } /** @@ -294,6 +313,8 @@ public function testGeometryCollectionWithInvalidGeometry(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); + $this->assertStringContainsString('Longitude must be between -180 and 180', $result['error']); + $this->assertNull($result['type']); } /** @@ -308,6 +329,7 @@ public function testMissingType(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('type', strtolower($result['error'])); + $this->assertSame('Missing type property', $result['error']); // docs/SPATIAL.md error table } /** @@ -323,6 +345,7 @@ public function testInvalidType(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertFalse($result['valid']); $this->assertStringContainsString('unknown', strtolower($result['error'])); + $this->assertSame('Unknown geometry type: InvalidType', $result['error']); // docs: "Unknown geometry type: X" } /** @@ -332,6 +355,11 @@ public function testNonArrayInput(): void { $result = $this->noneDB->validateGeoJSON('not an array'); $this->assertFalse($result['valid']); + $this->assertIsString($result['error']); + $this->assertNull($result['type']); + foreach ([null, 42, 1.5, true, new \stdClass()] as $value) { + $this->assertFalse($this->noneDB->validateGeoJSON($value)['valid'], gettype($value)); + } } /** @@ -346,5 +374,169 @@ public function testPoint3D(): void $result = $this->noneDB->validateGeoJSON($geometry); $this->assertTrue($result['valid']); + $this->assertSame(['valid' => true, 'error' => null, 'type' => 'Point'], $result); + } + + // ========== Validation rules (docs/SPATIAL.md "GeoJSON Data Format" and "Error Handling") ========== + + private function assertInvalid(array $geometry, ?string $error = null, string $contains = ''): void + { + $result = $this->noneDB->validateGeoJSON($geometry); + $label = json_encode($geometry); + $this->assertFalse($result['valid'], $label); + $this->assertNull($result['type'], $label); + $this->assertIsString($result['error'], $label); + if ($error !== null) { + $this->assertSame($error, $result['error'], $label); + } + if ($contains !== '') { + $this->assertStringContainsString($contains, $result['error'], $label); + } + } + + private function assertValid(array $geometry, string $type): void + { + $this->assertSame(['valid' => true, 'error' => null, 'type' => $type], $this->noneDB->validateGeoJSON($geometry), json_encode($geometry)); + } + + /** + * Longitude -180..180 and latitude -90..90, both ends included + */ + public function testCoordinateRangesIncludeTheirEnds(): void + { + foreach ([[180, 90], [-180, -90], [180, -90], [-180, 90], [0, 0], [-0.0, -0.0]] as $p) { + $this->assertValid(['type' => 'Point', 'coordinates' => $p], 'Point'); + } + foreach ([[180.0000001, 0], [-180.0000001, 0], [360, 0]] as $p) { + $this->assertInvalid(['type' => 'Point', 'coordinates' => $p], 'Longitude must be between -180 and 180'); + } + foreach ([[0, 90.0000001], [0, -90.0000001], [0, 180]] as $p) { + $this->assertInvalid(['type' => 'Point', 'coordinates' => $p], 'Latitude must be between -90 and 90'); + } + $this->assertInvalid(['type' => 'LineString', 'coordinates' => [[0, 0], [181, 0]]], null, 'Longitude must be between -180 and 180'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], [1, 91], [0, 0]]]], null, 'Latitude must be between -90 and 90'); + } + + /** + * docs: "Point coordinates must be finite numbers" for NaN, INF or strings + */ + public function testCoordinatesMustBeFiniteNumbers(): void + { + foreach ([[NAN, 0], [0, NAN], [INF, 0], [0, -INF], ['1', '2'], ['28.9', 41.0], [true, 1], [1, null], [[1], [2]], ['a', 'b']] as $p) { + $this->assertInvalid(['type' => 'Point', 'coordinates' => $p], 'Point coordinates must be finite numbers'); + } + $this->assertInvalid(['type' => 'LineString', 'coordinates' => [[0, 0], [NAN, 1]]], null, 'Point coordinates must be finite numbers'); + $this->assertInvalid(['type' => 'MultiPoint', 'coordinates' => [[0, 0], [1, INF]]], null, 'Point coordinates must be finite numbers'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], ['1', 1], [0, 0]]]], null, 'Point coordinates must be finite numbers'); + $this->assertValid(['type' => 'Point', 'coordinates' => [1, 2.5]], 'Point'); + } + + /** + * docs: flat coordinates, string-keyed coordinates and a non-string type are invalid (an invalid result, + * never accepted and never an exception) + */ + public function testMalformedStructuresAreInvalidNotErrors(): void + { + // flat coordinates + $this->assertInvalid(['type' => 'LineString', 'coordinates' => [1, 2]]); + $this->assertInvalid(['type' => 'LineString', 'coordinates' => [1, 2, 3, 4]]); + $this->assertInvalid(['type' => 'MultiPoint', 'coordinates' => [1, 2]]); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => [[1, 2], [3, 4]]]); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => [5, 6]]); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[1, 2], [3, 4], [5, 6], [1, 2]]]); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [1, 2, 3, 4]]); + $this->assertInvalid(['type' => 'MultiPolygon', 'coordinates' => [[[1, 2], [3, 4], [5, 6], [1, 2]]]]); + $this->assertInvalid(['type' => 'MultiPolygon', 'coordinates' => [5]]); + // string keys (JSON objects instead of arrays) + $this->assertInvalid(['type' => 'Point', 'coordinates' => ['lon' => 1, 'lat' => 2]], 'Point coordinates must be finite numbers'); + $this->assertInvalid(['type' => 'LineString', 'coordinates' => ['a' => [0, 0], 'b' => [1, 1]]]); + $this->assertInvalid(['type' => 'MultiPoint', 'coordinates' => ['a' => [0, 0]]]); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => ['a' => [[0, 0], [1, 1]]]]); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => ['outer' => [[0, 0], [1, 0], [1, 1], [0, 0]]]]); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [['a' => [0, 0], 'b' => [1, 0], 'c' => [1, 1], 'd' => [0, 0]]]]); + $this->assertInvalid(['type' => 'MultiPolygon', 'coordinates' => ['p' => [[[0, 0], [1, 0], [1, 1], [0, 0]]]]]); + $this->assertInvalid(['type' => 'GeometryCollection', 'geometries' => ['a' => ['type' => 'Point', 'coordinates' => [0, 0]]]]); + // a list in another order is not a list + $this->assertInvalid(['type' => 'LineString', 'coordinates' => [1 => [1, 1], 0 => [0, 0]]]); + // the type must be a string, and a known one (GeoJSON names are case-sensitive) + foreach ([true, false, 0, 1, 1.5, ['Point']] as $type) { + $this->assertInvalid(['type' => $type, 'coordinates' => [0, 0]]); + } + $this->assertInvalid(['type' => null, 'coordinates' => [0, 0]], 'Missing type property'); + $this->assertInvalid(['type' => 'point', 'coordinates' => [0, 0]], 'Unknown geometry type: point'); + $this->assertInvalid(['type' => 'Feature', 'geometry' => ['type' => 'Point', 'coordinates' => [0, 0]]]); + // coordinates missing or not an array + foreach (['Point', 'LineString', 'Polygon', 'MultiPoint', 'MultiLineString', 'MultiPolygon'] as $type) { + $this->assertInvalid(['type' => $type]); + $this->assertInvalid(['type' => $type, 'coordinates' => 'x'], 'Coordinates must be an array'); + $this->assertInvalid(['type' => $type, 'coordinates' => null]); + } + } + + /** + * docs: "A third value (altitude) is accepted and ignored" + */ + public function testAltitudeIsAcceptedInEveryType(): void + { + $this->assertValid(['type' => 'Point', 'coordinates' => [1, 2, 3]], 'Point'); + $this->assertValid(['type' => 'LineString', 'coordinates' => [[0, 0, 10], [1, 1, 20]]], 'LineString'); + $this->assertValid(['type' => 'MultiPoint', 'coordinates' => [[0, 0, 5]]], 'MultiPoint'); + $this->assertValid(['type' => 'Polygon', 'coordinates' => [[[0, 0, 1], [1, 0, 1], [1, 1, 1], [0, 0, 1]]]], 'Polygon'); + $this->assertValid(['type' => 'Polygon', 'coordinates' => [[[0, 0, 1], [1, 0], [1, 1, 7], [0, 0, 99]]]], 'Polygon'); // closed: lon/lat compared + } + + /** + * docs: rings need 4+ positions, first == last (numerically), no self-intersection; holes inside or on the + * outer ring; the errors name the ring + */ + public function testPolygonRingRules(): void + { + $outer = [[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]; + $this->assertValid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [4, 0], [0, 3], [0, 0]]]], 'Polygon'); // a triangle: 4 positions + $this->assertValid(['type' => 'Polygon', 'coordinates' => [[[29, 41], [30, 41], [30, 42], [29.0, 41.0]]]], 'Polygon'); // 29 == 29.0 + $this->assertValid(['type' => 'Polygon', 'coordinates' => [$outer, [[0, 0], [5, 0], [0, 5], [0, 0]]]], 'Polygon'); // hole vertices on the outer ring + $this->assertValid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [6, 5], [6, 6], [5, 5]]]], 'Polygon'); + + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => []], null); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], [0, 0]]]], 'Ring 0 must have at least 4 positions'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[]]], 'Ring 0 must have at least 4 positions'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], [1, 1], [0, 1]]]], 'Ring 0 must be closed (first and last positions must be identical)'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [1, 0], [1, 1], [0, 0.0000001]]]], 'Ring 0 must be closed (first and last positions must be identical)'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [10, 10], [10, 0], [0, 10], [0, 0]]]], 'Ring 0 must not intersect itself'); // bow-tie + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [[[0, 0], [2, 0], [1, 1], [2, 2], [0, 2], [1, 1], [0, 0]]]], 'Ring 0 must not intersect itself'); // touches itself + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [2, 1], [2, 2]]]], 'Ring 1 must have at least 4 positions'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [2, 1], [2, 2], [1, 2]]]], 'Ring 1 must be closed (first and last positions must be identical)'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [5, 5], [5, 1], [1, 5], [1, 1]]]], 'Ring 1 must not intersect itself'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[20, 20], [21, 20], [21, 21], [20, 20]]]], 'Hole 1 must lie inside the outer ring'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[5, 5], [11, 5], [5, 6], [5, 5]]]], 'Hole 1 must lie inside the outer ring'); // one vertex out + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [10.5, 5], [5, 6], [5, 5]]]], 'Hole 2 must lie inside the outer ring'); + $this->assertInvalid(['type' => 'Polygon', 'coordinates' => [$outer, [[1, 1], [2, 1], [1, NAN], [1, 1]]]], null, 'Invalid position at ring 1, index 2'); + } + + /** + * Multi-part types need at least one valid member; GeometryCollection may be empty (docs) + */ + public function testMultiPartAndCollectionRules(): void + { + $this->assertInvalid(['type' => 'MultiPoint', 'coordinates' => []]); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => []]); + $this->assertInvalid(['type' => 'MultiPolygon', 'coordinates' => []]); + $this->assertInvalid(['type' => 'MultiPoint', 'coordinates' => [[0, 0], [200, 0]]], null, 'Longitude must be between -180 and 180'); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => [[[0, 0], [1, 1]], [[0, 0]]]], null); + $this->assertInvalid(['type' => 'MultiLineString', 'coordinates' => [[[0, 0], [1, 1]], [[0, 0], [1, 95]]]], null, 'Latitude must be between -90 and 90'); + $this->assertInvalid(['type' => 'MultiPolygon', 'coordinates' => [[[[0, 0], [1, 0], [1, 1], [0, 0]]], [[[0, 0], [1, 0], [1, 1], [0, 1]]]]], null, + 'Ring 0 must be closed (first and last positions must be identical)'); + $this->assertValid(['type' => 'MultiLineString', 'coordinates' => [[[0, 0], [1, 1]]]], 'MultiLineString'); + + // "An empty collection is valid but has no location" + $this->assertValid(['type' => 'GeometryCollection', 'geometries' => []], 'GeometryCollection'); + $this->assertValid(['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => []]]], 'GeometryCollection'); + $this->assertValid(['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => [['type' => 'Point', 'coordinates' => [1, 1]]]]]], 'GeometryCollection'); + $this->assertInvalid(['type' => 'GeometryCollection']); + $this->assertInvalid(['type' => 'GeometryCollection', 'geometries' => 'x']); + $this->assertInvalid(['type' => 'GeometryCollection', 'geometries' => [5]]); + $this->assertInvalid(['type' => 'GeometryCollection', 'geometries' => ['Point']]); + $this->assertInvalid(['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => [['type' => 'Point', 'coordinates' => [0, 91]]]]]], null, + 'Latitude must be between -90 and 90'); } } diff --git a/tests/Unit/GeometryOperationsTest.php b/tests/Unit/GeometryOperationsTest.php index d672700..be304a1 100644 --- a/tests/Unit/GeometryOperationsTest.php +++ b/tests/Unit/GeometryOperationsTest.php @@ -2,12 +2,12 @@ /** * Geometry Operations Unit Tests * Tests MBR calculation, Haversine distance, Point-in-polygon, etc. - * @version 3.1.0 */ namespace noneDB\Tests\Unit; use noneDB\Tests\noneDBTestCase; +use noneDB\Tests\Integration\SpecSpatialTest as Model; class GeometryOperationsTest extends noneDBTestCase { @@ -32,6 +32,7 @@ public function testCalculateMBRPoint(): void $this->assertEquals(41.0086, $mbr[1]); // minLat $this->assertEquals(28.9803, $mbr[2]); // maxLon $this->assertEquals(41.0086, $mbr[3]); // maxLat + $this->assertSame([28.9803, 41.0086, 28.9803, 41.0086], $mbr); } /** @@ -56,6 +57,7 @@ public function testCalculateMBRLineString(): void $this->assertEquals(40.0, $mbr[1]); // minLat $this->assertEquals(30.0, $mbr[2]); // maxLon $this->assertEquals(41.0, $mbr[3]); // maxLat + $this->assertSame([28.0, 40.0, 30.0, 41.0], $mbr); } /** @@ -82,6 +84,37 @@ public function testCalculateMBRPolygon(): void $this->assertEquals(40.0, $mbr[1]); $this->assertEquals(30.0, $mbr[2]); $this->assertEquals(42.0, $mbr[3]); + $this->assertSame([28.0, 40.0, 30.0, 42.0], $mbr); + } + + /** + * MBR of the multi-part types and collections: every part counts; no position gives null + * (docs: an empty GeometryCollection has no location) + */ + public function testCalculateMBRMultiPartsAndCollections(): void + { + $method = $this->getPrivateMethod('calculateMBR'); + $cases = [ + [['type' => 'MultiPoint', 'coordinates' => [[3, 4], [-1, 2], [5, -6]]], [-1, -6, 5, 4]], + [['type' => 'MultiLineString', 'coordinates' => [[[0, 0], [1, 1]], [[10, -5], [12, 3]]]], [0, -5, 12, 3]], + [['type' => 'MultiPolygon', 'coordinates' => [ + [[[0, 0], [1, 0], [1, 1], [0, 0]]], + [[[5, 5], [8, 5], [8, 9], [5, 5]], [[6, 6], [7, 6], [7, 7], [6, 6]]], + ]], [0, 0, 8, 9]], + [['type' => 'Polygon', 'coordinates' => [[[0, 0], [10, 0], [10, 10], [0, 0]], [[6, 2], [8, 2], [8, 4], [6, 2]]]], [0, 0, 10, 10]], + [['type' => 'GeometryCollection', 'geometries' => [ + ['type' => 'Point', 'coordinates' => [50, 50]], + ['type' => 'GeometryCollection', 'geometries' => []], + ['type' => 'LineString', 'coordinates' => [[60, 60], [61, 49]]], + ]], [50, 49, 61, 60]], + [['type' => 'Point', 'coordinates' => [1, 2, 300]], [1, 2, 1, 2]], + ]; + foreach ($cases as [$geometry, $expected]) { + $this->assertEquals($expected, $method->invoke($this->noneDB, $geometry), json_encode($geometry)); + $this->assertEquals(Model::bbox($geometry), $method->invoke($this->noneDB, $geometry), 'model ' . json_encode($geometry)); + } + $this->assertNull($method->invoke($this->noneDB, ['type' => 'GeometryCollection', 'geometries' => []])); + $this->assertNull($method->invoke($this->noneDB, ['type' => 'GeometryCollection', 'geometries' => [['type' => 'GeometryCollection', 'geometries' => []]]])); } /** @@ -100,6 +133,7 @@ public function testMBRUnion(): void $this->assertEquals(40.0, $union[1]); // min of minLats $this->assertEquals(30.0, $union[2]); // max of maxLons $this->assertEquals(42.0, $union[3]); // max of maxLats + $this->assertSame([28.0, 40.0, 30.0, 42.0], $union); } /** @@ -114,6 +148,11 @@ public function testMBROverlapsTrue(): void $overlaps = $method->invoke($this->noneDB, $mbr1, $mbr2); $this->assertTrue($overlaps); + // docs (withinBBox): boxes that only touch count as well, so does containment + $this->assertTrue($method->invoke($this->noneDB, [0.0, 0.0, 1.0, 1.0], [1.0, 0.5, 2.0, 2.0]), 'touching edge'); + $this->assertTrue($method->invoke($this->noneDB, [0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 2.0, 2.0]), 'touching corner'); + $this->assertTrue($method->invoke($this->noneDB, [0.0, 0.0, 10.0, 10.0], [2.0, 2.0, 3.0, 3.0]), 'contained'); + $this->assertTrue($method->invoke($this->noneDB, [5.0, 5.0, 5.0, 5.0], [5.0, 5.0, 5.0, 5.0]), 'equal points'); } /** @@ -128,19 +167,9 @@ public function testMBROverlapsFalse(): void $overlaps = $method->invoke($this->noneDB, $mbr1, $mbr2); $this->assertFalse($overlaps); - } - - /** - * Test MBR area calculation - */ - public function testMBRArea(): void - { - $method = $this->getPrivateMethod('mbrArea'); - - $mbr = [0.0, 0.0, 2.0, 3.0]; // 2 x 3 = 6 - - $area = $method->invoke($this->noneDB, $mbr); - $this->assertEquals(6.0, $area); + $this->assertFalse($method->invoke($this->noneDB, [0.0, 0.0, 1.0, 1.0], [1.0000001, 0.0, 2.0, 1.0]), 'apart in x only'); + $this->assertFalse($method->invoke($this->noneDB, [0.0, 0.0, 1.0, 1.0], [0.0, 1.0000001, 1.0, 2.0]), 'apart in y only'); + $this->assertFalse($method->invoke($this->noneDB, [2.0, 2.0, 3.0, 3.0], [0.0, 0.0, 1.0, 1.0]), 'either order'); } // ========== Haversine Distance Tests ========== @@ -158,6 +187,11 @@ public function testHaversineDistance(): void // Allow 10km tolerance (in meters) $this->assertGreaterThan(340000, $distance); $this->assertLessThan(360000, $distance); + // docs: haversine with earth radius 6,371 km (independent implementation) + $this->assertEqualsWithDelta(Model::hav(28.9784, 41.0082, 32.8597, 39.9334), $distance, 1e-6); + $this->assertEqualsWithDelta(6371000 * M_PI / 180, $method->invoke($this->noneDB, 0, 0, 1, 0), 1e-6, 'one degree on the equator'); + $this->assertEqualsWithDelta(6371000 * M_PI, $method->invoke($this->noneDB, 10, 20, -170, -20), 1e-6, 'antipodes'); + $this->assertEqualsWithDelta($distance, $method->invoke($this->noneDB, 32.8597, 39.9334, 28.9784, 41.0082), 1e-9, 'symmetric'); } /** @@ -170,6 +204,8 @@ public function testHaversineDistanceSamePoint(): void $distance = $method->invoke($this->noneDB, 28.9784, 41.0082, 28.9784, 41.0082); $this->assertEquals(0, $distance); + $this->assertSame(0.0, $distance); + $this->assertEqualsWithDelta(0.0, $method->invoke($this->noneDB, 180, 10, -180, 10), 1e-6, 'the same meridian (sin(pi) is not exactly 0)'); } /** @@ -177,10 +213,12 @@ public function testHaversineDistanceSamePoint(): void */ public function testCircleToBBox(): void { - $method = $this->getPrivateMethod('circleToBBox'); + $method = $this->getPrivateMethod('circleToBBoxes'); - // 100km radius circle around Istanbul - $bbox = $method->invoke($this->noneDB, 28.9784, 41.0082, 100); + // 100km radius circle around Istanbul (one box: no antimeridian, no pole) + $boxes = $method->invoke($this->noneDB, 28.9784, 41.0082, 100000.0); + $this->assertCount(1, $boxes); + $bbox = $boxes[0]; // BBox should extend approximately 100km in each direction // At ~41 degrees latitude, 1 degree longitude ≈ 85km @@ -191,6 +229,53 @@ public function testCircleToBBox(): void $this->assertGreaterThan(41.0082, $bbox[3]); // maxLat } + /** + * Destination point on the sphere (independent of noneDB): start, bearing (radians), distance (m) + */ + private static function destination(float $lon, float $lat, float $bearing, float $meters): array + { + $d = $meters / 6371000.0; + $p1 = deg2rad($lat); + $p2 = asin(sin($p1) * cos($d) + cos($p1) * sin($d) * cos($bearing)); + $l2 = deg2rad($lon) + atan2(sin($bearing) * sin($d) * cos($p1), cos($d) - sin($p1) * sin($p2)); + $lon2 = fmod(rad2deg($l2) + 540.0, 360.0) - 180.0; + return [$lon2, rad2deg($p2)]; + } + + /** + * docs: "Circles that cross the antimeridian (±180°) or reach a pole are searched correctly": the search + * boxes cover every point of the circle + */ + public function testCircleSearchBoxesCoverTheCircle(): void + { + $method = $this->getPrivateMethod('circleToBBoxes'); + $cases = [ + [28.9784, 41.0082, 100000.0, 1], [179.9, 10.0, 50000.0, 2], [-179.9, -10.0, 50000.0, 2], + [0.0, 89.5, 100000.0, 1], [120.0, -89.9, 30000.0, 1], [0.0, 0.0, 5000000.0, 1], [170.0, 60.0, 2000000.0, 2], + ]; + foreach ($cases as [$lon, $lat, $radius, $count]) { + $boxes = $method->invoke($this->noneDB, $lon, $lat, $radius); + $this->assertCount($count, $boxes, "($lon, $lat, $radius): one box, or two split at the antimeridian"); + foreach ($boxes as $box) { + $this->assertGreaterThanOrEqual(-180.0, $box[0]); + $this->assertLessThanOrEqual(180.0, $box[2]); + $this->assertGreaterThanOrEqual(-90.0, $box[1]); + $this->assertLessThanOrEqual(90.0, $box[3]); + } + for ($i = 0; $i < 360; $i += 5) { + foreach ([1.0, 0.5] as $fraction) { + [$x, $y] = self::destination($lon, $lat, deg2rad($i), $radius * $fraction); + $this->assertEqualsWithDelta($radius * $fraction, Model::hav($lon, $lat, $x, $y), 0.01, 'setup: on the circle'); + $inside = false; + foreach ($boxes as $box) { + $inside = $inside || ($box[0] <= $x && $x <= $box[2] && $box[1] <= $y && $y <= $box[3]); + } + $this->assertTrue($inside, "($lon, $lat, $radius): bearing $i at " . ($fraction * 100) . "% = ($x, $y) is covered by " . json_encode($boxes)); + } + } + } + } + // ========== Point in Polygon Tests ========== /** @@ -247,6 +332,25 @@ public function testPointOnPolygonEdge(): void $inside = $method->invoke($this->noneDB, 5, 0, $polygon); // Edge behavior may vary, but typically inside $this->assertTrue($inside); + + // docs (withinPolygon): "The polygon is closed: points on its boundary are inside, on every edge" + $closed = $this->getPrivateMethod('pointInPolygonClosed'); + $coords = $polygon['coordinates']; + foreach ([[5, 0], [10, 5], [5, 10], [0, 5], [0, 0], [10, 0], [10, 10], [0, 10]] as [$x, $y]) { + $this->assertTrue($closed->invoke($this->noneDB, $x, $y, $coords), "boundary point ($x, $y)"); + } + foreach ([[5, -0.0000001], [10.0000001, 5], [5, 10.0000001], [-0.0000001, 5]] as [$x, $y]) { + $this->assertFalse($closed->invoke($this->noneDB, $x, $y, $coords), "outside ($x, $y)"); + } + $triangle = [[[0, 0], [8, 0], [0, 8], [0, 0]]]; + foreach ([[4, 4], [2, 6], [0.5, 7.5], [7.75, 0.25]] as [$x, $y]) { + $this->assertTrue($closed->invoke($this->noneDB, $x, $y, $triangle), "on the diagonal ($x, $y)"); + } + $this->assertFalse($closed->invoke($this->noneDB, 4.0000001, 4, $triangle)); + $withHole = [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]], [[3, 3], [7, 3], [7, 7], [3, 7], [3, 3]]]; + $this->assertTrue($closed->invoke($this->noneDB, 3, 5, $withHole), 'a hole\'s boundary belongs to the polygon'); + $this->assertTrue($closed->invoke($this->noneDB, 7, 7, $withHole)); + $this->assertFalse($closed->invoke($this->noneDB, 5, 5, $withHole), 'inside the hole'); } /** @@ -280,13 +384,13 @@ public function testPointInPolygonWithHole(): void */ public function testLineSegmentsIntersect(): void { - $method = $this->getPrivateMethod('lineSegmentsIntersect'); + $method = $this->getPrivateMethod('segmentsMeet'); // X pattern $intersects = $method->invoke( - $this->noneDB, - [0, 0], [10, 10], // Line 1: diagonal - [0, 10], [10, 0] // Line 2: other diagonal + null, + 0, 0, 10, 10, // Line 1: diagonal + 0, 10, 10, 0 // Line 2: other diagonal ); $this->assertTrue($intersects); @@ -297,12 +401,12 @@ public function testLineSegmentsIntersect(): void */ public function testLineSegmentsParallel(): void { - $method = $this->getPrivateMethod('lineSegmentsIntersect'); + $method = $this->getPrivateMethod('segmentsMeet'); $intersects = $method->invoke( - $this->noneDB, - [0, 0], [10, 0], // Horizontal line at y=0 - [0, 5], [10, 5] // Horizontal line at y=5 + null, + 0, 0, 10, 0, // Horizontal line at y=0 + 0, 5, 10, 5 // Horizontal line at y=5 ); $this->assertFalse($intersects); @@ -313,117 +417,37 @@ public function testLineSegmentsParallel(): void */ public function testLineSegmentsNoIntersect(): void { - $method = $this->getPrivateMethod('lineSegmentsIntersect'); + $method = $this->getPrivateMethod('segmentsMeet'); $intersects = $method->invoke( - $this->noneDB, - [0, 0], [5, 5], // Short diagonal - [6, 0], [10, 4] // Other segment far away + null, + 0, 0, 5, 5, // Short diagonal + 6, 0, 10, 4 // Other segment far away ); $this->assertFalse($intersects); } - // ========== Polygon Intersection Tests ========== - - /** - * Test overlapping polygons - */ - public function testPolygonsIntersectOverlapping(): void - { - $method = $this->getPrivateMethod('polygonsIntersect'); - - $poly1 = [ - 'type' => 'Polygon', - 'coordinates' => [[[0, 0], [10, 0], [10, 10], [0, 10], [0, 0]]] - ]; - $poly2 = [ - 'type' => 'Polygon', - 'coordinates' => [[[5, 5], [15, 5], [15, 15], [5, 15], [5, 5]]] - ]; - - $intersects = $method->invoke($this->noneDB, $poly1, $poly2); - $this->assertTrue($intersects); - } - - /** - * Test non-overlapping polygons - */ - public function testPolygonsIntersectNonOverlapping(): void - { - $method = $this->getPrivateMethod('polygonsIntersect'); - - $poly1 = [ - 'type' => 'Polygon', - 'coordinates' => [[[0, 0], [5, 0], [5, 5], [0, 5], [0, 0]]] - ]; - $poly2 = [ - 'type' => 'Polygon', - 'coordinates' => [[[10, 10], [15, 10], [15, 15], [10, 15], [10, 10]]] - ]; - - $intersects = $method->invoke($this->noneDB, $poly1, $poly2); - $this->assertFalse($intersects); - } - - /** - * Test one polygon inside another - */ - public function testPolygonsIntersectContained(): void - { - $method = $this->getPrivateMethod('polygonsIntersect'); - - $poly1 = [ - 'type' => 'Polygon', - 'coordinates' => [[[0, 0], [20, 0], [20, 20], [0, 20], [0, 0]]] - ]; - $poly2 = [ - 'type' => 'Polygon', - 'coordinates' => [[[5, 5], [15, 5], [15, 15], [5, 15], [5, 5]]] - ]; - - $intersects = $method->invoke($this->noneDB, $poly1, $poly2); - $this->assertTrue($intersects); - } - - // ========== Centroid Tests ========== - /** - * Test centroid calculation for Point + * docs (Validation Cost): edges "share a point" when they cross, touch or overlap: closed segments, exact */ - public function testGetGeometryCentroidPoint(): void + public function testSegmentsMeetAtSharedPointsOnly(): void { - $method = $this->getPrivateMethod('getGeometryCentroid'); - - $point = [ - 'type' => 'Point', - 'coordinates' => [28.9803, 41.0086] + $method = $this->getPrivateMethod('segmentsMeet'); + $cases = [ + [[0, 0, 10, 0, 10, 0, 20, 5], true], // a shared end point + [[0, 0, 10, 0, 5, 0, 5, 5], true], // T: an end on the other's inside + [[0, 0, 10, 0, 5, 0, 15, 0], true], // collinear overlap + [[0, 0, 10, 0, 10, 0, 15, 0], true], // collinear, touching ends + [[0, 0, 10, 0, 11, 0, 15, 0], false], // collinear, apart + [[0, 0, 10, 10, 5, 5.0000001, 6, 6.0000001], false], // parallel, just apart + [[0, 0, 10, 10, 10, 0, 6, 4.0000001], false], // ends just short of the other + [[0, 0, 10, 10, 10, 0, 5, 5], true], // ends on the other + [[0, 0, 0, 10, 0, 5, 0, 7], true], // vertical, contained ]; - - $centroid = $method->invoke($this->noneDB, $point); - - $this->assertEquals(28.9803, $centroid[0]); - $this->assertEquals(41.0086, $centroid[1]); - } - - /** - * Test centroid calculation for Polygon - */ - public function testGetGeometryCentroidPolygon(): void - { - $method = $this->getPrivateMethod('getGeometryCentroid'); - - $polygon = [ - 'type' => 'Polygon', - 'coordinates' => [[ - [0, 0], [10, 0], [10, 10], [0, 10], [0, 0] - ]] - ]; - - $centroid = $method->invoke($this->noneDB, $polygon); - - // Centroid of a square should be at center - $this->assertEquals(5.0, $centroid[0]); - $this->assertEquals(5.0, $centroid[1]); + foreach ($cases as [$p, $expected]) { + $this->assertSame($expected, $method->invoke(null, ...$p), json_encode($p)); + $this->assertSame($expected, $method->invoke(null, $p[4], $p[5], $p[6], $p[7], $p[0], $p[1], $p[2], $p[3]), 'swapped ' . json_encode($p)); + } } } diff --git a/tests/Unit/GeometryValidationTest.php b/tests/Unit/GeometryValidationTest.php new file mode 100644 index 0000000..f4d0ba0 --- /dev/null +++ b/tests/Unit/GeometryValidationTest.php @@ -0,0 +1,488 @@ +counter('sweepBudget')->setValue(null, $budget); + } + + protected function tearDown(): void + { + if (property_exists(\noneDB::class, 'sweepBudget')) { + $this->setSweepBudget(null); + } + parent::tearDown(); + } + + private function call(string $method, ...$args) + { + return \noneDB\Tests\Support\Testing::call($this->db(), $method, ...$args); + } + + private const STEP = 1 / 64; + + /** + * A comb: $teeth edges zigzag between x = 0 and x = 100, STEP apart in y (every edge overlaps every + * other in x, the sort-and-sweep compared all pairs), closed around the right side + */ + private static function comb(int $teeth): array + { + $ring = []; + for ($i = 0; $i < $teeth; $i++) { + $ring[] = [$i % 2 === 0 ? 0 : 100, $i * self::STEP]; + } + $last = ($teeth - 1) * self::STEP; + $ring[] = [150, $last]; + $ring[] = [150, -1]; + $ring[] = [0, -1]; + $ring[] = $ring[0]; + return $ring; + } + + private static function log2(int $n): float + { + return log(max(2, $n), 2); + } + + /** @test */ + public function ringBuiltAgainstTheSweepIsCheckedInNearLinearWork(): void + { + $pairs = $this->counter('segmentPairTests'); + $db = $this->db(); + $n = 3000; + $ring = $this->comb($n); + $edges = count($ring) - 1; + + $pairs->setValue(null, 0); + $this->assertTrue($db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [$ring]])['valid']); + $this->assertLessThan(40 * $edges * self::log2($edges), $pairs->getValue(), + 'edge pairs compared (all pairs: about 4,500,000)'); + + // The same comb with one tooth reaching into the next one: found, also at the far end + foreach ([3, (int)($n / 2), $n - 3] as $tooth) { + $bad = $ring; + $bad[$tooth][1] += 2.5 * self::STEP; // above the vertex two steps up: crosses the edge leaving it + $pairs->setValue(null, 0); + $this->assertFalse($db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [$bad]])['valid'], "tooth $tooth"); + $this->assertLessThan(40 * $edges * self::log2($edges), $pairs->getValue()); + } + } + + /** @test */ + public function manyHolesAreCheckedInNearLinearWork(): void + { + $points = $this->counter('pointEdgeTests'); + $db = $this->db(); + $outer = []; + $n = 4000; + for ($i = 0; $i < $n; $i++) { + $outer[] = [cos(2 * M_PI * $i / $n) * 50, sin(2 * M_PI * $i / $n) * 50]; + } + $outer[] = $outer[0]; + $holes = []; + for ($i = 0; $i < 30; $i++) { + for ($j = 0; $j < 30; $j++) { + $x = -30 + 2 * $i; + $y = -30 + 2 * $j; + $holes[] = [[$x, $y], [$x + 1, $y], [$x, $y + 1], [$x, $y]]; + } + } + $vertices = $n + 4 * count($holes); + + $points->setValue(null, 0); + $this->assertTrue($db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => array_merge([$outer], $holes)])['valid']); + $this->assertGreaterThan(0, $points->getValue()); + $this->assertLessThan(40 * $vertices * self::log2($vertices), $points->getValue(), + 'point-edge tests (each hole vertex against every edge: about 28,800,000)'); + + $holes[450] = [[60, 0], [61, 0], [60, 1], [60, 0]]; + $points->setValue(null, 0); + $result = $db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => array_merge([$outer], $holes)]); + $this->assertFalse($result['valid']); + $this->assertSame('Hole 451 must lie inside the outer ring', $result['error']); + $this->assertLessThan(40 * $vertices * self::log2($vertices), $points->getValue()); + } + + /** All-pairs reference (exact for the small integers and dyadic values used here) */ + private static function refRingIntersects(array $ring): bool + { + $n = count($ring) - 1; + for ($i = 0; $i < $n; $i++) { + for ($j = $i + 1; $j < $n; $j++) { + $d = $j - $i; + if ($d === 1 || $d === $n - 1) { + continue; + } + if (self::refMeet($ring[$i], $ring[$i + 1], $ring[$j], $ring[$j + 1])) { + return true; + } + } + } + return false; + } + + private static function refOrient(array $a, array $b, array $c): int + { + $v = ($b[0] - $a[0]) * ($c[1] - $a[1]) - ($b[1] - $a[1]) * ($c[0] - $a[0]); + return $v <=> 0; + } + + private static function refOnBox(array $a, array $b, array $p): bool + { + return min($a[0], $b[0]) <= $p[0] && $p[0] <= max($a[0], $b[0]) && min($a[1], $b[1]) <= $p[1] && $p[1] <= max($a[1], $b[1]); + } + + private static function refMeet(array $p, array $q, array $r, array $s): bool + { + $o1 = self::refOrient($p, $q, $r); + $o2 = self::refOrient($p, $q, $s); + $o3 = self::refOrient($r, $s, $p); + $o4 = self::refOrient($r, $s, $q); + if ($o1 * $o2 < 0 && $o3 * $o4 < 0) { + return true; + } + return ($o1 === 0 && self::refOnBox($p, $q, $r)) || ($o2 === 0 && self::refOnBox($p, $q, $s)) + || ($o3 === 0 && self::refOnBox($r, $s, $p)) || ($o4 === 0 && self::refOnBox($r, $s, $q)); + } + + /** 1 inside, 0 on the boundary, -1 outside (winding number, horizontal ray) */ + private static function refLocate(array $p, array $ring): int + { + $n = count($ring) - 1; + $winding = 0; + for ($i = 0; $i < $n; $i++) { + $a = $ring[$i]; + $b = $ring[$i + 1]; + if (self::refOrient($a, $b, $p) === 0 && self::refOnBox($a, $b, $p)) { + return 0; + } + if ($a[1] <= $p[1]) { + if ($b[1] > $p[1] && self::refOrient($a, $b, $p) > 0) { + $winding++; + } + } elseif ($b[1] <= $p[1] && self::refOrient($a, $b, $p) < 0) { + $winding--; + } + } + return $winding !== 0 ? 1 : -1; + } + + private static function randomRing(int $size, int $grid): array + { + $ring = []; + for ($i = 0; $i < $size; $i++) { + $ring[] = [mt_rand(0, $grid), mt_rand(0, $grid)]; + } + $ring[] = $ring[0]; + return $ring; + } + + /** A star-shaped (simple) ring around ($cx, $cy) */ + private static function starRing(int $size, float $cx, float $cy, float $radius, bool $integer, bool $dyadic = false): array + { + $angles = []; + for ($i = 0; $i < $size; $i++) { + $angles[] = mt_rand(0, 1000000) / 1000000 * 2 * M_PI; + } + sort($angles); + $ring = []; + foreach ($angles as $a) { + $r = $radius * (0.3 + 0.7 * mt_rand(0, 1000) / 1000); + $x = $cx + $r * cos($a); + $y = $cy + $r * sin($a); + if ($integer) { + $ring[] = [(int)round($x), (int)round($y)]; + } else { + $ring[] = $dyadic ? [round($x * 4096) / 4096, round($y * 4096) / 4096] : [$x, $y]; // dyadic: exact midpoints + } + } + $ring[] = $ring[0]; + return $ring; + } + + /** @test */ + public function sweepEqualsAllPairsOnRandomRings(): void + { + $checked = ['valid' => 0, 'invalid' => 0]; + foreach ([null, 0] as $budget) { + $this->setSweepBudget($budget); + mt_srand(20261026); + for ($round = 0; $round < 3000; $round++) { + $kind = $round % 4; + if ($kind === 0) { + $ring = self::randomRing(mt_rand(3, 12), mt_rand(2, 7)); // many collinear, touching, repeated + } elseif ($kind === 1) { + $ring = self::starRing(mt_rand(3, 40), 20, 20, 15, true); // mostly valid, some degenerate + } elseif ($kind === 2) { + $ring = self::starRing(mt_rand(3, 60), 0.5, 0.25, 0.2, false); // floats + } else { + $ring = self::comb(mt_rand(2, 30)); + $k = mt_rand(0, count($ring) - 2); + $ring[$k] = [$ring[$k][0] + mt_rand(-2, 2) * 25, $ring[$k][1] + mt_rand(-2, 2) * self::STEP / 2]; + if ($k === 0) { + $ring[count($ring) - 1] = $ring[0]; + } + } + $expected = self::refRingIntersects($ring); + $checked[$expected ? 'invalid' : 'valid']++; + $this->assertSame($expected, $this->call('ringSelfIntersects', $ring), + 'budget ' . var_export($budget, true) . ': ' . json_encode($ring)); + } + } + $this->assertGreaterThan(500, $checked['valid']); + $this->assertGreaterThan(500, $checked['invalid']); + } + + /** @test */ + public function holeCheckEqualsTheDefinitionOnRandomPolygons(): void + { + foreach ([null, 0] as $budget) { + $this->setSweepBudget($budget); + mt_srand(7); + $seen = [1 => 0, 0 => 0, -1 => 0]; + for ($round = 0; $round < 400; $round++) { + $outer = self::starRing(mt_rand(4, 30), 10, 10, 9, $round % 2 === 0, true); + if (self::refRingIntersects($outer)) { + continue; + } + $points = []; + for ($i = 0; $i < 40; $i++) { + if ($i % 5 === 0) { + $points[] = $outer[mt_rand(0, count($outer) - 2)]; // a vertex + } elseif ($i % 5 === 1) { + $e = mt_rand(0, count($outer) - 2); // an edge midpoint + $points[] = [($outer[$e][0] + $outer[$e + 1][0]) / 2, ($outer[$e][1] + $outer[$e + 1][1]) / 2]; + } else { + $points[] = [mt_rand(0, 40) / 2, mt_rand(0, 40) / 2]; + } + } + $expected = array_map(function ($p) use ($outer) { return self::refLocate($p, $outer); }, $points); + foreach ($expected as $e) { + $seen[$e]++; + } + $actual = $this->call('locatePointsInRing', $outer, $points); + $this->assertSame($expected, $actual, 'budget ' . var_export($budget, true) . ': ' . json_encode([$outer, $points])); + + // Through validateGeoJSON: holes made of these points are valid exactly when every vertex is in or on + $hole = [$points[2], $points[3], $points[4], $points[2]]; + $valid = min($expected[2], $expected[3], $expected[4]) >= 0 && !self::refRingIntersects($hole); + $this->assertSame($valid, $this->db()->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [$outer, $hole]])['valid']); + } + $this->assertGreaterThan(100, min($seen)); + } + } + + /** @test */ + public function orientationIsExact(): void + { + // Points near the line y = x (Shewchuk's example): naive floating-point arithmetic gets many wrong + $eps = 2 ** -53; + $wrongNaive = 0; + foreach ([[0.5, 12, 24], [0.5, 17.300000000000001, 24.700000000000003], [1.0, 3, 9]] as [$base, $q, $r]) { + for ($i = 0; $i < 64; $i++) { + for ($j = 0; $j < 64; $j++) { + $px = $base + $i * $eps * ($base >= 1 ? 2 : 1); + $py = $base + $j * $eps * ($base >= 1 ? 2 : 1); + $expected = self::exactOrientSign([$q, $q], [$r, $r], [$px, $py]); + $this->assertSame($expected, $this->call('orient', (float)$q, (float)$q, (float)$r, (float)$r, $px, $py), "$i $j"); + if (self::refOrient([$q, $q], [$r, $r], [$px, $py]) !== $expected) { + $wrongNaive++; + } + } + } + } + $this->assertGreaterThan(100, $wrongNaive, 'the cases are hard for plain floating point'); + + mt_srand(99); + for ($k = 0; $k < 2000; $k++) { + $p = []; + for ($m = 0; $m < 6; $m++) { + $p[] = mt_rand(-(2 ** 30), 2 ** 30) * 2 ** -22 + (mt_rand(0, 1) ? 0.1 : 0); + } + if ($k % 2) { + $p[4] = $p[0] + ($p[2] - $p[0]) * 0.5; // near the line + $p[5] = $p[1] + ($p[3] - $p[1]) * 0.5; + } + $this->assertSame(self::exactOrientSign([$p[0], $p[1]], [$p[2], $p[3]], [$p[4], $p[5]]), + $this->call('orient', $p[0], $p[1], $p[2], $p[3], $p[4], $p[5]), json_encode($p)); + } + } + + /** + * Exact sign of (b - a) x (c - a) for finite doubles, with integer arithmetic: every double is + * m * 2^e; all values are scaled to one exponent and multiplied as base-2^24 digit lists + */ + private static function exactOrientSign(array $a, array $b, array $c): int + { + $values = [$a[0], $a[1], $b[0], $b[1], $c[0], $c[1]]; + $parts = []; + $minExp = PHP_INT_MAX; + foreach ($values as $v) { + [$m, $e] = self::decompose((float)$v); + $parts[] = [$m, $e]; + if ($m !== 0) { + $minExp = min($minExp, $e); + } + } + if ($minExp === PHP_INT_MAX) { + return 0; + } + $big = []; + foreach ($parts as [$m, $e]) { + $big[] = $m === 0 ? [0] : self::bigShift(self::bigFromInt($m), $e - $minExp); + } + $bx = self::bigSub($big[2], $big[0]); + $by = self::bigSub($big[3], $big[1]); + $cx = self::bigSub($big[4], $big[0]); + $cy = self::bigSub($big[5], $big[1]); + $det = self::bigSub(self::bigMul($bx, $cy), self::bigMul($by, $cx)); + return self::bigSign($det); + } + + /** [integer mantissa, exponent] with v = m * 2^e exactly */ + private static function decompose(float $v): array + { + if ($v == 0.0) { + return [0, 0]; + } + $e = 0; + $m = $v; + while (floor($m) != $m) { + $m *= 2; + $e--; + } + while (abs($m) >= 2 ** 53) { + $m /= 2; + $e++; + } + return [(int)$m, $e]; + } + + // Signed big integers: little-endian digit lists in base 2^24 (digits may be negative until normalized) + private static function bigFromInt(int $v): array + { + $digits = []; + $neg = $v < 0; + $v = abs($v); + do { + $digits[] = ($neg ? -1 : 1) * ($v & 0xFFFFFF); + $v >>= 24; + } while ($v > 0); + return $digits; + } + + private static function bigNormalize(array $d): array + { + // Every digit but the top one into [0, 2^24); the top one keeps the sign (|top| < 2^24) + $base = 0x1000000; + $carry = 0; + $count = count($d); + for ($i = 0; $i < $count; $i++) { + $x = $d[$i] + $carry; + $r = (($x % $base) + $base) % $base; + $carry = intdiv($x - $r, $base); + $d[$i] = $r; + } + while ($carry >= $base || $carry <= -$base) { + $r = (($carry % $base) + $base) % $base; + $d[] = $r; + $carry = intdiv($carry - $r, $base); + } + if ($carry !== 0) { + $d[] = $carry; + } + while (count($d) > 1 && end($d) === 0) { + array_pop($d); + } + return $d; + } + + private static function bigShift(array $d, int $bits): array + { + $whole = intdiv($bits, 24); + $rest = $bits % 24; + $out = array_merge(array_fill(0, $whole, 0), array_map(function ($x) use ($rest) { return $x * (1 << $rest); }, $d)); + return self::bigNormalize($out); + } + + private static function bigSub(array $a, array $b): array + { + $n = max(count($a), count($b)); + $out = []; + for ($i = 0; $i < $n; $i++) { + $out[$i] = ($a[$i] ?? 0) - ($b[$i] ?? 0); + } + return self::bigNormalize($out); + } + + private static function bigMul(array $a, array $b): array + { + $out = array_fill(0, count($a) + count($b), 0); + foreach ($a as $i => $x) { + foreach ($b as $j => $y) { + $out[$i + $j] += $x * $y; // |digit| < 2^24: a few products per position stay below 2^53 + } + } + return self::bigNormalize($out); + } + + private static function bigSign(array $d): int + { + // After normalization every digit but the top one is in [0, 2^24); the top one has the sign + $top = end($d); + if ($top !== 0) { + return $top <=> 0; + } + return 0; + } + + /** @test */ + public function knownShapesKeepTheirResult(): void + { + $db = $this->db(); + foreach ([null, 0] as $budget) { + $this->setSweepBudget($budget); + $cases = [ + 'square' => [[[0, 0], [4, 0], [4, 4], [0, 4], [0, 0]], true], + 'bowtie' => [[[0, 0], [2, 2], [2, 0], [0, 2], [0, 0]], false], + 'vertex on edge' => [[[0, 0], [4, 0], [4, 4], [2, 0], [0, 4], [0, 0]], false], + 'repeated vertex' => [[[0, 0], [2, 0], [2, 2], [1, 1], [2, 0], [3, -1], [0, -1], [0, 0]], false], + 'duplicate point' => [[[0, 0], [4, 0], [4, 0], [4, 4], [0, 4], [0, 0]], false], + 'spike' => [[[0, 0], [4, 0], [2, 0], [2, 3], [0, 3], [0, 0]], false], + 'triangle' => [[[0, 0], [4, 0], [0, 3], [0, 0]], true], + 'flat triangle' => [[[0, 0], [1, 0], [2, 0], [0, 0]], true], + 'vertical edges' => [[[0, 0], [0, 4], [1, 4], [1, 1], [2, 1], [2, 4], [3, 4], [3, 0], [0, 0]], true], + 'vertex on a vertical edge' => [[[0, 0], [2, 0], [2, 4], [0, 4], [2, 2], [0, 0]], false], + 'collinear vertex' => [[[0, 0], [2, 0], [4, 0], [4, 4], [0, 4], [0, 0]], true], + 'edges meeting at a vertical' => [[[0, 0], [2, 1], [2, 3], [0, 4], [1, 2], [0, 0]], true], + // The only crossing (x = 10.5) is between two edges that become neighbours in the sweep + // when the wedge between them ends at x = 5 + 'crossing after a wedge' => [[[1, 6], [5, 5], [1, 4], [1, -3], [2, -3], [2, 0], [20, 10], [22, 10], [22, -5], [20, 0], [0, 10], [1, 6]], false], + ]; + foreach ($cases as $name => [$ring, $valid]) { + $result = $db->validateGeoJSON(['type' => 'Polygon', 'coordinates' => [$ring]]); + $this->assertSame($valid, $result['valid'], "$name, budget " . var_export($budget, true)); + $this->assertSame($valid ? null : 'Ring 0 must not intersect itself', $result['error'], "$name: the documented message"); + $this->assertSame($valid, self::refRingIntersects($ring) === false, "$name: the all-pairs definition agrees"); + } + } + } +} diff --git a/tests/Unit/HashDBNameTest.php b/tests/Unit/HashDBNameTest.php deleted file mode 100644 index 52a912d..0000000 --- a/tests/Unit/HashDBNameTest.php +++ /dev/null @@ -1,133 +0,0 @@ -getPrivateMethod('hashDBName'); - $hash = $method->invoke($this->noneDB, 'testdb'); - - $this->assertEquals(20, strlen($hash), 'Hash should be exactly 20 characters'); - } - - /** - * @test - */ - public function hashIsConsistentForSameInput(): void - { - $method = $this->getPrivateMethod('hashDBName'); - - $hash1 = $method->invoke($this->noneDB, 'mydb'); - $hash2 = $method->invoke($this->noneDB, 'mydb'); - - $this->assertEquals($hash1, $hash2, 'Same input should produce same hash'); - } - - /** - * @test - */ - public function differentInputsProduceDifferentHashes(): void - { - $method = $this->getPrivateMethod('hashDBName'); - - $hash1 = $method->invoke($this->noneDB, 'database1'); - $hash2 = $method->invoke($this->noneDB, 'database2'); - - $this->assertNotEquals($hash1, $hash2, 'Different inputs should produce different hashes'); - } - - /** - * @test - */ - public function hashWorksWithEmptyString(): void - { - $method = $this->getPrivateMethod('hashDBName'); - $hash = $method->invoke($this->noneDB, ''); - - $this->assertEquals(20, strlen($hash), 'Empty string should still produce 20 char hash'); - $this->assertIsString($hash, 'Hash should be a string'); - } - - /** - * @test - */ - public function hashWorksWithSpecialCharacters(): void - { - $method = $this->getPrivateMethod('hashDBName'); - $hash = $method->invoke($this->noneDB, "test'db-name 123"); - - $this->assertEquals(20, strlen($hash), 'Special chars should produce 20 char hash'); - } - - /** - * @test - */ - public function hashWorksWithUnicodeCharacters(): void - { - $method = $this->getPrivateMethod('hashDBName'); - $hash = $method->invoke($this->noneDB, 'veritabanı'); - - $this->assertEquals(20, strlen($hash), 'Unicode chars should produce 20 char hash'); - } - - /** - * @test - */ - public function hashWorksWithVeryLongString(): void - { - $method = $this->getPrivateMethod('hashDBName'); - $longString = str_repeat('a', 10000); - $hash = $method->invoke($this->noneDB, $longString); - - $this->assertEquals(20, strlen($hash), 'Very long string should produce 20 char hash'); - } - - /** - * @test - */ - public function hashOnlyContainsHexCharacters(): void - { - $method = $this->getPrivateMethod('hashDBName'); - $hash = $method->invoke($this->noneDB, 'testdb'); - - $this->assertMatchesRegularExpression('/^[a-f0-9]+$/', $hash, 'Hash should only contain hex characters'); - } - - /** - * @test - */ - public function hashIsCaseSensitive(): void - { - $method = $this->getPrivateMethod('hashDBName'); - - $hash1 = $method->invoke($this->noneDB, 'TestDB'); - $hash2 = $method->invoke($this->noneDB, 'testdb'); - - $this->assertNotEquals($hash1, $hash2, 'Hash should be case sensitive'); - } - - /** - * @test - */ - public function hashWithWhitespace(): void - { - $method = $this->getPrivateMethod('hashDBName'); - - $hash1 = $method->invoke($this->noneDB, 'test db'); - $hash2 = $method->invoke($this->noneDB, 'testdb'); - - $this->assertNotEquals($hash1, $hash2, 'Whitespace should affect hash'); - } -} diff --git a/tests/Unit/LimitTest.php b/tests/Unit/LimitTest.php index d938225..c07de31 100644 --- a/tests/Unit/LimitTest.php +++ b/tests/Unit/LimitTest.php @@ -33,8 +33,7 @@ public function validLimitReturnsCorrectCount(): void $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 3); - $this->assertIsArray($result); - $this->assertCount(3, $result); + $this->assertSame(array_slice($array, 0, 3), $result); } /** @@ -45,8 +44,7 @@ public function limitReturnsFirstNElements(): void $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 2); - $this->assertEquals(1, $result[0]['id']); - $this->assertEquals(2, $result[1]['id']); + $this->assertSame([['id' => 1, 'name' => 'Item 1'], ['id' => 2, 'name' => 'Item 2']], $result); } /** @@ -83,6 +81,11 @@ public function limitNonIntegerReturnsFalse(): void $result = $this->noneDB->limit($array, '3'); $this->assertFalse($result, 'String limit should return false'); + + $this->assertFalse($this->noneDB->limit($array, 2.0), 'a float, even a whole one'); + $this->assertFalse($this->noneDB->limit($array, null)); + $this->assertFalse($this->noneDB->limit($array, true)); + $this->assertFalse($this->noneDB->limit($array), 'the default limit 0 is not a limit'); } /** @@ -99,10 +102,12 @@ public function flatArrayReturnsFalse(): void /** * @test */ - public function emptyArrayReturnsFalse(): void + public function emptyArrayReturnsEmptyArray(): void { - // v3.2.0: an empty result set limited is still an empty result set (it used to be false) + // An empty result set limited is still an empty result set (not false) $this->assertSame([], $this->noneDB->limit([], 3)); + $this->assertSame([], $this->noneDB->limit([], 1)); + $this->assertFalse($this->noneDB->limit([], 0), 'an invalid limit is invalid also for an empty array'); } /** @@ -113,8 +118,8 @@ public function limitExceedsArraySizeReturnsAll(): void $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 100); - $this->assertIsArray($result); - $this->assertCount(5, $result, 'Should return all elements when limit exceeds array size'); + $this->assertSame($array, $result, 'Should return all elements when limit exceeds array size'); + $this->assertSame($array, $this->noneDB->limit($array, 5), 'limit equal to the size'); } /** @@ -140,9 +145,7 @@ public function limitOneReturnsFirstElement(): void $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 1); - $this->assertIsArray($result); - $this->assertCount(1, $result); - $this->assertEquals(1, $result[0]['id']); + $this->assertSame([['id' => 1, 'name' => 'Item 1']], $result); } /** @@ -153,8 +156,8 @@ public function preservesArrayStructure(): void $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 2); - $this->assertArrayHasKey('id', $result[0]); - $this->assertArrayHasKey('name', $result[0]); + $this->assertSame($array[0], $result[0]); + $this->assertSame($array[1], $result[1]); } /** @@ -170,9 +173,7 @@ public function worksWithAssociativeInnerArrays(): void $result = $this->noneDB->limit($array, 2); - $this->assertIsArray($result); - $this->assertCount(2, $result); - $this->assertEquals('john', $result[0]['username']); + $this->assertSame([['username' => 'john', 'email' => 'john@test.com'], ['username' => 'jane', 'email' => 'jane@test.com']], $result); } /** @@ -188,24 +189,21 @@ public function worksWithMixedValueTypes(): void $result = $this->noneDB->limit($array, 2); - $this->assertIsArray($result); - $this->assertCount(2, $result); - $this->assertTrue($result[0]['active']); - $this->assertEquals(85.5, $result[0]['score']); + $this->assertSame([['id' => 1, 'active' => true, 'score' => 85.5], ['id' => 2, 'active' => false, 'score' => 92.0]], $result); } /** * @test */ - public function preservesNumericArrayKeys(): void + public function returnsAListOfTheFirstRecords(): void { $array = $this->getSampleArray(); $result = $this->noneDB->limit($array, 3); - // Keys should be reindexed (0, 1, 2) - $this->assertArrayHasKey(0, $result); - $this->assertArrayHasKey(1, $result); - $this->assertArrayHasKey(2, $result); + $this->assertSame([0, 1, 2], array_keys($result)); + // A result with other keys (e.g. after array_filter) gives the first N records as a list + $filtered = [3 => ['id' => 4], 7 => ['id' => 8], 9 => ['id' => 10]]; + $this->assertSame([['id' => 4], ['id' => 8]], $this->noneDB->limit($filtered, 2)); } /** @@ -221,8 +219,6 @@ public function nestedMultidimensionalArrayWorks(): void $result = $this->noneDB->limit($array, 2); - $this->assertIsArray($result); - $this->assertCount(2, $result); - $this->assertEquals('value1', $result[0]['data']['nested']); + $this->assertSame([['data' => ['nested' => 'value1']], ['data' => ['nested' => 'value2']]], $result); } } diff --git a/tests/Unit/ShardFiltersTest.php b/tests/Unit/ShardFiltersTest.php new file mode 100644 index 0000000..aec2e89 --- /dev/null +++ b/tests/Unit/ShardFiltersTest.php @@ -0,0 +1,110 @@ + [], + 'one' => ['sx'], + 'integers' => ['i1', 'i2', 'i-7', 'i9223372036854775807'], + 'many' => array_map(function ($n) { return 's' . "user{$n}@test.com"; }, range(0, 9999)), + ]; + foreach ([64, 2048, 1 << 19] as $bits) { + foreach ($sets as $label => $ids) { + $this->assertSame($this->shardFilterOfIds($ids, $bits), Testing::callStatic('ShardFilters::filter', $ids, $bits), "$label, $bits bits"); + } + } + } + + /** + * @test §6.5.2 "Lookup": from a body held in memory, the shards a value may be in are those whose filter has all six of + * its probe bits, ascending. Filters and pages from the test's own model; one page, many, and pages of a single row; + * values present, and values of no shard (a saturated filter holds them too, by the model) + */ + public function aHeldBodyAnswersWithTheShardsWhoseFiltersHoldTheValue(): void + { + foreach ([[64, 8, 4], [2048, 8, 8], [2048, 16, 11], [2048, 32768, 9], [65536, 1024, 40], [1 << 19, 128, 100]] as [$bits, $width, $shards]) { + $rb = intdiv($width, 8); + $per = $this->globalPageRows($bits, $width); + $stored = $per * $rb + 4; + $body = str_repeat("\0", intdiv($bits, $per) * $stored); + $filters = []; + $present = []; + for ($shard = 0; $shard < $shards; $shard++) { + $ids = array_map(function ($n) use ($shard) { return "s$shard-$n"; }, range(0, 19)); + $present = array_merge($present, array_slice($ids, 0, 2)); + $filters[$shard] = $this->shardFilterOfIds($ids, $bits); + foreach ($ids as $id) { + foreach ($this->filterProbes($id, $bits) as $r) { + $at = intdiv($r, $per) * $stored + ($r % $per) * $rb + intdiv($shard, 8); + $body[$at] = chr(ord($body[$at]) | (1 << ($shard % 8))); + } + } + } + foreach (array_merge($present, array_map(function ($n) { return "none$n"; }, range(0, 199))) as $id) { + $expected = []; + foreach ($filters as $shard => $filter) { + $all = true; + foreach ($this->filterProbes($id, $bits) as $r) { + $all = $all && (ord($filter[intdiv($r, 8)]) & (1 << ($r % 8))) !== 0; + } + if ($all) { + $expected[] = $shard; + } + } + $this->assertSame($expected, Testing::callStatic('ShardFilters::holdersIn', $body, $bits, $rb, $id), "$bits bits, $width wide: $id"); + } + } + } + + /** + * @test a shard's column in rows [first, first + n) of a page takes bits first … first + n - 1 of its filter; + * every other bit of every row stays. Rows of 1 to 4096 bytes: pages of 4096 rows down to pages of 1 row. + */ + public function aColumnTakesItsFilterBitsAndKeepsEveryOtherBit(): void + { + $bits = 2048; + $filter = random_bytes(intdiv($bits, 8)); + $cases = [[8, 0], [8, 7], [16, 8], [16, 13], [32, 31], [64, 40], [8192, 0], [8192, 4095], [16384, 9000], [32768, 32767]]; + foreach ($cases as [$width, $shard]) { + $rb = intdiv($width, 8); + $per = $this->globalPageRows($bits, $width); + $pages = intdiv($bits, $per); + foreach (array_unique([0, min(1, $pages - 1), min(5, $pages - 1), $pages - 1]) as $page) { + $first = $page * $per; + $rows = random_bytes($per * $rb); + $expected = $rows; + for ($r = 0; $r < $per; $r++) { + $at = $r * $rb + intdiv($shard, 8); + $bit = 1 << ($shard % 8); + $set = (ord($filter[intdiv($first + $r, 8)]) >> (($first + $r) % 8)) & 1; + $expected[$at] = chr($set === 1 ? ord($expected[$at]) | $bit : ord($expected[$at]) & (0xFF ^ $bit)); + } + $label = "width $width ($per rows per page), shard $shard, page $page"; + $this->assertSame($expected, Testing::callStatic('ShardFilters::withColumn', $rows, $first, $rb, $shard, $filter), $label); + $this->assertSame(str_repeat("\0", $per * $rb) | ($expected & $this->columnOnly($per, $rb, $shard)), + Testing::callStatic('ShardFilters::withColumn', str_repeat("\0", $per * $rb), $first, $rb, $shard, $filter), "$label, from empty rows"); + } + } + } + + /** Rows with only $shard's bit set in each */ + private function columnOnly(int $per, int $rb, int $shard): string + { + $row = str_repeat("\0", $rb); + $row[intdiv($shard, 8)] = chr(1 << ($shard % 8)); + return str_repeat($row, $per); + } +} diff --git a/tests/Unit/SourceWriteGuardTest.php b/tests/Unit/SourceWriteGuardTest.php new file mode 100644 index 0000000..86aafcc --- /dev/null +++ b/tests/Unit/SourceWriteGuardTest.php @@ -0,0 +1,141 @@ + ["unlink@123", "fopen(c+b)@456", ...] + */ + public static function rawWriteCalls(string $file): array + { + $tokens = token_get_all((string)file_get_contents($file)); + $function = null; + $functionDepth = null; + $depth = 0; + $calls = []; + $skip = function (int $i, int $step) use ($tokens) { + while (isset($tokens[$i]) && is_array($tokens[$i]) && in_array($tokens[$i][0], [T_WHITESPACE, T_COMMENT, T_DOC_COMMENT], true)) { + $i += $step; + } + return $i; + }; + foreach ($tokens as $i => $token) { + if ($token === '{' || (is_array($token) && in_array($token[0], [T_CURLY_OPEN, T_DOLLAR_OPEN_CURLY_BRACES], true))) { + $depth++; + continue; + } + if ($token === '}') { + $depth--; + if ($functionDepth !== null && $depth < $functionDepth) { + $function = null; + $functionDepth = null; + } + continue; + } + if (!is_array($token)) { + continue; + } + if ($token[0] === T_FUNCTION && $function === null) { + $before = $skip($i - 1, -1); + if (is_array($tokens[$before]) && $tokens[$before][0] === T_USE) { + continue; // "use function ...": an import, not a declaration + } + $j = $skip($i + 1, 1); + if (is_array($tokens[$j]) && $tokens[$j][0] === T_STRING) { + $function = $tokens[$j][1]; + $functionDepth = $depth + 1; + } + continue; + } + if ($token[0] !== T_STRING || !in_array(strtolower($token[1]), self::RAW, true)) { + continue; + } + $next = $skip($i + 1, 1); + $previous = $skip($i - 1, -1); + if ($tokens[$next] !== '(' || (is_array($tokens[$previous]) + && in_array($tokens[$previous][0], [T_OBJECT_OPERATOR, T_DOUBLE_COLON, T_FUNCTION, T_NEW], true))) { + continue; // not a call of the global function + } + $label = $token[1]; + if (strtolower($token[1]) === 'fopen') { + // Read-only modes are fine + $nesting = 0; + $argument = 0; + $mode = null; + for ($k = $next + 1; isset($tokens[$k]); $k++) { + $t = $tokens[$k]; + if ($t === '(') { + $nesting++; + } elseif ($t === ')') { + if ($nesting-- === 0) { + break; + } + } elseif ($t === ',' && $nesting === 0) { + $argument++; + } elseif ($argument === 1 && is_array($t) && $t[0] === T_CONSTANT_ENCAPSED_STRING) { + $mode = trim($t[1], "'\""); + } + } + if ($mode !== null && strpbrk($mode, 'waxc+') === false) { + continue; + } + $label .= '(' . ($mode ?? '?') . ')'; + } + $calls[$function ?? '(file)'][] = $label . '@' . $token[2]; + } + return $calls; + } + + /** @test */ + public function databaseFilesAreOnlyWrittenThroughTheHelpers(): void + { + $calls = self::rawWriteCalls(dirname(__DIR__, 2) . '/noneDB.php'); + $outside = array_diff_key($calls, array_flip(self::ALLOWED)); + $this->assertSame([], $outside, 'raw file-system writes outside the write helpers (use fsUnlink/fsRename/' + . 'fsTouch/fsTruncate/fsMkdir/fsRmdir/writeFileAtomic/appendToFile)'); + // The scanner finds what it should (the helpers themselves use the raw calls) + $this->assertArrayHasKey('fsUnlink', $calls); + $this->assertArrayHasKey('appendToFile', $calls); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 7faefad..86fc42d 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -5,14 +5,37 @@ * This file is loaded before running any tests. */ -// Composer autoloader -require_once __DIR__ . '/../vendor/autoload.php'; +// The tests run against the test builds: noneDB.php of src/ and noneDB-migrate.php of migrate/, each with its test +// seams (fault points, test hooks, counters: tools/build.php). Written to tests/build/ only when they change; the +// engine is loaded before Composer's autoloader could reach the production noneDB.php. The tool next to it loads it. +require_once __DIR__ . '/../tools/build.php'; +(static function (): void { + foreach (['noneDB.php' => nonedb_build(dirname(__DIR__), true)['code'], 'noneDB-migrate.php' => nonedb_build_tool(dirname(__DIR__), true)['code']] as $name => $code) { + $file = __DIR__ . '/build/' . $name; + if (!is_file($file) || file_get_contents($file) !== $code) { + if (!is_dir(dirname($file))) { + mkdir(dirname($file), 0777, true); + } + $tmp = $file . '.tmp' . getmypid(); + file_put_contents($tmp, $code); + rename($tmp, $file); + } + } +})(); +define('NONEDB_TEST_LIBRARY', __DIR__ . '/build/noneDB.php'); +define('NONEDB_TEST_TOOL', __DIR__ . '/build/noneDB-migrate.php'); +// The configuration template ships next to noneDB.php: next to the test build too (getConfigTemplate()) +if (!is_file(__DIR__ . '/build/.nonedb.example') || file_get_contents(__DIR__ . '/build/.nonedb.example') !== file_get_contents(__DIR__ . '/../.nonedb.example')) { + copy(__DIR__ . '/../.nonedb.example', __DIR__ . '/build/.nonedb.example'); +} +require_once NONEDB_TEST_LIBRARY; -// Load noneDB class -require_once __DIR__ . '/../noneDB.php'; +// Composer autoloader (the test classes, tests/Support/Testing.php) +require_once __DIR__ . '/../vendor/autoload.php'; -// Define test database directory constant -define('TEST_DB_DIR', __DIR__ . '/test_db/'); +// Define test database directory constant (NONEDB_TEST_DB_DIR: another directory, so test runs don't share one) +$testDbDir = getenv('NONEDB_TEST_DB_DIR'); +define('TEST_DB_DIR', is_string($testDbDir) && $testDbDir !== '' ? rtrim($testDbDir, '/') . '/' : __DIR__ . '/test_db/'); // Create test db directory if not exists if (!file_exists(TEST_DB_DIR)) { diff --git a/tests/buffer_test.php b/tests/buffer_test.php deleted file mode 100644 index 23468e4..0000000 --- a/tests/buffer_test.php +++ /dev/null @@ -1,124 +0,0 @@ -isBufferingEnabled() ? green("YES") : red("NO")) . "\n"; -$info = $db->getBufferInfo($testDb); -echo " Size limit: " . number_format($info['sizeLimit']) . " bytes (" . round($info['sizeLimit']/1024/1024, 1) . "MB)\n"; -echo " Count limit: " . number_format($info['countLimit']) . " records\n"; -echo " Flush interval: " . $info['flushInterval'] . " seconds\n\n"; - -// Test 2: Insert with buffer (empty DB) -echo yellow("Test 2: Buffered Insert (Empty DB)\n"); -$insertCount = 1000; - -$start = microtime(true); -for ($i = 0; $i < $insertCount; $i++) { - $db->insert($testDb, [ - 'name' => 'User' . $i, - 'email' => "user{$i}@test.com", - 'score' => rand(1, 100) - ]); -} -$bufferedTime = (microtime(true) - $start) * 1000; - -$info = $db->getBufferInfo($testDb); -$bufferRecords = $info['buffers']['main']['records'] ?? 0; -echo " Inserted: {$insertCount} records\n"; -echo " Time: " . green(round($bufferedTime, 1) . " ms") . "\n"; -echo " Buffer records: {$bufferRecords}\n"; - -// Test 3: Manual flush -echo "\n" . yellow("Test 3: Manual Flush\n"); -$flushResult = $db->flush($testDb); -echo " Flushed: " . $flushResult['flushed'] . " records\n"; -echo " Success: " . ($flushResult['success'] ? green("YES") : red("NO")) . "\n"; - -// Test 4: Read after flush -echo "\n" . yellow("Test 4: Read Verification\n"); -$data = $db->find($testDb, []); -echo " Records in DB: " . count($data) . "\n"; -echo " Expected: {$insertCount}\n"; -echo " Match: " . (count($data) === $insertCount ? green("YES") : red("NO")) . "\n"; - -// Test 5: THE REAL BUFFER ADVANTAGE - Insert into large database -echo "\n" . cyan("═══════════════════════════════════════════════════════════════\n"); -echo cyan(" TEST 5: Buffer Advantage on Large Database (10K records)\n"); -echo cyan("═══════════════════════════════════════════════════════════════\n\n"); - -$largeDb = 'large_buffer_test_' . time(); - -// Create a 10K record database first -echo yellow(" Step 1: Creating 10K record database...\n"); -$bulkData = []; -for ($i = 0; $i < 10000; $i++) { - $bulkData[] = ['name' => "User$i", 'score' => rand(1,100)]; -} -$db->insert($largeDb, $bulkData); -$db->flush($largeDb); -echo " Created 10K records\n\n"; - -// Test A: Buffered individual inserts -echo yellow(" Step 2: Adding 100 records WITH buffer...\n"); -$db->enableBuffering(true); -$start = microtime(true); -for ($i = 0; $i < 100; $i++) { - $db->insert($largeDb, ['name' => "NewUser$i", 'type' => 'buffered']); -} -$bufferedLargeTime = (microtime(true) - $start) * 1000; -$db->flush($largeDb); -echo " Time: " . green(round($bufferedLargeTime, 1) . " ms") . " (100 inserts)\n"; -echo " Per insert: " . green(round($bufferedLargeTime/100, 2) . " ms") . "\n\n"; - -// Test B: Non-buffered individual inserts (only 20 - it's slow!) -echo yellow(" Step 3: Adding 20 records WITHOUT buffer...\n"); -$db->enableBuffering(false); -$start = microtime(true); -for ($i = 0; $i < 20; $i++) { - $db->insert($largeDb, ['name' => "SlowUser$i", 'type' => 'nobuffer']); -} -$nonBufferedLargeTime = (microtime(true) - $start) * 1000; -echo " Time: " . red(round($nonBufferedLargeTime, 1) . " ms") . " (20 inserts)\n"; -echo " Per insert: " . red(round($nonBufferedLargeTime/20, 2) . " ms") . "\n\n"; - -// Calculate speedup -$perInsertBuffered = $bufferedLargeTime / 100; -$perInsertNonBuffered = $nonBufferedLargeTime / 20; -$speedup = $perInsertNonBuffered / $perInsertBuffered; - -echo cyan(" ┌────────────────────────────────────────────────────────────┐\n"); -echo cyan(" │ RESULT: Buffer is ") . green(round($speedup, 0) . "x FASTER") . cyan(" on large databases! │\n"); -echo cyan(" │ │\n"); -echo cyan(" │ Buffered: ") . sprintf("%-6s", round($perInsertBuffered, 2) . " ms") . cyan(" per insert │\n"); -echo cyan(" │ Non-buffered: ") . sprintf("%-6s", round($perInsertNonBuffered, 2) . " ms") . cyan(" per insert │\n"); -echo cyan(" └────────────────────────────────────────────────────────────┘\n"); - -// Cleanup -echo "\n" . yellow("Cleanup\n"); -$db->enableBuffering(true); -$files = glob(__DIR__ . '/../db/*buffer_test*'); -foreach ($files as $f) @unlink($f); -echo " Cleaned up test files\n"; - -echo "\n" . green("Buffer test completed!\n"); diff --git a/tests/concurrent_insert_test.php b/tests/concurrent_insert_test.php deleted file mode 100644 index 9e51bf3..0000000 --- a/tests/concurrent_insert_test.php +++ /dev/null @@ -1,184 +0,0 @@ -insert($dbName, [ - 'worker_id' => $workerId, - 'sequence' => $i, - 'timestamp' => microtime(true), - 'data' => "Record {$i} from worker {$workerId}" - ]); - - if (isset($result['n']) && $result['n'] > 0) { - $inserted++; - } - - // Küçük bir rastgele gecikme - race condition şansını artırır - usleep(rand(0, 1000)); -} - -echo json_encode(['worker_id' => $workerId, 'inserted' => $inserted]); -PHP; - -$workerFile = __DIR__ . '/concurrent_worker.php'; -file_put_contents($workerFile, $workerScript); - -// Temiz başlangıç -$db = new noneDB(); - -echo "Test Parametreleri:\n"; -echo " • Database: {$testDbName}\n"; -echo " • Process sayısı: {$processCount}\n"; -echo " • Her process'in insert sayısı: {$insertsPerProcess}\n"; -echo " • Beklenen toplam kayıt: " . ($processCount * $insertsPerProcess) . "\n\n"; - -echo yellow("▶ Process'ler başlatılıyor...\n\n"); - -// Parallel process'leri başlat -$processes = []; -$pipes = []; - -for ($i = 0; $i < $processCount; $i++) { - $workerId = "worker_{$i}"; - - $descriptorspec = [ - 0 => ["pipe", "r"], - 1 => ["pipe", "w"], - 2 => ["pipe", "w"] - ]; - - $cmd = "{$phpPath} {$workerFile} {$testDbName} {$workerId} {$insertsPerProcess}"; - $process = proc_open($cmd, $descriptorspec, $p); - - if (is_resource($process)) { - $processes[$workerId] = $process; - $pipes[$workerId] = $p; - echo " ✓ {$workerId} başlatıldı\n"; - } -} - -echo "\n" . yellow("▶ Process'ler tamamlanması bekleniyor...\n\n"); - -// Sonuçları topla -$results = []; -foreach ($processes as $workerId => $process) { - $output = stream_get_contents($pipes[$workerId][1]); - $errors = stream_get_contents($pipes[$workerId][2]); - - fclose($pipes[$workerId][0]); - fclose($pipes[$workerId][1]); - fclose($pipes[$workerId][2]); - - $exitCode = proc_close($process); - - if ($output) { - $result = json_decode($output, true); - if ($result) { - $results[$workerId] = $result; - echo " ✓ {$workerId}: {$result['inserted']} kayıt eklendi (rapor edilen)\n"; - } - } - - if ($errors) { - echo red(" ✗ {$workerId} hata: {$errors}\n"); - } -} - -// Gerçek kayıt sayısını kontrol et -echo "\n" . yellow("▶ Veritabanı doğrulanıyor...\n\n"); - -$allRecords = $db->find($testDbName, []); -$actualCount = is_array($allRecords) ? count($allRecords) : 0; -$expectedCount = $processCount * $insertsPerProcess; - -// Worker bazlı analiz -$workerCounts = []; -if (is_array($allRecords)) { - foreach ($allRecords as $record) { - if (isset($record['worker_id'])) { - $wid = $record['worker_id']; - $workerCounts[$wid] = ($workerCounts[$wid] ?? 0) + 1; - } - } -} - -echo "Worker Bazlı Sonuçlar:\n"; -foreach ($workerCounts as $wid => $count) { - $status = $count === $insertsPerProcess ? green("✓") : red("✗"); - echo " {$status} {$wid}: {$count}/{$insertsPerProcess} kayıt\n"; -} - -echo "\n"; -echo "═══════════════════════════════════════════════════════════════\n"; -echo "SONUÇ:\n"; -echo " • Beklenen kayıt sayısı: {$expectedCount}\n"; -echo " • Gerçek kayıt sayısı: {$actualCount}\n"; - -if ($actualCount === $expectedCount) { - echo "\n" . green(" ✓ BAŞARILI: Tüm kayıtlar doğru şekilde eklendi!\n"); - echo green(" Race condition tespit edilmedi.\n"); -} else { - $lost = $expectedCount - $actualCount; - echo "\n" . red(" ✗ BAŞARISIZ: {$lost} kayıt kayboldu!\n"); - echo red(" Race condition tespit edildi!\n"); - echo "\n"; - echo yellow(" Çözüm önerileri:\n"); - echo " 1. Atomik dosya kilitleme (flock ile read-modify-write)\n"; - echo " 2. Veritabanı seviyesinde mutex\n"; - echo " 3. Optimistik kilitleme (version numarası ile)\n"; -} -echo "═══════════════════════════════════════════════════════════════\n"; - -// Temizlik -unlink($workerFile); - -// Test veritabanı dosyalarını temizle -$dbDir = __DIR__ . '/../db/'; -$files = glob($dbDir . '*' . $testDbName . '*'); -foreach ($files as $file) { - unlink($file); -} - -echo "\n" . blue("Test tamamlandı. Test veritabanı silindi.\n"); diff --git a/tests/concurrent_stress_test.php b/tests/concurrent_stress_test.php deleted file mode 100644 index c0aff08..0000000 --- a/tests/concurrent_stress_test.php +++ /dev/null @@ -1,122 +0,0 @@ -insert($argv[1], [ - 'worker_id' => $argv[2], - 'seq' => $i, - 'ts' => microtime(true) - ]); - if (isset($result['n']) && $result['n'] > 0) $inserted++; -} -echo json_encode(['w' => $argv[2], 'n' => $inserted]); -PHP; - -$workerFile = __DIR__ . '/stress_worker.php'; -file_put_contents($workerFile, $workerScript); - -$db = new noneDB(); -$expectedCount = $processCount * $insertsPerProcess; - -echo "Config: {$processCount} process x {$insertsPerProcess} insert = {$expectedCount} kayıt bekleniyor\n\n"; - -// Start all processes -$processes = []; -$pipes = []; -for ($i = 0; $i < $processCount; $i++) { - $cmd = "{$phpPath} {$workerFile} {$testDbName} w{$i} {$insertsPerProcess}"; - $process = proc_open($cmd, [0=>["pipe","r"], 1=>["pipe","w"], 2=>["pipe","w"]], $p); - if (is_resource($process)) { - $processes["w{$i}"] = $process; - $pipes["w{$i}"] = $p; - } -} - -// Collect results -$totalReported = 0; -foreach ($processes as $wid => $process) { - $output = stream_get_contents($pipes[$wid][1]); - fclose($pipes[$wid][0]); - fclose($pipes[$wid][1]); - fclose($pipes[$wid][2]); - proc_close($process); - - if ($output) { - $r = json_decode($output, true); - if ($r) { - $totalReported += $r['n']; - echo " {$wid}: {$r['n']} kayıt (rapor)\n"; - } - } -} - -// Verify -$allRecords = $db->find($testDbName, []); -$actualCount = is_array($allRecords) ? count($allRecords) : 0; - -// Worker breakdown -$workerCounts = []; -if (is_array($allRecords)) { - foreach ($allRecords as $rec) { - if (isset($rec['worker_id'])) { - $workerCounts[$rec['worker_id']] = ($workerCounts[$rec['worker_id']] ?? 0) + 1; - } - } -} - -echo "\n═══════════════════════════════════════════════════════════════\n"; -echo "SONUÇ:\n"; -echo " • Worker'ların rapor ettiği: {$totalReported}\n"; -echo " • Beklenen: {$expectedCount}\n"; -echo " • Gerçek: {$actualCount}\n"; - -if ($actualCount === $expectedCount) { - echo "\n" . green(" ✓ BAŞARILI - Veri kaybı yok!\n"); -} else { - $lost = $expectedCount - $actualCount; - $lostPct = round(($lost / $expectedCount) * 100, 1); - echo "\n" . red(" ✗ BAŞARISIZ - {$lost} kayıt kayboldu (%{$lostPct})\n"); - - echo "\nWorker bazlı kayıp:\n"; - for ($i = 0; $i < $processCount; $i++) { - $wid = "w{$i}"; - $actual = $workerCounts[$wid] ?? 0; - $lost = $insertsPerProcess - $actual; - if ($lost > 0) { - echo red(" {$wid}: {$actual}/{$insertsPerProcess} ({$lost} kayıp)\n"); - } else { - echo green(" {$wid}: {$actual}/{$insertsPerProcess} (tamam)\n"); - } - } -} -echo "═══════════════════════════════════════════════════════════════\n"; - -// Cleanup -unlink($workerFile); -$files = glob(__DIR__ . '/../db/*' . $testDbName . '*'); -foreach ($files as $f) unlink($f); diff --git a/tests/noneDBTestCase.php b/tests/noneDBTestCase.php index db066ae..fd5811f 100644 --- a/tests/noneDBTestCase.php +++ b/tests/noneDBTestCase.php @@ -3,8 +3,6 @@ namespace noneDB\Tests; use PHPUnit\Framework\TestCase; -use ReflectionClass; -use ReflectionMethod; /** * Base test case class for noneDB tests @@ -49,12 +47,10 @@ protected function setUp(): void 'dbDir' => $this->testDbDir, 'autoCreateDB' => true, 'shardingEnabled' => true, - 'shardSize' => 10000, - 'autoMigrate' => true + 'shardSize' => 10000 ]); - // Buffer is enabled by default (v2.3.0+) - // getDatabaseContents() flushes buffer automatically for consistency + // getDatabaseContents() flushes the write buffer for consistency } /** @@ -62,6 +58,12 @@ protected function setUp(): void */ protected function tearDown(): void { + // PHPUnit keeps every test object until the end of the run: drop the instance (and its engine) + $this->noneDB = null; + // Process-wide test hooks never outlive the test that set them + foreach (['syncFn', 'flushFn', 'writeFn', 'renameFn', 'osFamily'] as $hook) { + \noneDB\Tests\Support\Testing::set($hook, null); + } // Clean test directory after each test $this->cleanTestDirectory(); @@ -87,11 +89,10 @@ protected function cleanTestDirectory(): void return; } - $files = glob($this->testDbDir . '*'); - foreach ($files as $file) { - if (is_file($file)) { - unlink($file); - clearstatcache(true, $file); + // Everything in the directory goes: store files, database directories, maintenance leftovers + foreach (scandir($this->testDbDir) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeDirectoryTree(rtrim($this->testDbDir, '/') . '/' . $entry); } } @@ -100,53 +101,108 @@ protected function cleanTestDirectory(): void } /** - * Get a private/protected method for testing - * - * @param string $methodName The name of the method - * @return ReflectionMethod + * Remove a file or a directory with everything in it (no error when it doesn't exist). + * Close noneDB's kept lock handles first (\noneDB::clearStaticCache()) when the tree holds a store. */ - protected function getPrivateMethod(string $methodName): ReflectionMethod + protected function removeDirectoryTree(string $path): void { - $reflector = new ReflectionClass(\noneDB::class); - $method = $reflector->getMethod($methodName); - if (PHP_VERSION_ID < 80100) { - $method->setAccessible(true); // no effect since 8.1, deprecated in 8.5 + $path = rtrim($path, '/'); + if (is_dir($path) && !is_link($path)) { + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeDirectoryTree($path . '/' . $entry); + } + } + @chmod($path, 0777); + @rmdir($path); + } elseif (file_exists($path) || is_link($path)) { + @unlink($path); } + clearstatcache(true, $path); + } - return $method; + /** + * Locator of a database: the name of its directory {locator}.ndb4 in dbDir (docs/STORAGE_FORMAT.md §1), + * computed independently of noneDB; $key defaults to the key the test instances use + */ + protected function locatorOf(string $dbName, string $key = 'test_secret_key_for_unit_tests'): string + { + return substr(hash_hmac('sha256', 'nonedb4/db/' . $dbName, $key), 0, 32); + } + + /** + * keyCheck of a key: how STORE lists it (docs/STORAGE_FORMAT.md §1), computed independently of noneDB + */ + protected function keyCheckOf(string $key = 'test_secret_key_for_unit_tests'): string + { + return substr(hash_hmac('sha256', 'nonedb4/key', $key), 0, 32); } /** - * Get a private/protected property value + * Encode a metadata record (STORE, FORMAT, CURRENT, metadata, ...) as docs/STORAGE_FORMAT.md §2 specifies: + * one line of JSON (unescaped slashes and Unicode, zero fractions kept, serialize_precision -1), members in + * the given order, "checksum" last = SHA-256 of the encoding without it followed by "\n". + * An array formatOptions member that is empty is written as {} (§4.1); other members that must be + * objects are passed as objects (decode with json_decode($raw) to keep them). * - * @param string $propertyName The name of the property - * @return mixed + * @param array|object $record */ - protected function getPrivateProperty(string $propertyName) + /** + * A sealed record of STORAGE_FORMAT.md §2 read as the format says: the checksum over the bytes before + * ',"checksum"' plus "}\n", magic and kind; null when it doesn't verify + */ + protected function openSealedRecord(string $raw, string $kind): ?array { - $reflector = new ReflectionClass(\noneDB::class); - $property = $reflector->getProperty($propertyName); - if (PHP_VERSION_ID < 80100) { - $property->setAccessible(true); // no effect since 8.1, deprecated in 8.5 + if (!preg_match('/\A(.*),"checksum":"([0-9a-f]{64})"\}\n\z/s', $raw, $m) || hash('sha256', $m[1] . "}\n") !== $m[2]) { + return null; } + $record = json_decode($raw, true); + return is_array($record) && ($record['magic'] ?? null) === 'NONEDB4' && ($record['kind'] ?? null) === $kind ? $record : null; + } - return $property->getValue($this->noneDB); + protected function sealRecord($record): string + { + $record = (array)$record; + unset($record['checksum']); + if (array_key_exists('formatOptions', $record) && $record['formatOptions'] === []) { + $record['formatOptions'] = new \stdClass(); + } + $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION; + $precision = ini_get('serialize_precision'); + ini_set('serialize_precision', '-1'); + try { + $body = json_encode($record, $flags); + $record['checksum'] = hash('sha256', $body . "\n"); + return json_encode($record, $flags) . "\n"; + } finally { + ini_set('serialize_precision', (string)$precision); + } } /** - * Set a private/protected property value - * - * @param string $propertyName The name of the property - * @param mixed $value The value to set + * An internal method of the engine, called through noneDB\Tests\Support\Testing: + * ->invoke($db, ...$args) calls it for that instance + */ + protected function getPrivateMethod(string $methodName): InternalMethod + { + return new InternalMethod($methodName); + } + + /** + * A value of $this->noneDB's engine (a setting or a cache) + * @return mixed + */ + protected function getPrivateProperty(string $propertyName) + { + return \noneDB\Tests\Support\Testing::getOf($this->noneDB, $propertyName); + } + + /** + * Set a value of $this->noneDB's engine (a setting or a cache) */ protected function setPrivateProperty(string $propertyName, $value): void { - $reflector = new ReflectionClass(\noneDB::class); - $property = $reflector->getProperty($propertyName); - if (PHP_VERSION_ID < 80100) { - $property->setAccessible(true); // no effect since 8.1, deprecated in 8.5 - } - $property->setValue($this->noneDB, $value); + \noneDB\Tests\Support\Testing::setOf($this->noneDB, $propertyName, $value); } /** @@ -174,21 +230,84 @@ protected function insertSampleData(string $dbName, int $count = 3): array } /** - * Get the database file path + * Main data file of a (single-layout) database * * @param string $dbName Database name - * @return string Full path to database file + * @return string Full path to its data file */ protected function getDbFilePath(string $dbName): string { - // Sanitize name the same way noneDB does - $sanitizedName = preg_replace("/[^A-Za-z0-9' -]/", '', $dbName); + return $this->dbPath($dbName, 'data'); + } - $hashMethod = $this->getPrivateMethod('hashDBName'); - $hash = $hashMethod->invoke($this->noneDB, $sanitizedName); - $dbDir = $this->getPrivateProperty('dbDir'); + /** + * Path of a file of a database in the storage layout (docs/STORAGE_FORMAT.md), by its role: + * home, lock, format, current, gen the directory, LOCK, FORMAT, CURRENT, active generation + * data / kix [shard] segments/{main|sN}.jsonl / .kix + * shards, indexes shards.json, indexes.json + * buffer, flushing buffer/pending.jsonl, buffer/flushing.{K}.jsonl + * fx / rx [shard], gfx / grx secondary indexes + * recovery, recovery1, quarantine recovery/log.jsonl, log.1.jsonl, recovery/ + * The active generation is read from CURRENT (generation 1 when there is none). + */ + protected function dbPath(string $dbName, string $role = 'data', ...$args): string + { + $home = rtrim($this->testDbDir, '/') . '/' . $this->locatorOf($dbName) . '.ndb4/'; + $current = @file_get_contents($home . 'CURRENT'); + $record = $current === false ? null : json_decode($current, true); + $gen = $home . 'generations/' . (is_array($record) && is_int($record['generation'] ?? null) ? $record['generation'] : 1) . '/'; + $seg = function ($shard) { return $shard === null ? 'main' : 's' . (int)$shard; }; + $token = function ($field) { + $field = (string)$field; + return preg_match('/^[a-z0-9_]{1,40}$/', $field) === 1 ? $field + : substr(strtolower(preg_replace('/[^A-Za-z0-9_]/', '_', $field)), 0, 24) . '-' . substr(hash('sha256', $field), 0, 12); + }; + switch ($role) { + case 'home': return $home; + case 'lock': return $home . 'LOCK'; + case 'format': return $home . 'FORMAT'; + case 'current': return $home . 'CURRENT'; + case 'gen': return $gen; + case 'data': return $gen . 'segments/' . $seg($args[0] ?? null) . '.jsonl'; + case 'kix': return $gen . 'segments/' . $seg($args[0] ?? null) . '.kix'; + case 'shards': return $gen . 'shards.json'; + case 'indexes': return $gen . 'indexes.json'; + case 'buffer': return $gen . 'buffer/pending.jsonl'; + case 'flushing': return $gen . 'buffer/flushing.' . (int)$args[0] . '.jsonl'; + case 'fx': return $gen . 'indexes/' . $seg($args[1] ?? null) . '.' . $token($args[0]) . '.fx'; + case 'gfx': return $gen . 'indexes/' . $token($args[0]) . '.gfx'; + case 'rx': return $gen . 'spatial/' . $seg($args[1] ?? null) . '.' . $token($args[0]) . '.rx'; + case 'grx': return $gen . 'spatial/' . $token($args[0]) . '.grx'; + case 'recovery': return $home . 'recovery/log.jsonl'; + case 'recovery1': return $home . 'recovery/log.1.jsonl'; + case 'quarantine': return $home . 'recovery/'; + } + throw new \InvalidArgumentException("unknown database file role $role"); + } - return $dbDir . $hash . '-' . $sanitizedName . '.nonedb'; + /** + * Every file of a database (recursively), relative to its directory, sorted + * @return string[] + */ + protected function dbFiles(string $dbName): array + { + $home = $this->dbPath($dbName, 'home'); + $files = []; + $walk = function (string $dir) use (&$walk, &$files, $home): void { + foreach (@scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + if (is_dir($dir . $entry)) { + $walk($dir . $entry . '/'); + } else { + $files[] = substr($dir . $entry, strlen($home)); + } + } + }; + $walk($home); + sort($files); + return $files; } /** @@ -216,7 +335,8 @@ protected function assertDatabaseNotExists(string $dbName): void /** * Get database contents directly from file * Flushes any buffered data first to ensure consistency - * v3.0: JSONL-only format - uses .jidx index to determine valid records + * Computed from the data file alone, like an index rebuilt from it: the last line of a key wins, + * a tombstone {"__nonedb_deleted":key} deletes (the .kix file is a checkpoint that may lag behind) * * @param string $dbName Database name * @return array|null Returns normalized format: ['data' => [...records...]] @@ -232,49 +352,591 @@ protected function getDatabaseContents(string $dbName): ?array return null; } - $contents = file_get_contents($filePath); - $indexPath = $filePath . '.jidx'; - - // JSONL format with index - only return records that exist in index $records = []; + foreach (explode("\n", (string)file_get_contents($filePath)) as $line) { + $record = json_decode($line, true); + if (!is_array($record)) { + continue; + } + if (count($record) === 1 && isset($record['__nonedb_deleted'])) { + unset($records[$record['__nonedb_deleted']]); + } elseif (isset($record['key'])) { + $key = $record['key']; + unset($record['key']); + $records[$key] = $record; + } + } - if (file_exists($indexPath)) { - $index = json_decode(file_get_contents($indexPath), true); - if ($index !== null && isset($index['o'])) { - // Read all lines and filter by index - $lines = explode("\n", trim($contents)); - foreach ($lines as $line) { - $line = trim($line); - if (empty($line)) continue; - $record = json_decode($line, true); - if ($record !== null && isset($record['key'])) { - $key = $record['key']; - // Only include records that exist in index (not deleted) - if (isset($index['o'][$key])) { - unset($record['key']); - $records[$key] = $record; - } + // Sort by key to maintain order + ksort($records); + return ['data' => array_values($records)]; + } + + // ------------------------------------------------------------------ + // Key-index checkpoints (docs/STORAGE_FORMAT.md §6.4), read and written here independently of the + // engine: tests check the files against the documented format, and craft files to set up states + // ------------------------------------------------------------------ + + /** + * Decode a .kix file: ['header' => array, 'entries' => [key => [offset, length]]] (live keys, ascending). + * Throws when the file breaks the documented format (header, block size, block crc, body length). + */ + /** A derived file's header with its crc as the last member (STORAGE_FORMAT.md §6.5) */ + protected function sealHeaderLine(string $json): string + { + return substr($json, 0, -1) . ',"crc":' . (int)hexdec(hash('crc32b', $json)) . '}'; + } + + /** [header, nodes] of an .rx file (STORAGE_FORMAT.md §6.5): id => [level, box, items], a leaf's items as entries of five integers */ + protected function readRTreeFile(string $path): array + { + [$line, $body] = explode("\n", (string)file_get_contents($path), 2); + $end = strpos($body, "\n"); + $nodes = $end === false ? null : json_decode(substr($body, 0, $end), true); + if (is_array($nodes)) { + $at = $end + 1; + foreach ($nodes as $id => $node) { + if ($node[0] === 0) { + $entries = []; + for ($i = 0; $i < $node[2]; $i++, $at += 40) { + $entries[] = array_values(unpack('P5', $body, $at)); } + $nodes[$id][2] = $entries; + } + } + } + return [$this->openHeaderLine($path, $line), $nodes]; + } + + /** Write an .rx file (STORAGE_FORMAT.md §6.5) of readRTreeFile()'s header and nodes, with the crcs that fit */ + protected function writeRTreeFile(string $path, array $header, array $nodes): void + { + $entries = ''; + foreach ($nodes as $id => $node) { + if ($node[0] === 0) { + foreach ($node[2] as $entry) { + $entries .= pack('P5', ...$entry); + } + $nodes[$id][2] = count($node[2]); + } + } + $body = json_encode($nodes) . "\n" . $entries; + $header['bc'] = crc32($body); + file_put_contents($path, $this->sealHeaderLine(json_encode($header)) . "\n" . $body); + } + + /** The members of a derived file's header; throws when its crc doesn't fit (§6.5) */ + protected function openHeaderLine(string $path, string $line): ?array + { + $header = json_decode($line, true); + if (!is_array($header)) { + return null; + } + if (array_key_last($header) !== 'crc') { + throw new \UnexpectedValueException("$path: the header's last member is not crc"); + } + $crc = $header['crc']; + unset($header['crc']); + $bare = substr($line, 0, (int)strrpos($line, ',"crc":')) . '}'; + if ($crc !== (int)hexdec(hash('crc32b', $bare))) { + throw new \UnexpectedValueException("$path: the header's crc doesn't fit"); + } + return $header; + } + + protected function readKeyIndexFile(string $path): array + { + $raw = file_get_contents($path); + if ($raw === false || strlen($raw) < 512 || $raw[511] !== "\n") { + throw new \UnexpectedValueException("$path: no 512-byte header"); + } + $header = $this->openHeaderLine($path, rtrim(substr($raw, 0, 511), ' ')); + if (!is_array($header) || ($header['kind'] ?? null) !== 'kix' || ($header['v'] ?? null) !== 1) { + throw new \UnexpectedValueException("$path: not a kix v1 header"); + } + $body = substr($raw, 512); + $dense = $header['layout'] === 'dense'; + $per = $dense ? 341 : 255; + $width = $dense ? 12 : 16; + $blocks = $header['slots'] === 0 ? 0 : intdiv($header['slots'] - 1, $per) + 1; + $expected = $blocks === 0 ? 0 : ($blocks - 1) * 4096 + ($dense ? ($header['slots'] - ($blocks - 1) * $per) * 12 + 4 : 4096); + if (strlen($body) !== $expected) { + throw new \UnexpectedValueException("$path: body is " . strlen($body) . " bytes, the header says $expected"); + } + $entries = []; + for ($b = 0; $b < $blocks; $b++) { + $block = substr($body, $b * 4096, 4096); + $data = substr($block, 0, -4); + if (unpack('V', substr($block, -4))[1] !== (int)hexdec(hash('crc32b', $data))) { + throw new \UnexpectedValueException("$path: block $b crc mismatch"); + } + $m = min($per, $header['slots'] - $b * $per); + for ($i = 0; $i < $m; $i++) { + if ($dense) { + $offset = unpack('P', $data, 8 * $i)[1]; + $length = unpack('V', $data, 8 * $m + 4 * $i)[1]; + $key = $header['base'] + $b * $per + $i; + } else { + $key = $header['base'] + unpack('V', $data, 4 * $i)[1]; + $offset = unpack('P', $data, 4 * $m + 8 * $i)[1]; + $length = unpack('V', $data, 12 * $m + 4 * $i)[1]; + } + if ($length > 0) { + $entries[$key] = [$offset, $length]; + } + } + } + ksort($entries); + return ['header' => $header, 'entries' => $entries]; + } + + /** + * Write a .kix file as the documented format describes it ($header: every member but layout, base and + * slots, which follow from $entries and $base; layout chosen by the documented rule unless given) + */ + protected function writeKeyIndexFile(string $path, array $header, array $entries, int $base = 0, ?string $layout = null): void + { + ksort($entries); + $live = count($entries); + $maxLocal = $live === 0 ? -1 : max(array_keys($entries)) - $base; + $slots = $maxLocal + 1; + $layout = $layout ?? ($slots <= max(1024, 4 * $live) ? 'dense' : 'sparse'); + $body = ''; + if ($layout === 'dense') { + for ($first = 0; $first < $slots; $first += 341) { + $m = min(341, $slots - $first); + $offsets = ''; + $lengths = ''; + for ($i = 0; $i < $m; $i++) { + [$offset, $length] = $entries[$base + $first + $i] ?? [0, 0]; + $offsets .= pack('P', $offset); + $lengths .= pack('V', $length); } + $chunk = $offsets . $lengths; + $body .= $chunk . pack('V', (int)hexdec(hash('crc32b', $chunk))); } } else { - // No index file - read all records (legacy or new DB) - $lines = explode("\n", trim($contents)); - foreach ($lines as $line) { - $line = trim($line); - if (empty($line)) continue; - $record = json_decode($line, true); - if ($record !== null) { - $key = $record['key'] ?? count($records); - unset($record['key']); - $records[$key] = $record; + $slots = $live; + foreach (array_chunk($entries, 255, true) as $group) { + $keys = ''; + $offsets = ''; + $lengths = ''; + foreach ($group as $key => [$offset, $length]) { + $keys .= pack('V', $key - $base); + $offsets .= pack('P', $offset); + $lengths .= pack('V', $length); } + $chunk = str_pad($keys . $offsets . $lengths, 4092, "\0"); + $body .= $chunk . pack('V', (int)hexdec(hash('crc32b', $chunk))); } } + $header = array_merge(['kind' => 'kix', 'v' => 1], $header, ['layout' => $layout, 'base' => $base, 'slots' => $slots, 'live' => $live]); + file_put_contents($path, str_pad($this->sealHeaderLine(json_encode($header)), 511, ' ') . "\n" . $body); + } - // Sort by key to maintain order - ksort($records); - return ['data' => array_values($records)]; + // ------------------------------------------------------------------ + // Paged field indexes .fx (docs/STORAGE_FORMAT.md §6.5.1), read and written independently + // ------------------------------------------------------------------ + + /** Canonical JSON of .fx data pages (STORAGE_FORMAT.md §6.5.1) */ + private const FX_JSON = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_LINE_TERMINATORS; + + /** + * Decode an .fx file: [header, map]. The map has the shape of the values: bucket id => key when there is + * one, => [keys] (ascending) when there are several. Throws when the file breaks the documented format: + * header, page crc, directory order and coverage, entry encoding. + */ + protected function readFieldIndexFile(string $path): array + { + $raw = (string)file_get_contents($path); + $newline = strpos($raw, "\n"); + if ($newline === false) { + throw new \UnexpectedValueException("$path: no header line"); + } + $header = $this->openHeaderLine($path, substr($raw, 0, $newline)); + if (!is_array($header) || ($header['v'] ?? null) !== 1 || ($header['kind'] ?? null) !== 'fx' || !is_array($header['pg'] ?? null)) { + throw new \UnexpectedValueException("$path: not a paged field index header"); + } + if (array_slice(array_keys($header), 0, 4) !== ['kind', 'v', 'field', 'w'] || !is_string($header['w']) || !preg_match('/^[0-9a-f]{16}$/', $header['w'])) { + throw new \UnexpectedValueException("$path: the header doesn't start with kind, v, field and a 16-hex-digit write token w"); + } + $pg = $header['pg']; + $members = array_keys($pg); + sort($members); + if ($members !== ['data', 'levels', 'pages', 'root']) { + throw new \UnexpectedValueException("$path: pg holds root, levels, pages and data, nothing else"); + } + $body = substr($raw, $newline + 1); + // A page without its crc (checked) + $page = function (int $offset, int $length) use ($body, $path): string { + $bytes = substr($body, $offset, $length); + if (strlen($bytes) !== $length || $length < 6) { + throw new \UnexpectedValueException("$path: page at $offset is outside the body"); + } + $data = substr($bytes, 0, -4); + if (unpack('V', substr($bytes, -4))[1] !== (int)hexdec(hash('crc32b', $data))) { + throw new \UnexpectedValueException("$path: page at $offset crc mismatch"); + } + return $data; + }; + // A page with a table (directory): [entries, table offsets]; the walk below checks that the table names + // exactly the entries it finds + $tabled = function (string $data, int $offset) use ($path): array { + $count = unpack('v', $data, strlen($data) - 2)[1]; + $tableAt = strlen($data) - 2 - 4 * $count; + if ($tableAt < 0) { + throw new \UnexpectedValueException("$path: page at $offset has a table larger than the page"); + } + $table = $count === 0 ? [] : array_values(unpack('V' . $count, $data, $tableAt)); + return [substr($data, 0, $tableAt), $table]; + }; + $checkTable = function (array $starts, array $table, int $offset) use ($path) { + if ($starts !== $table) { + throw new \UnexpectedValueException("$path: the table of the page at $offset doesn't name its entries"); + } + }; + // Walk the directory from the root: every data page once, in order, first ids matching + $dataPages = []; + $walk = function (int $offset, int $length, int $level, ?string $first) use (&$walk, &$dataPages, $page, $tabled, $checkTable, $path) { + if ($level === 0) { + $dataPages[] = [$offset, $length, $first]; + return; + } + [$data, $table] = $tabled($page($offset, $length), $offset); + $starts = []; + for ($o = 0, $previous = null; $o < strlen($data);) { + $starts[] = $o; + $n = unpack('V', $data, $o)[1]; + $id = substr($data, $o + 4, $n); + $child = unpack('V2', $data, $o + 4 + $n); + if ($previous !== null && strcmp($previous, $id) >= 0) { + throw new \UnexpectedValueException("$path: directory not ascending"); + } + $walk($child[1], $child[2], $level - 1, $id); + $previous = $id; + $o += 4 + $n + 8; + } + $checkTable($starts, $table, $offset); + }; + $walk($pg['root'][0], $pg['root'][1], $pg['levels'], null); + $map = []; + $expected = 0; + foreach ($dataPages as [$offset, $length, $first]) { + if ($offset !== $expected) { + throw new \UnexpectedValueException("$path: data pages not contiguous"); + } + $expected = $offset + $length; + $data = $page($offset, $length); + // [id, keys] of the page's entries, in page order: one JSON object, "id":key, or "id":[keys] for + // several keys (ascending); canonical JSON + $entries = []; + $object = json_decode($data, true); + if (!is_array($object) || ($object === [] ? $data !== '{}' : json_encode($object, self::FX_JSON) !== $data)) { + throw new \UnexpectedValueException("$path: the data page at $offset is not a JSON object in canonical form"); + } + foreach ($object as $id => $keys) { + $ok = is_int($keys) || (is_array($keys) && count($keys) !== 1 && $keys === array_values($keys)); + foreach (is_array($keys) ? $keys : [] as $i => $key) { + $ok = $ok && is_int($key) && ($i === 0 || $keys[$i - 1] < $key); + } + if (!$ok) { + throw new \UnexpectedValueException("$path: entry $id of the data page at $offset is not a key or a list of keys"); + } + $entries[] = [(string)$id, is_int($keys) ? [$keys] : $keys]; + } + foreach ($entries as $i => [$id, $values]) { + if ($i === 0 && $first !== null && $first !== $id) { + throw new \UnexpectedValueException("$path: directory entry doesn't name the first id of its page"); + } + if ($map !== [] && strcmp((string)array_key_last($map), $id) >= 0) { + throw new \UnexpectedValueException("$path: entries not ascending"); + } + $map[$id] = count($values) === 1 ? $values[0] : $values; + } + } + if ($expected !== $pg['data']) { + throw new \UnexpectedValueException("$path: data pages end at $expected, the header says {$pg['data']}"); + } + return [$header, $map]; + } + + /** + * Write an .fx file as §6.5.1 describes it ($header: kind, field, s, e, h, ...; pg is computed; $map in the shape + * readFieldIndexFile() returns) + */ + protected function writeFieldIndexFile(string $path, array $header, array $map): void + { + ksort($map, SORT_STRING); + // A page holds about 4096 bytes in all; an entry larger than that is a page of its own. A data page: its + // entries as one JSON object, then the crc. A directory page: entries, their table (u32 offset each, + // then the u16 count), the crc of all that. + $crc = function (string $data): string { + return $data . pack('V', (int)hexdec(hash('crc32b', $data))); + }; + $sealPage = function (array $entries) use ($crc): string { + $data = implode('', $entries); + $offsets = []; + $at = 0; + foreach ($entries as $entry) { + $offsets[] = $at; + $at += strlen($entry); + } + return $crc($data . (empty($offsets) ? '' : pack('V*', ...$offsets)) . pack('v', count($offsets))); + }; + $sealJson = function (array $members) use ($crc): string { + return $crc('{' . implode(',', $members) . '}'); + }; + $fits = function (array $entries, string $entry): bool { + return strlen(implode('', $entries)) + strlen($entry) + 4 * (count($entries) + 1) + 2 + 4 <= 4096; + }; + $fitsJson = function (array $members, string $member): bool { + return strlen('{' . implode(',', array_merge($members, [$member])) . '}') + 4 <= 4096; + }; + $body = ''; + $level = []; + $current = []; + $first = null; + foreach ($map as $id => $values) { + $id = (string)$id; + $values = (array)$values; + sort($values); + $entry = json_encode($id, self::FX_JSON) . ':' . (count($values) === 1 ? (string)$values[0] : json_encode($values)); + if (!empty($current) && !$fitsJson($current, $entry)) { + $sealed = $sealJson($current); + $level[] = [$first, strlen($body), strlen($sealed)]; + $body .= $sealed; + $current = []; + } + if (empty($current)) { + $first = $id; + } + $current[] = $entry; + } + if (!empty($current) || empty($level)) { + $sealed = $sealJson($current); + $level[] = [$first ?? '', strlen($body), strlen($sealed)]; + $body .= $sealed; + } + $dataBytes = strlen($body); + $pages = count($level); + $levels = 0; + while (count($level) > 1) { + $levels++; + $next = []; + $current = []; + $first = null; + foreach ($level as [$id, $offset, $length]) { + $entry = pack('V', strlen($id)) . $id . pack('V2', $offset, $length); + // A directory page holds at least two entries, even above 4096 bytes + if (count($current) >= 2 && !$fits($current, $entry)) { + $sealed = $sealPage($current); + $next[] = [$first, strlen($body), strlen($sealed)]; + $body .= $sealed; + $current = []; + } + if (empty($current)) { + $first = $id; + } + $current[] = $entry; + } + $sealed = $sealPage($current); + $next[] = [$first, strlen($body), strlen($sealed)]; + $body .= $sealed; + $pages += count($next); + $level = $next; + } + // The header starts with kind, v, field and the write token w, in this order; every write has a new + // token (16 random hex digits), as the engine's + $header = ['kind' => $header['kind'], 'v' => 1, 'field' => $header['field'], 'w' => bin2hex(random_bytes(8))] + $header; + $header['pg'] = ['root' => [$level[0][1], $level[0][2]], 'levels' => $levels, 'pages' => $pages, 'data' => $dataBytes]; + file_put_contents($path, $this->sealHeaderLine(json_encode($header)) . "\n" . $body); + } + + // ------------------------------------------------------------------ + // Global field index .gfx (docs/STORAGE_FORMAT.md §6.5.2), read and written independently + // ------------------------------------------------------------------ + + /** The bucket id of an indexable value (§6.5.1): its type letter, then its value */ + protected function bucketIdOf($value): string + { + if ($value === null) { + return 'n'; + } + if (is_bool($value)) { + return $value ? 't' : 'f'; + } + if (is_int($value)) { + return 'i' . $value; + } + if (is_float($value)) { + return 'd' . bin2hex(pack('E', $value + 0.0)); + } + return 's' . $value; + } + + /** The six probe rows of a bucket id in filters of $bits bits */ + protected function filterProbes(string $id, int $bits): array + { + $d = hash('xxh3', $id, true); + $x = unpack('N', substr($d, 0, 4))[1]; + $y = unpack('N', substr($d, 4, 4))[1] | 1; + $rows = []; + for ($i = 0; $i < 6; $i++) { + $rows[] = (($x + $i * $y) % 4294967296) % $bits; + } + return $rows; + } + + /** The filter of a shard holding these values: $bits bits (bit r: byte ⌊r / 8⌋, value 1 << (r mod 8)) */ + protected function shardFilter(array $values, int $bits): string + { + return $this->shardFilterOfIds(array_map([$this, 'bucketIdOf'], $values), $bits); + } + + /** The filter of a shard whose field index holds these bucket ids */ + protected function shardFilterOfIds(array $ids, int $bits): string + { + $filter = str_repeat("\0", intdiv($bits, 8)); + foreach ($ids as $id) { + foreach ($this->filterProbes((string)$id, $bits) as $r) { + $filter[intdiv($r, 8)] = chr(ord($filter[intdiv($r, 8)]) | (1 << ($r % 8))); + } + } + return $filter; + } + + /** True when a filter may hold the value (its six probe bits are set) */ + protected function filterMayHold(string $filter, $value): bool + { + foreach ($this->filterProbes($this->bucketIdOf($value), 8 * strlen($filter)) as $r) { + if ((ord($filter[intdiv($r, 8)]) & (1 << ($r % 8))) === 0) { + return false; + } + } + return true; + } + + /** Rows per page of a .gfx (§6.5.2): P = min(M, max(1, 4096 / (W / 8))) */ + protected function globalPageRows(int $bits, int $width): int + { + return min($bits, max(1, intdiv(4096, intdiv($width, 8)))); + } + + /** Where shard $shard's bit of row $row is in a .gfx file: [byte offset in the file, bit value] */ + protected function globalBitAt(string $path, array $header, int $row, int $shard): array + { + $rb = intdiv($header['width'], 8); + $per = $this->globalPageRows($header['bits'], $header['width']); + $at = strpos((string)file_get_contents($path), "\n") + 1 + intdiv($row, $per) * ($per * $rb + 4) + ($row % $per) * $rb + intdiv($shard, 8); + return [$at, 1 << ($shard % 8)]; + } + + /** + * Decode a .gfx file: [header, filters] (shard id => its filter, $bits bits, for every shard of sh). Throws when + * the file breaks §6.5.2: header members and their order, sizes, shard ids, body length, page crcs. + */ + protected function readGlobalFieldFile(string $path): array + { + $raw = (string)file_get_contents($path); + $newline = strpos($raw, "\n"); + if ($newline === false) { + throw new \UnexpectedValueException("$path: no header line"); + } + $header = $this->openHeaderLine($path, substr($raw, 0, $newline)); + if (!is_array($header) || ($header['kind'] ?? null) !== 'gfx' || ($header['v'] ?? null) !== 1) { + throw new \UnexpectedValueException("$path: not a global field index header"); + } + if (array_slice(array_keys($header), 0, 4) !== ['kind', 'v', 'field', 'w'] || !is_string($header['w']) || !preg_match('/^[0-9a-f]{16}$/', $header['w'])) { + throw new \UnexpectedValueException("$path: the header doesn't start with kind, v, field and a 16-hex-digit write token w"); + } + $bits = $header['bits'] ?? null; + $width = $header['width'] ?? null; + foreach ([[$bits, 64], [$width, 8]] as [$size, $min]) { + if (!is_int($size) || $size < $min || ($size & ($size - 1)) !== 0) { + throw new \UnexpectedValueException("$path: bits (at least 64) and width (at least 8) are powers of two"); + } + } + if (!is_array($header['sh'] ?? null)) { + throw new \UnexpectedValueException("$path: no shard entries"); + } + foreach ($header['sh'] as $shard => $entry) { + if (!is_int($shard) || $shard < 0 || $shard >= $width || !is_array($entry) || !isset($entry['s'], $entry['e'], $entry['h'])) { + throw new \UnexpectedValueException("$path: shard entry $shard is not a shard below the width with s, e and h"); + } + } + $rb = intdiv($width, 8); + $per = $this->globalPageRows($bits, $width); + $body = substr($raw, $newline + 1); + if (strlen($body) !== intdiv($bits, $per) * ($per * $rb + 4)) { + throw new \UnexpectedValueException("$path: the body is not bits / P pages of P rows of width / 8 bytes and a crc"); + } + $rows = ''; + for ($p = 0; $p < intdiv($bits, $per); $p++) { + $page = substr($body, $p * ($per * $rb + 4), $per * $rb); + if (unpack('V', substr($body, $p * ($per * $rb + 4) + $per * $rb, 4))[1] !== (int)hexdec(hash('crc32b', $page))) { + throw new \UnexpectedValueException("$path: page $p crc mismatch"); + } + $rows .= $page; + } + $filters = []; + foreach (array_keys($header['sh']) as $shard) { + $filter = str_repeat("\0", intdiv($bits, 8)); + for ($r = 0; $r < $bits; $r++) { + if (ord($rows[$r * $rb + intdiv($shard, 8)]) & (1 << ($shard % 8))) { + $filter[intdiv($r, 8)] = chr(ord($filter[intdiv($r, 8)]) | (1 << ($r % 8))); + } + } + $filters[$shard] = $filter; + } + return [$header, $filters]; + } + + /** + * Write a .gfx file as §6.5.2 describes it ($header: field, le, bits, width, sh; $filters: shard => its filter; + * every other bit zero) + */ + protected function writeGlobalFieldFile(string $path, array $header, array $filters): void + { + $bits = $header['bits']; + $rb = intdiv($header['width'], 8); + $rows = str_repeat("\0", $bits * $rb); + foreach ($filters as $shard => $filter) { + for ($r = 0; $r < $bits; $r++) { + if (ord($filter[intdiv($r, 8)]) & (1 << ($r % 8))) { + $at = $r * $rb + intdiv($shard, 8); + $rows[$at] = chr(ord($rows[$at]) | (1 << ($shard % 8))); + } + } + } + $per = $this->globalPageRows($bits, $header['width']); + $body = ''; + for ($p = 0; $p < intdiv($bits, $per); $p++) { + $page = substr($rows, $p * $per * $rb, $per * $rb); + $body .= $page . pack('V', (int)hexdec(hash('crc32b', $page))); + } + $header = ['kind' => 'gfx', 'v' => 1, 'field' => $header['field'], 'w' => bin2hex(random_bytes(8))] + $header; + $header['sh'] = (object)$header['sh']; // an object, also for shards 0 … n + file_put_contents($path, $this->sealHeaderLine(json_encode($header)) . "\n" . $body); + } + + /** + * The key index a data file implies (independent model): live key => [offset, length] of its last line + */ + protected function keyIndexOfData(string $dataFile): array + { + $entries = []; + $offset = 0; + foreach (explode("\n", (string)file_get_contents($dataFile)) as $i => $line) { + $record = json_decode($line, true); + if (is_array($record) && count($record) === 1 && isset($record['__nonedb_deleted'])) { + unset($entries[$record['__nonedb_deleted']]); + } elseif (is_array($record) && isset($record['key'])) { + $entries[$record['key']] = [$offset, strlen($line)]; + } + $offset += strlen($line) + 1; + } + ksort($entries); + return $entries; } /** @@ -289,8 +951,8 @@ protected function createTestInstance(): \noneDB 'dbDir' => $this->testDbDir, 'autoCreateDB' => true, 'shardingEnabled' => true, - 'shardSize' => 10000, - 'autoMigrate' => true + 'shardSize' => 10000 ]); } } + diff --git a/tests/sleekdb_vs_nonedb_benchmark.php b/tests/sleekdb_vs_nonedb_benchmark.php deleted file mode 100644 index 14233b1..0000000 --- a/tests/sleekdb_vs_nonedb_benchmark.php +++ /dev/null @@ -1,420 +0,0 @@ -= 1000) return round($ms / 1000, 2) . " s"; - return round($ms, 1) . " ms"; -} - -// Format memory -function formatMemory($bytes) { - if ($bytes >= 1073741824) return round($bytes / 1073741824, 2) . " GB"; - if ($bytes >= 1048576) return round($bytes / 1048576, 1) . " MB"; - return round($bytes / 1024, 1) . " KB"; -} - -// Generate test record -function generateRecord($i) { - $cities = ['Istanbul', 'Ankara', 'Izmir', 'Bursa', 'Antalya']; - $depts = ['IT', 'HR', 'Sales', 'Marketing', 'Finance']; - return [ - "name" => "User" . $i, - "email" => "user{$i}@test.com", - "age" => 20 + ($i % 50), - "salary" => 5000 + ($i % 10000), - "city" => $cities[$i % 5], - "department" => $depts[$i % 5], - "active" => ($i % 3 !== 0) - ]; -} - -// Test directories -$sleekDbDir = sys_get_temp_dir() . '/sleekdb_bench/'; -$noneDbDir = sys_get_temp_dir() . '/nonedb_bench/'; - -// Cleanup function -function cleanup($sleekDbDir, $noneDbDir) { - // Remove SleekDB files - if (is_dir($sleekDbDir)) { - $files = new RecursiveIteratorIterator( - new RecursiveDirectoryIterator($sleekDbDir, RecursiveDirectoryIterator::SKIP_DOTS), - RecursiveIteratorIterator::CHILD_FIRST - ); - foreach ($files as $file) { - $file->isDir() ? rmdir($file->getRealPath()) : unlink($file->getRealPath()); - } - rmdir($sleekDbDir); - } - - // Remove noneDB files - $noneFiles = glob($noneDbDir . '*'); - foreach ($noneFiles as $f) { - if (is_file($f)) @unlink($f); - } - if (is_dir($noneDbDir)) @rmdir($noneDbDir); -} - -// Create directories -if (!is_dir($sleekDbDir)) mkdir($sleekDbDir, 0777, true); -if (!is_dir($noneDbDir)) mkdir($noneDbDir, 0777, true); - -echo blue("╔══════════════════════════════════════════════════════════════════════╗\n"); -echo blue("║ SleekDB vs noneDB Performance Benchmark ║\n"); -echo blue("╚══════════════════════════════════════════════════════════════════════╝\n\n"); - -echo "PHP Version: " . PHP_VERSION . "\n"; -echo "SleekDB: v2.15 (cache OFF)\n"; -echo "noneDB: v2.3.0 (sharding ON, buffer ON/OFF)\n\n"; - -// Test sizes -$sizes = isset($argv[1]) ? array_map('intval', explode(',', $argv[1])) : [100, 1000, 10000, 50000, 100000]; - -// Results storage -$results = []; - -foreach ($sizes as $size) { - echo yellow("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"); - echo yellow(" Testing with " . number_format($size) . " records\n"); - echo yellow("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n"); - - // Cleanup before test - cleanup($sleekDbDir, $noneDbDir); - if (!is_dir($sleekDbDir)) mkdir($sleekDbDir, 0777, true); - if (!is_dir($noneDbDir)) mkdir($noneDbDir, 0777, true); - - // Prepare test data - $data = []; - for ($i = 0; $i < $size; $i++) { - $data[] = generateRecord($i); - } - - $results[$size] = [ - 'sleekdb' => [], - 'nonedb_default' => [], - 'nonedb_nobuffer' => [] - ]; - - // ===================================================================== - // SLEEKDB TESTS - // ===================================================================== - echo cyan(" ┌─ SleekDB (cache OFF) ─────────────────────────────────────────────┐\n"); - - $sleekConfig = [ - "auto_cache" => false, - "cache_lifetime" => null, - "timeout" => false - ]; - - // Bulk Insert - $store = new Store("benchmark", $sleekDbDir, $sleekConfig); - gc_collect_cycles(); - $memBefore = memory_get_usage(true); - $start = microtime(true); - $store->insertMany($data); - $sleekBulkInsert = (microtime(true) - $start) * 1000; - $sleekBulkMem = memory_get_peak_usage(true) - $memBefore; - $results[$size]['sleekdb']['bulk_insert'] = $sleekBulkInsert; - $results[$size]['sleekdb']['bulk_insert_mem'] = $sleekBulkMem; - echo " │ Bulk Insert: " . green(formatTime($sleekBulkInsert)) . " (mem: " . formatMemory($sleekBulkMem) . ")\n"; - - // Find All - gc_collect_cycles(); - $start = microtime(true); - $allData = $store->findAll(); - $sleekFindAll = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['find_all'] = $sleekFindAll; - echo " │ Find All: " . green(formatTime($sleekFindAll)) . "\n"; - - // Find by ID - $testId = (int)($size / 2); - gc_collect_cycles(); - $start = microtime(true); - $record = $store->findById($testId); - $sleekFindId = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['find_id'] = $sleekFindId; - echo " │ Find by ID: " . green(formatTime($sleekFindId)) . "\n"; - - // Find by Filter - gc_collect_cycles(); - $start = microtime(true); - $filtered = $store->findBy(["city", "=", "Istanbul"]); - $sleekFindFilter = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['find_filter'] = $sleekFindFilter; - echo " │ Find by Filter: " . green(formatTime($sleekFindFilter)) . "\n"; - - // Count - gc_collect_cycles(); - $start = microtime(true); - $count = $store->count(); - $sleekCount = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['count'] = $sleekCount; - echo " │ Count: " . green(formatTime($sleekCount)) . "\n"; - - // Sequential Insert (100 records on existing DB) - gc_collect_cycles(); - $start = microtime(true); - for ($i = 0; $i < 100; $i++) { - $store->insert(generateRecord($size + $i)); - } - $sleekSeqInsert = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['seq_insert'] = $sleekSeqInsert; - echo " │ Seq Insert (100): " . green(formatTime($sleekSeqInsert)) . "\n"; - - // Update (using QueryBuilder) - gc_collect_cycles(); - $start = microtime(true); - $store->createQueryBuilder() - ->where(["city", "=", "Istanbul"]) - ->getQuery() - ->update(["region" => "Marmara"]); - $sleekUpdate = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['update'] = $sleekUpdate; - echo " │ Update: " . green(formatTime($sleekUpdate)) . "\n"; - - // Delete (using QueryBuilder) - gc_collect_cycles(); - $start = microtime(true); - $store->createQueryBuilder() - ->where(["department", "=", "HR"]) - ->getQuery() - ->delete(); - $sleekDelete = (microtime(true) - $start) * 1000; - $results[$size]['sleekdb']['delete'] = $sleekDelete; - echo " │ Delete: " . green(formatTime($sleekDelete)) . "\n"; - - echo cyan(" └──────────────────────────────────────────────────────────────────────┘\n\n"); - - // Cleanup SleekDB - cleanup($sleekDbDir, $noneDbDir); - if (!is_dir($noneDbDir)) mkdir($noneDbDir, 0777, true); - - // ===================================================================== - // NONEDB (Buffer ON, Sharding ON; the default is buffer OFF since v3.2) - // ===================================================================== - echo magenta(" ┌─ noneDB (buffer ON, sharding ON) ────────────────────────────────┐\n"); - - $nonedb = new noneDB(['secretKey' => 'nonedb-benchmark-key-not-for-production', 'dbDir' => $noneDbDir, 'bufferEnabled' => true]); - - // Bulk Insert - gc_collect_cycles(); - $memBefore = memory_get_usage(true); - $start = microtime(true); - $nonedb->insert("benchmark", $data); - $nonedb->flush("benchmark"); - $noneBulkInsert = (microtime(true) - $start) * 1000; - $noneBulkMem = memory_get_peak_usage(true) - $memBefore; - $results[$size]['nonedb_default']['bulk_insert'] = $noneBulkInsert; - $results[$size]['nonedb_default']['bulk_insert_mem'] = $noneBulkMem; - echo " │ Bulk Insert: " . green(formatTime($noneBulkInsert)) . " (mem: " . formatMemory($noneBulkMem) . ")\n"; - - // Find All - gc_collect_cycles(); - $start = microtime(true); - $allData = $nonedb->find("benchmark", []); - $noneFindAll = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['find_all'] = $noneFindAll; - echo " │ Find All: " . green(formatTime($noneFindAll)) . "\n"; - - // Find by Key - $testKey = (int)($size / 2); - gc_collect_cycles(); - $start = microtime(true); - $record = $nonedb->find("benchmark", ["key" => $testKey]); - $noneFindKey = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['find_id'] = $noneFindKey; - echo " │ Find by Key: " . green(formatTime($noneFindKey)) . "\n"; - - // Find by Filter - gc_collect_cycles(); - $start = microtime(true); - $filtered = $nonedb->find("benchmark", ["city" => "Istanbul"]); - $noneFindFilter = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['find_filter'] = $noneFindFilter; - echo " │ Find by Filter: " . green(formatTime($noneFindFilter)) . "\n"; - - // Count - gc_collect_cycles(); - $start = microtime(true); - $count = $nonedb->count("benchmark"); - $noneCount = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['count'] = $noneCount; - echo " │ Count: " . green(formatTime($noneCount)) . "\n"; - - // Sequential Insert (100 records with buffer) - gc_collect_cycles(); - $start = microtime(true); - for ($i = 0; $i < 100; $i++) { - $nonedb->insert("benchmark", generateRecord($size + $i)); - } - $nonedb->flush("benchmark"); - $noneSeqInsert = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['seq_insert'] = $noneSeqInsert; - echo " │ Seq Insert (100): " . green(formatTime($noneSeqInsert)) . "\n"; - - // Update - gc_collect_cycles(); - $start = microtime(true); - $nonedb->update("benchmark", [["city" => "Istanbul"], ["set" => ["region" => "Marmara"]]]); - $noneUpdate = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['update'] = $noneUpdate; - echo " │ Update: " . green(formatTime($noneUpdate)) . "\n"; - - // Delete - gc_collect_cycles(); - $start = microtime(true); - $nonedb->delete("benchmark", ["department" => "HR"]); - $noneDelete = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_default']['delete'] = $noneDelete; - echo " │ Delete: " . green(formatTime($noneDelete)) . "\n"; - - echo magenta(" └──────────────────────────────────────────────────────────────────────┘\n\n"); - - // Cleanup noneDB - $noneFiles = glob($noneDbDir . '*'); - foreach ($noneFiles as $f) @unlink($f); - - // ===================================================================== - // NONEDB (Buffer OFF) - // ===================================================================== - echo magenta(" ┌─ noneDB (buffer OFF, sharding ON) ────────────────────────────────┐\n"); - - $nonedb2 = new noneDB(['secretKey' => 'nonedb-benchmark-key-not-for-production', 'dbDir' => $noneDbDir, 'bufferEnabled' => false]); - - // Bulk Insert (no buffer) - gc_collect_cycles(); - $memBefore = memory_get_usage(true); - $start = microtime(true); - $nonedb2->insert("benchmark", $data); - $noneNoBufBulk = (microtime(true) - $start) * 1000; - $noneNoBufMem = memory_get_peak_usage(true) - $memBefore; - $results[$size]['nonedb_nobuffer']['bulk_insert'] = $noneNoBufBulk; - $results[$size]['nonedb_nobuffer']['bulk_insert_mem'] = $noneNoBufMem; - echo " │ Bulk Insert: " . green(formatTime($noneNoBufBulk)) . " (mem: " . formatMemory($noneNoBufMem) . ")\n"; - - // Find All - gc_collect_cycles(); - $start = microtime(true); - $allData = $nonedb2->find("benchmark", []); - $noneNoBufFindAll = (microtime(true) - $start) * 1000; - $results[$size]['nonedb_nobuffer']['find_all'] = $noneNoBufFindAll; - echo " │ Find All: " . green(formatTime($noneNoBufFindAll)) . "\n"; - - // Sequential Insert (only 10 - no buffer is SLOW on large DB) - $seqCount = ($size >= 50000) ? 10 : 100; - gc_collect_cycles(); - $start = microtime(true); - for ($i = 0; $i < $seqCount; $i++) { - $nonedb2->insert("benchmark", generateRecord($size + $i)); - } - $noneNoBufSeq = (microtime(true) - $start) * 1000; - $noneNoBufSeqNorm = ($seqCount == 10) ? $noneNoBufSeq * 10 : $noneNoBufSeq; // Normalize to 100 - $results[$size]['nonedb_nobuffer']['seq_insert'] = $noneNoBufSeqNorm; - echo " │ Seq Insert (" . $seqCount . "): " . green(formatTime($noneNoBufSeq)) . ($seqCount == 10 ? " (×10 = " . formatTime($noneNoBufSeqNorm) . ")" : "") . "\n"; - - echo magenta(" └──────────────────────────────────────────────────────────────────────┘\n\n"); - - // Cleanup - cleanup($sleekDbDir, $noneDbDir); - if (!is_dir($sleekDbDir)) mkdir($sleekDbDir, 0777, true); - if (!is_dir($noneDbDir)) mkdir($noneDbDir, 0777, true); -} - -// Final cleanup -cleanup($sleekDbDir, $noneDbDir); - -// ===================================================================== -// PRINT MARKDOWN TABLES -// ===================================================================== -echo blue("\n╔══════════════════════════════════════════════════════════════════════╗\n"); -echo blue("║ MARKDOWN TABLES FOR README ║\n"); -echo blue("╚══════════════════════════════════════════════════════════════════════╝\n\n"); - -echo "## SleekDB vs noneDB Performance Comparison\n\n"; -echo "Tested on PHP " . PHP_VERSION . ", " . PHP_OS . "\n\n"; - -// Bulk Insert Table -echo "### Bulk Insert\n"; -echo "| Records | SleekDB | noneDB (buffer) | noneDB (no buffer) |\n"; -echo "|---------|---------|-----------------|--------------------|\n"; -foreach ($sizes as $size) { - $label = $size >= 1000 ? ($size / 1000) . "K" : $size; - $sleek = formatTime($results[$size]['sleekdb']['bulk_insert']); - $noneB = formatTime($results[$size]['nonedb_default']['bulk_insert']); - $noneNB = formatTime($results[$size]['nonedb_nobuffer']['bulk_insert']); - echo "| {$label} | {$sleek} | {$noneB} | {$noneNB} |\n"; -} -echo "\n"; - -// Sequential Insert Table -echo "### Sequential Insert (100 records on existing DB)\n"; -echo "| Records | SleekDB | noneDB (buffer) | noneDB (no buffer) |\n"; -echo "|---------|---------|-----------------|--------------------|\n"; -foreach ($sizes as $size) { - $label = $size >= 1000 ? ($size / 1000) . "K" : $size; - $sleek = formatTime($results[$size]['sleekdb']['seq_insert']); - $noneB = formatTime($results[$size]['nonedb_default']['seq_insert']); - $noneNB = formatTime($results[$size]['nonedb_nobuffer']['seq_insert'] ?? 0); - echo "| {$label} | {$sleek} | {$noneB} | {$noneNB} |\n"; -} -echo "\n"; - -// Find All Table -echo "### Find All Records\n"; -echo "| Records | SleekDB | noneDB |\n"; -echo "|---------|---------|--------|\n"; -foreach ($sizes as $size) { - $label = $size >= 1000 ? ($size / 1000) . "K" : $size; - $sleek = formatTime($results[$size]['sleekdb']['find_all']); - $none = formatTime($results[$size]['nonedb_default']['find_all']); - echo "| {$label} | {$sleek} | {$none} |\n"; -} -echo "\n"; - -// Find by ID Table -echo "### Find by ID/Key\n"; -echo "| Records | SleekDB | noneDB |\n"; -echo "|---------|---------|--------|\n"; -foreach ($sizes as $size) { - $label = $size >= 1000 ? ($size / 1000) . "K" : $size; - $sleek = formatTime($results[$size]['sleekdb']['find_id']); - $none = formatTime($results[$size]['nonedb_default']['find_id']); - echo "| {$label} | {$sleek} | {$none} |\n"; -} -echo "\n"; - -// Memory Usage Table -echo "### Memory Usage (Bulk Insert)\n"; -echo "| Records | SleekDB | noneDB |\n"; -echo "|---------|---------|--------|\n"; -foreach ($sizes as $size) { - $label = $size >= 1000 ? ($size / 1000) . "K" : $size; - $sleek = formatMemory($results[$size]['sleekdb']['bulk_insert_mem']); - $none = formatMemory($results[$size]['nonedb_default']['bulk_insert_mem']); - echo "| {$label} | {$sleek} | {$none} |\n"; -} -echo "\n"; - -echo green("\nBenchmark completed!\n"); diff --git a/tools/build.php b/tools/build.php new file mode 100644 index 0000000..0992f00 --- /dev/null +++ b/tools/build.php @@ -0,0 +1,292 @@ + {", the bodies of its modules in order and "}". + * - noneDB-migrate.php (the migration tool, docs/MIGRATION.md) from migrate/: migrate/header.php, every class file of + * migrate/manifest.php in a "namespace noneDB\Migrate... { ... }" block, then migrate/footer.php (its + * "namespace { ... }" block: the command line entry). The tool loads noneDB.php; the engine knows nothing of it. + * + * php tools/build.php write both (each only when its content changes) + * php tools/build.php --check exit 1 when an artifact is not the build of its sources + * php tools/build.php --stdout print the build of noneDB.php + * php tools/build.php --map line ranges of the modules in noneDB.php + * php tools/build.php --test FILE the test build of noneDB.php to FILE + * php tools/build.php --test-tool FILE the test build of noneDB-migrate.php to FILE + * + * A class file /.../.php is "...;\n" and the rest of the + * file (use statements and one class named ), copied as it is. + * A module file src//.php is "_ {\n", + * the body and "}\n", LF line ends only. The traits are never loaded: they only let editors understand + * the files. The output depends on nothing but the files. + * + * Test seams: the artifacts hold only what they need to run. A line ending in "// @test" (fault points, test hooks + * and counters) and the lines from one ending in "// @test-begin" to one ending in "// @test-end" are left out of + * them; a line ending in "// @prod" is its production form and is left out of the test build ($testBuild), which + * the tests load (tests/bootstrap.php writes both to tests/build/). + */ + +if (!function_exists('nonedb_build')) { + /** + * @return array ['code' => string, 'map' => list of [class, module, first line, last line]] + * @throws RuntimeException when a file is missing or malformed, or the result isn't valid PHP + */ + function nonedb_build(string $root, bool $testBuild = false): array + { + $src = rtrim($root, '/') . '/src/'; + nonedb_build_read($src . 'manifest.php'); // exists, LF only + $manifest = (static function (string $file) { + return require $file; + })($src . 'manifest.php'); + if (!is_array($manifest) || !isset($manifest['header'], $manifest['classes']) || !is_array($manifest['classes'])) { + throw new RuntimeException('src/manifest.php must return [header, classes]'); + } + $code = nonedb_build_read($src . $manifest['header']); + $read = function (string $file) use ($testBuild): string { + return nonedb_build_variant(nonedb_build_read($file), $testBuild, $file); + }; + $map = []; + $expected = []; + foreach ($manifest['files'] ?? [] as $file) { + nonedb_build_class($src, $file, 'noneDB\\Internal', $read, $code, $map, $expected); + } + $code .= "namespace {\n"; + foreach ($manifest['classes'] as $class) { + $name = $class['name']; + $expected[] = $name; + $code .= $class['before'] . "class $name {\n"; + foreach ($class['modules'] as $module) { + $file = "$name/$module.php"; + $content = $read($src . $file); + if (!preg_match('/\A<\?php\n(?:\/\/[^\n]*\n)*trait (\w+) \{\n(.*)\}\n\z/s', $content, $m)) { + throw new RuntimeException("src/$file: expected \" $code, 'map' => $map]; + } + + /** + * One variant of a source file: production (lines marked "// @test" and "// @test-begin" … "// @test-end" + * blocks left out, "// @prod" markers removed) or the test build (lines marked "// @prod" left out) + */ + function nonedb_build_variant(string $content, bool $testBuild, string $file): string + { + $out = []; + $inBlock = false; + foreach (explode("\n", $content) as $n => $line) { + $marker = preg_match('#//\s*@(test-begin|test-end|test|prod)\s*$#', $line, $m) ? $m[1] : null; + if ($marker === 'test-begin') { + if ($inBlock) { + throw new RuntimeException("$file:" . ($n + 1) . ': nested @test-begin'); + } + $inBlock = true; + } + $keep = $testBuild ? $marker !== 'prod' : !$inBlock && $marker !== 'test'; + if ($marker === 'test-end') { + if (!$inBlock) { + throw new RuntimeException("$file:" . ($n + 1) . ': @test-end without @test-begin'); + } + $inBlock = false; + } + if ($keep) { + $out[] = $marker === 'prod' ? rtrim(preg_replace('#\s*//\s*@prod\s*$#', '', $line)) : $line; + } + } + if ($inBlock) { + throw new RuntimeException("$file: @test-begin without @test-end"); + } + return implode("\n", $out); + } + + /** + * One class file of a manifest into a bracketed namespace block of $code: "...;" and the rest, which declares the class named as the file + */ + function nonedb_build_class(string $dir, string $file, string $prefix, callable $read, string &$code, array &$map, array &$expected): void + { + $content = $read($dir . $file); + if (!preg_match('/\A<\?php\n(?:\/\/[^\n]*\n)*namespace (' . preg_quote($prefix, '/') . '(?:\\\\\w+)*);\n(.*)\z/s', $content, $m)) { + throw new RuntimeException("$dir$file: expected \" string, 'map' => list of [namespace, class, first line, last line]] + * @throws RuntimeException when a file is missing or malformed, or the result isn't valid PHP + */ + function nonedb_build_tool(string $root, bool $testBuild = false): array + { + $dir = rtrim($root, '/') . '/migrate/'; + nonedb_build_read($dir . 'manifest.php'); + $manifest = (static function (string $file) { + return require $file; + })($dir . 'manifest.php'); + if (!is_array($manifest) || !isset($manifest['header'], $manifest['files'], $manifest['footer']) || !is_array($manifest['files'])) { + throw new RuntimeException('migrate/manifest.php must return [header, files, footer]'); + } + $read = function (string $file) use ($testBuild): string { + return nonedb_build_variant(nonedb_build_read($file), $testBuild, $file); + }; + $code = nonedb_build_read($dir . $manifest['header']); + $map = []; + $expected = []; + foreach ($manifest['files'] as $file) { + nonedb_build_class($dir, $file, 'noneDB\\Migrate', $read, $code, $map, $expected); + } + if (!preg_match('/\A<\?php\n(?:\/\/[^\n]*\n)*(namespace \{\n.*\}\n)\z/s', $read($dir . $manifest['footer']), $m)) { + throw new RuntimeException('migrate/' . $manifest['footer'] . ': expected " $code, 'map' => $map]; + } + + function nonedb_build_read(string $file): string + { + if (!is_file($file)) { + throw new RuntimeException("missing $file"); + } + $content = (string)file_get_contents($file); + if (strpos($content, "\r") !== false) { + throw new RuntimeException("$file: CR characters (use LF line ends)"); + } + return $content; + } + + /** + * The build declares exactly the expected classes, no trait, has balanced braces and passes php -l + */ + function nonedb_build_verify(string $code, array $classes): void + { + $tokens = token_get_all($code); + $declared = []; + $depth = 0; + $previous = null; + foreach ($tokens as $i => $token) { + if (is_array($token)) { + if ($token[0] === T_TRAIT) { + throw new RuntimeException('the build contains a trait'); + } + if ($token[0] === T_CURLY_OPEN || $token[0] === T_DOLLAR_OPEN_CURLY_BRACES) { + $depth++; + } + if ($token[0] === T_CLASS && $previous !== T_DOUBLE_COLON && $previous !== T_NEW) { + for ($j = $i + 1; is_array($tokens[$j]) && $tokens[$j][0] === T_WHITESPACE; $j++) { + } + $declared[] = is_array($tokens[$j]) ? $tokens[$j][1] : ''; + } + if ($token[0] !== T_WHITESPACE && $token[0] !== T_COMMENT && $token[0] !== T_DOC_COMMENT) { + $previous = $token[0]; + } + continue; + } + if ($token === '{') { + $depth++; + } elseif ($token === '}') { + if (--$depth < 0) { + throw new RuntimeException('unbalanced braces in the build'); + } + } + $previous = $token; + } + if ($depth !== 0) { + throw new RuntimeException('unbalanced braces in the build'); + } + if ($declared !== $classes) { + throw new RuntimeException('the build must declare exactly the classes ' . implode(', ', $classes) . ', found: ' . implode(', ', $declared)); + } + $process = proc_open([PHP_BINARY, '-n', '-l'], [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + if (!is_resource($process)) { + throw new RuntimeException('cannot run php -l'); + } + fwrite($pipes[0], $code); + fclose($pipes[0]); + $output = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + if (proc_close($process) !== 0) { + throw new RuntimeException('php -l: ' . trim($output)); + } + } +} + +if (PHP_SAPI === 'cli' && isset($argv[0]) && realpath($argv[0]) === __FILE__) { + $root = dirname(__DIR__); + $mode = $argv[1] ?? ''; + $usage = "usage: php tools/build.php [--check|--stdout|--map|--test |--test-tool ]\n"; + if (!in_array($mode, ['', '--check', '--stdout', '--map', '--test', '--test-tool'], true)) { + fwrite(STDERR, $usage); + exit(2); + } + try { + $builds = [ + 'noneDB.php' => nonedb_build($root, $mode === '--test'), + 'noneDB-migrate.php' => nonedb_build_tool($root, $mode === '--test-tool'), + ]; + } catch (RuntimeException $e) { + fwrite(STDERR, 'build failed: ' . $e->getMessage() . "\n"); + exit(1); + } + if ($mode === '--test' || $mode === '--test-tool') { + // A test build (seams included) to a file the caller names + $code = $builds[$mode === '--test' ? 'noneDB.php' : 'noneDB-migrate.php']['code']; + $file = $argv[2] ?? ''; + if ($file === '' || file_put_contents($file, $code) !== strlen($code)) { + fwrite(STDERR, $usage); + exit(2); + } + exit(0); + } + if ($mode === '--stdout') { + echo $builds['noneDB.php']['code']; + exit(0); + } + if ($mode === '--map') { + foreach ($builds['noneDB.php']['map'] as [$class, $module, $first, $last]) { + printf("%-32s %-24s %6d-%d\n", $class, $module, $first, $last); + } + exit(0); + } + $status = 0; + foreach ($builds as $name => $build) { + $target = "$root/$name"; + $current = is_file($target) ? (string)file_get_contents($target) : null; + if ($current === $build['code']) { + echo "$name is up to date\n"; + continue; + } + if ($mode === '--check') { + fwrite(STDERR, "$name is not the build of its sources: edit them and run php tools/build.php\n"); + $status = 1; + continue; + } + $tmp = $target . '.tmp' . bin2hex(random_bytes(4)); + if (file_put_contents($tmp, $build['code']) !== strlen($build['code']) || !rename($tmp, $target)) { + @unlink($tmp); + fwrite(STDERR, "cannot write $name\n"); + exit(1); + } + echo "$name built (" . substr_count($build['code'], "\n") . " lines)\n"; + } + exit($status); +} diff --git a/tools/make-migration-fixtures.php b/tools/make-migration-fixtures.php new file mode 100644 index 0000000..d7699ff --- /dev/null +++ b/tools/make-migration-fixtures.php @@ -0,0 +1,451 @@ + ['git:620ca05', $php74, 'v1'], + '2.0' => ['git:v2.0.0', $php74, 'v2'], + '2.1' => ['git:v2.1.0', $php74, 'v2'], + '2.2' => ['git:v2.2.0', $php74, 'v2'], + '2.3' => ['git:20dab4b', $php74, 'v2'], + '3.0' => ['git:v3.0.0', $php74, 'v3'], + '3.1' => ['git:v3.1.0', $php74, 'v3'], + '3.2' => ['git:v3.2.0', $php8, 'v3'], + '3.3-dev' => ['file:build/baseline/noneDB-3.3dev.php', $php8, 'v3'], +]; + +// ---------------------------------------------------------------- scenarios (adapter $a, the release's $db) + +$basic = <<<'PHP' +$a->insert('users', [ + ['n' => 'a', 'i' => 1, 'f' => 2.5, 'z' => 2.0, 'b' => true, 'u' => 'çğş/€'], + ['n' => 'b', 'i' => 2, 'nested' => ['x' => 1, 'y' => [1, 2]], 'nil' => null], + ['n' => 'c', 'i' => 3], + ['n' => 'd', 'i' => 4, 'list' => [3, 2, 1]], + ['n' => 'e', 'i' => 5, 'b' => false], + ['n' => 'f', 'i' => 6], +]); +$a->update('users', ['key' => 1], ['n' => 'b2', 'extra' => 'x']); +$a->delete('users', ['key' => 2]); +$a->delete('users', ['key' => 5]); +PHP; +$sharded = <<<'PHP' +$rows = []; +for ($i = 0; $i < 23; $i++) { + $rows[] = ['n' => $i, 'c' => 'c' . ($i % 3)]; +} +$a->insert('users', $rows); +$a->update('users', ['key' => 3], ['c' => 'c1']); +$a->delete('users', ['key' => 7]); +$a->delete('users', ['key' => 22]); +PHP; +$compacted = <<<'PHP' +$rows = []; +for ($i = 0; $i < 12; $i++) { + $rows[] = ['n' => $i]; +} +$a->insert('users', $rows); +foreach ([2, 3, 8] as $k) { + $a->delete('users', ['key' => $k]); +} +$a->compact('users'); +PHP; +$deletes = <<<'PHP' +$rows = []; +for ($i = 0; $i < 30; $i++) { + $rows[] = ['n' => $i]; +} +$a->insert('users', $rows); +foreach ([0, 4, 29] as $k) { + $a->delete('users', ['key' => $k]); +} +$a->update('users', ['key' => 5], ['n' => 50]); +$a->update('users', ['key' => 5], ['n' => 500]); +PHP; +$buffered = <<<'PHP' +$a->buffer(true); +$a->insert('users', [['n' => 'a']]); +$a->insert('users', [['n' => 'b']]); +$a->insert('users', [['n' => 'c'], ['n' => 'd']]); +$a->kill(); +PHP; +$shardedBuffered = $sharded . "\n" . <<<'PHP' +$a->buffer(true); +$a->insert('users', [['n' => 'x']]); +$a->insert('users', [['n' => 'y'], ['n' => 'z']]); +$a->kill(); +PHP; +$fieldIndex = <<<'PHP' +$rows = []; +foreach (['ist', 'ank', 'izm', 'ist', 'bur', 'ist', 'ank', 'izm'] as $i => $c) { + $rows[] = ['n' => $i, 'city' => $c]; +} +$a->insert('users', $rows); +$a->createFieldIndex('users', 'city'); +$a->insert('users', [['n' => 8, 'city' => 'ist'], ['n' => 9, 'city' => 'van']]); +$a->delete('users', ['key' => 2]); +PHP; +$spatial = <<<'PHP' +$rows = []; +foreach ([[28.97, 41.01], [32.85, 39.93], [27.14, 38.42], [29.06, 40.19], [30.71, 36.89], [43.38, 38.5]] as $i => $p) { + $rows[] = ['n' => $i, 'loc' => ['type' => 'Point', 'coordinates' => $p]]; +} +$a->insert('users', $rows); +$a->createSpatialIndex('users', 'loc'); +$a->delete('users', ['key' => 1]); +PHP; +$v1Quirks = <<<'PHP' +$a->insert('users', [['n' => 'a'], ['n' => 'b']]); +clearstatcache(); +$db->insert('users', ['x', 'y']); +$a->update('users', ['key' => 0], ['extra' => 1]); +$a->delete('users', ['key' => 9]); +PHP; +$twoKeys = <<<'PHP' +$a->insert('users', [['n' => 'mine']]); +$other = $a->open(['secretKey' => V3_OTHER_KEY]); +$other->insert('orders', [['n' => 'theirs']]); +PHP; + +$v2Releases = ['2.0', '2.1', '2.2', '2.3']; +$allReleases = array_keys($releases); +$fixtures = []; +foreach ($allReleases as $r) { + $fixtures["$r-basic"] = ['steps' => [[$r, [], $basic]]]; + $fixtures["$r-deletes"] = ['steps' => [[$r, [], $deletes]]]; +} +foreach (array_merge($v2Releases, ['3.0', '3.1', '3.2', '3.3-dev']) as $r) { + $fixtures["$r-sharded"] = ['steps' => [[$r, ['shardSize' => 5], $sharded]]]; + $fixtures["$r-compacted"] = ['steps' => [[$r, ['shardSize' => 5], $compacted]]]; + $fixtures["$r-single-compacted"] = ['steps' => [[$r, [], $compacted]]]; +} +foreach (['2.3', '3.0', '3.1', '3.2', '3.3-dev'] as $r) { + $fixtures["$r-buffered"] = ['steps' => [[$r, [], $buffered]]]; + $fixtures["$r-sharded-buffered"] = ['steps' => [[$r, ['shardSize' => 5], $shardedBuffered]]]; +} +foreach (['3.0', '3.1', '3.2', '3.3-dev'] as $r) { + $fixtures["$r-field-index"] = ['steps' => [[$r, [], $fieldIndex]]]; + $fixtures["$r-sharded-field-index"] = ['steps' => [[$r, ['shardSize' => 5], $fieldIndex]]]; +} +foreach (['3.1', '3.2', '3.3-dev'] as $r) { + $fixtures["$r-spatial"] = ['steps' => [[$r, [], $spatial]]]; +} +$fixtures['1.4-quirks'] = ['steps' => [['1.4', [], $v1Quirks]]]; +$fixtures['2.2-empty-name'] = ['steps' => [['2.2', [], "\$a->insert('!!', [['n' => 1]]);\n\$a->insert('users', [['n' => 1]]);"]]]; +$fixtures['3.2-two-keys'] = ['steps' => [['3.2', [], $twoKeys]]]; +$fixtures['3.2-two-databases'] = ['steps' => [['3.2', ['shardSize' => 5], $sharded . "\n" . str_replace("'users'", "'orders'", $fieldIndex)]]]; +// One database written by several releases, each opening what the one before left +$fixtures['2.2-3.1-3.2-converted'] = ['steps' => [ + ['2.2', [], "\$a->insert('users', [['n' => 0], ['n' => 1], ['n' => 2], ['n' => 3], ['n' => 4]]);\n\$a->delete('users', ['key' => 1]);\n\$a->delete('users', ['key' => 4]);"], + ['3.1', [], "\$a->buffer(false);\n\$a->insert('users', [['n' => 'v31']]);\n\$a->delete('users', ['key' => 0]);"], + ['3.2', [], "\$a->insert('users', [['n' => 'v32']]);\n\$a->delete('users', ['key' => 2]);"], +]]; +$fixtures['3.1-3.3-dev-read'] = ['steps' => [['3.1', [], "\$a->buffer(false);\n" . $deletes], ['3.3-dev', [], "\$db->find('users', 0);"]]]; +$fixtures['3.1-3.3-dev-written'] = ['steps' => [['3.1', [], "\$a->buffer(false);\n" . $deletes], ['3.3-dev', [], "\$a->insert('users', [['n' => 'v33']]);"]]]; +// Crafted crash states: what an interrupted process leaves, made by hand on a release's own files +$flushing = function (string $dir) { + foreach (glob("$dir/*.nonedb.buffer") as $buffer) { + rename($buffer, "$buffer.flushing"); + } +}; +foreach (['2.3', '3.0', '3.1'] as $r) { + $fixtures["$r-flushing"] = ['steps' => [[$r, [], $buffered]], 'craft' => ['rename *.nonedb.buffer to *.nonedb.buffer.flushing (a flush that died after its rename)', $flushing], + 'oracles' => [$r, '3.2']]; +} +$fixtures['3.2-flushing-stored'] = ['steps' => [['3.2', [], $buffered]], 'craft' => ['a flush that died after storing its first record: that record appended and indexed under the flush\'s first key, the buffer renamed to .flushing.{that key}', function (string $dir) { + $data = glob("$dir/*-users.nonedb")[0]; + $buffer = "$data.buffer"; + $index = json_decode((string)file_get_contents("$data.jidx"), true); + $first = (int)$index['n']; + $lines = explode("\n", trim((string)file_get_contents($buffer))); + $record = json_decode($lines[0], true) + ['key' => $first]; + $line = json_encode($record, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRESERVE_ZERO_FRACTION); + $offset = filesize($data); + file_put_contents($data, $line . "\n", FILE_APPEND); + $index['o'][(string)$first] = [$offset, strlen($line)]; + $index['n'] = $first + 1; + $index['s'] = $offset + strlen($line) + 1; + file_put_contents("$data.jidx", json_encode($index)); + rename($buffer, "$buffer.flushing.$first"); +}]]; +$fixtures['3.2-lines-after-s'] = ['steps' => [['3.2', [], $deletes]], 'craft' => ['append a complete record line after the index\'s s (a write that died before its index)', function (string $dir) { + $data = glob("$dir/*-users.nonedb")[0]; + file_put_contents($data, "{\"n\":\"ghost\",\"key\":30}\n", FILE_APPEND); +}]]; +foreach (['3.0', '3.1', '3.2'] as $r) { + $fixtures["$r-index-lost"] = ['steps' => [[$r, [], ($r === '3.2' ? '' : "\$a->buffer(false);\n") . $deletes]], 'craft' => ['remove the .jidx of the data file', function (string $dir) { + unlink(glob("$dir/*-users.nonedb.jidx")[0]); + }], 'oracles' => [$r, '3.2']]; +} +// 3.3-dev wrote its checkpoints now and then and read the data file itself without one (MIGRATION.md §4.1 J33) +$fixtures['3.3-dev-index-lost'] = ['steps' => [['3.3-dev', [], $deletes]], 'craft' => ['remove the .jidx of the data file (a checkpoint never written or lost)', function (string $dir) { + unlink(glob("$dir/*-users.nonedb.jidx")[0]); +}]]; +$fixtures['3.1-meta-shard-size'] = ['steps' => [['3.1', ['shardSize' => 5], "\$a->buffer(false);\n" . $sharded]], 'craft' => ['set the meta\'s shardSize to 7', function (string $dir) { + $meta = glob("$dir/*-users.nonedb.meta")[0]; + $m = json_decode((string)file_get_contents($meta), true); + $m['shardSize'] = 7; + file_put_contents($meta, json_encode($m, JSON_PRETTY_PRINT)); +}]]; +$fixtures['3.2-meta-lost'] = ['steps' => [['3.2', ['shardSize' => 5], $sharded]], 'craft' => ['remove the meta of a sharded database', function (string $dir) { + unlink(glob("$dir/*-users.nonedb.meta")[0]); +}]]; +$fixtures['3.2-leftovers'] = ['steps' => [['3.2', [], "\$a->insert('users', [['n' => 1]]);\n\$a->insert('gone', [['n' => 1]]);\n\$db->dropDB('gone');"]]]; + +// ---------------------------------------------------------------- running a release + +/** The release's noneDB.php for these settings: releases before 3.0 get them written into the file */ +function releaseLibrary(string $root, array $release, array $settings, string $dbDir): string +{ + [$source, , $family] = $release; + $code = strncmp($source, 'git:', 4) === 0 + ? (string)shell_exec('git -C ' . escapeshellarg($root) . ' show ' . escapeshellarg(substr($source, 4) . ':noneDB.php')) + : (string)file_get_contents($root . '/' . substr($source, 5)); + if ($code === '') { + throw new RuntimeException("cannot read $source"); + } + if ($family !== 'v3') { + $set = ['dbDir' => var_export($dbDir . '/', true), 'secretKey' => var_export($settings['key'], true)]; + if (isset($settings['shardSize'])) { + $set['shardSize'] = (string)(int)$settings['shardSize']; + } + foreach ($set as $property => $value) { + $code = preg_replace('/(private \$' . $property . '\s*=\s*)[^;]+;/', '${1}' . $value . ';', $code, 1, $count); + if ($count !== 1 && $property !== 'shardSize') { + throw new RuntimeException("$source: no \$$property property"); + } + } + } + return $code; +} + +/** Run $body with the release in a new process; the adapter $a hides the differences of the releases' APIs */ +function runRelease(string $root, array $release, array $settings, string $work, string $dbDir, string $key, string $body): void +{ + [, $php, $family] = $release; + $lib = "$work/lib-" . bin2hex(random_bytes(4)) . '.php'; + file_put_contents($lib, releaseLibrary($root, $release, ['key' => $key] + $settings, $dbDir)); + $config = $family === 'v3' ? var_export(['secretKey' => $key, 'dbDir' => $dbDir . '/', 'autoCreateDB' => true] + + (isset($settings['shardSize']) ? ['shardSize' => $settings['shardSize']] : []), true) : 'null'; + $script = "$work/run-" . bin2hex(random_bytes(4)) . '.php'; + file_put_contents($script, 'family = $family; + $this->config = $config; + $this->db = $this->open([]); + } + public function open(array $over) { + return $this->family === "v3" ? new noneDB($over + $this->config) : new noneDB(); + } + // 1.4 reads a file with the size stat() cached for the process, so a second write in one process loses the + // first: each operation runs as in its own request + private function request() { if ($this->family === "v1") { clearstatcache(); } } + public function insert($db, array $rows) { $this->request(); $r = $this->db->insert($db, count($rows) === 1 ? $rows[0] : $rows); if (isset($r["error"])) { throw new RuntimeException("insert: " . $r["error"]); } } + public function update($db, array $filter, array $set) { $this->request(); $this->db->update($db, [$filter, ["set" => $set]]); } + public function delete($db, array $filter) { $this->request(); $this->db->delete($db, $filter); } + public function compact($db) { $this->db->compact($db); } + public function createFieldIndex($db, $field) { $this->db->createFieldIndex($db, $field); } + public function createSpatialIndex($db, $field) { $this->db->createSpatialIndex($db, $field); } + public function buffer(bool $on) { + if (method_exists($this->db, "enableBuffering")) { + $this->db->enableBuffering($on); + } elseif ($this->family === "v3") { + $this->db = $this->open(["bufferEnabled" => $on]); + } + } + public function kill() { posix_kill(getmypid(), 9); } +} +$a = new Adapter(' . var_export($family, true) . ', ' . $config . '); +$db = $a->db; +' . $body . "\n"); + $process = proc_open([$php, $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $work); + $out = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($process); + unlink($script); + unlink($lib); + $killed = strpos($body, '$a->kill()') !== false; + if ((!$killed && $status !== 0) || trim($out) !== '') { + throw new RuntimeException("release run failed ($status): " . trim($out) . "\n--- body:\n$body"); + } +} + +/** + * What a release returns for the databases of a directory, read in a new process on a copy: records by key (as find() + * returns them, a record of 1.4 gets its position as "key"), count(), and the key a new record gets (inserted, flushed, + * found again) + */ +function oracle(string $root, array $release, array $settings, string $work, string $dir, array $names, string $key): array +{ + $copy = "$work/oracle-" . bin2hex(random_bytes(4)); + copyTree($dir, $copy); + $out = "$work/oracle-" . bin2hex(random_bytes(4)) . '.json'; + $body = '$result = []; +foreach (' . var_export($names, true) . ' as $name) { + clearstatcache(); + $rows = $db->find($name, 0); + $records = []; + foreach (is_array($rows) ? $rows : [] as $i => $row) { + if ($row === null) { + continue; + } + if (!array_key_exists("key", $row)) { + $row["key"] = $i; // 1.4: the raw data array, the position is the key + } + $records[$row["key"]] = $row; + } + ksort($records); + $pairs = []; + foreach ($records as $k => $record) { + $pairs[] = [$k, $record]; // [key, record]: a JSON object keyed 0, 1, … would come back as a list + } + $count = method_exists($db, "count") ? $db->count($name) : count($records); + $a->insert($name, [["__probe" => 1]]); + if (method_exists($db, "flush")) { + $db->flush($name); + } + $next = null; + clearstatcache(); + foreach ($db->find($name, 0) as $i => $row) { + if (is_array($row) && isset($row["__probe"])) { + $next = $row["key"] ?? $i; + } + } + $result[$name] = ["records" => $pairs, "count" => $count, "nextKey" => $next]; +} +file_put_contents(' . var_export($out, true) . ', json_encode($result, JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));'; + runRelease($root, $release, $settings, $work, $copy, $key, $body); + $result = json_decode((string)file_get_contents($out), true); + unlink($out); + removeTree($copy); + return $result; +} + +function copyTree(string $from, string $to): void +{ + @mkdir($to, 0777, true); + foreach (scandir($from) as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + is_dir("$from/$entry") ? copyTree("$from/$entry", "$to/$entry") : copy("$from/$entry", "$to/$entry"); + } +} + +function removeTree(string $dir): void +{ + foreach (is_dir($dir) ? scandir($dir) : [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + is_dir("$dir/$entry") ? removeTree("$dir/$entry") : unlink("$dir/$entry"); + } + } + @rmdir($dir); +} + +/** Database names of a directory: the names in the old file names (the key decides which are whose) */ +function databaseNames(string $dir): array +{ + $names = []; + foreach (scandir($dir) as $entry) { + if (preg_match('/^[0-9a-f]{20}-(.*?)(?:_s\d+)?\.nonedb(?:\.meta)?$/', $entry, $m) && $m[1] !== '') { + $names[$m[1]] = true; + } + } + ksort($names); + return array_keys($names); +} + +// ---------------------------------------------------------------- main + +$only = null; +$target = "$root/tests/Migration/fixtures"; +foreach (array_slice($argv, 1) as $arg) { + if (strncmp($arg, '--only=', 7) === 0) { + $only = substr($arg, 7); + } elseif (strncmp($arg, '--target=', 9) === 0) { + $target = rtrim(substr($arg, 9), '/'); + } else { + fwrite(STDERR, "usage: php tools/make-migration-fixtures.php [--only=REGEX] [--target=DIR]\n"); + exit(64); + } +} +$versions = []; +foreach ([$php74, $php8] as $php) { + $versions[$php] = trim((string)shell_exec(escapeshellarg($php) . ' -r "echo PHP_VERSION;"')); + if ($versions[$php] === '') { + fwrite(STDERR, "no PHP at $php (set NONEDB_PHP74 / NONEDB_PHP8)\n"); + exit(1); + } +} +$work = sys_get_temp_dir() . '/nonedb-fixtures-' . getmypid(); +foreach ($fixtures as $name => $fixture) { + if ($only !== null && !preg_match('/' . $only . '/', $name)) { + continue; + } + removeTree($work); + mkdir("$work/db", 0777, true); + // One key for the whole fixture: the default of the first release (later releases of a chain open its files) + $key = $releases[$fixture['steps'][0][0]][2] === 'v3' ? V3_KEY : OLD_KEY; + $steps = []; + foreach ($fixture['steps'] as [$label, $settings, $body]) { + runRelease($root, $releases[$label], $settings, $work, "$work/db", $key, $body); + $steps[] = ['release' => $label, 'php' => $versions[$releases[$label][1]], 'settings' => $settings, 'code' => $body]; + } + $crafted = null; + if (isset($fixture['craft'])) { + [$crafted, $craft] = $fixture['craft']; + $craft("$work/db"); + } + // Read with the settings of the last writer: before 3.2 a release maps keys with its own shard size + [$last, $lastSettings] = end($fixture['steps']); + $names = databaseNames("$work/db"); + $oracles = []; + foreach ($fixture['oracles'] ?? [$last] as $label) { + $oracles[$label] = oracle($root, $releases[$label], $lastSettings, $work, "$work/db", $names, $key); + } + removeTree("$target/$name"); + copyTree("$work/db", "$target/$name/db"); + file_put_contents("$target/$name/fixture.json", json_encode([ + 'fixture' => $name, + 'sourceKey' => $key, + 'steps' => $steps, + 'crafted' => $crafted, + 'oracles' => $oracles, + ], JSON_PRETTY_PRINT | JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) . "\n"); + echo "$name\n"; +} +removeTree($work); diff --git a/tools/soak.php b/tools/soak.php new file mode 100644 index 0000000..c38d71e --- /dev/null +++ b/tools/soak.php @@ -0,0 +1,1076 @@ + sha256 */ +function snapshot(string $dir): array +{ + $files = []; + $walk = function (string $path, string $rel) use (&$walk, &$files): void { + foreach (scandir($path) ?: [] as $entry) { + if ($entry === '.' || $entry === '..') { + continue; + } + is_dir("$path/$entry") ? $walk("$path/$entry", "$rel$entry/") : $files["$rel$entry"] = hash_file('sha256', "$path/$entry"); + } + }; + $walk(rtrim($dir, '/'), ''); + ksort($files); + return $files; +} + +/** The record "big" holds for key $i (`u` excepted: the big writer changes it) */ +function bigRecord(int $i): array +{ + return ['i' => $i, 'h' => crc32("big$i"), 'g' => 'g' . ($i % 1000), 'u' => 0]; +} + +/** Settings of every process of a run (env: SOAK_WORK, SOAK_SECRET, SOAK_CACHE) */ +function engine(array $more = []): noneDB +{ + return new noneDB($more + ['secretKey' => getenv('SOAK_SECRET'), 'dbDir' => getenv('SOAK_WORK') . '/store/', + 'cacheMemoryLimit' => (int)getenv('SOAK_CACHE'), 'lockTimeout' => 30]); +} + +/** Wait while the supervisor pauses the workers (between two operations: no operation is under way) */ +function paused(string $name): void +{ + $work = getenv('SOAK_WORK'); + while (is_file("$work/pause")) { + touch("$work/paused.$name"); + usleep(50000); + } + @unlink("$work/paused.$name"); +} + +/** A read or a compaction of a worker: a lock timeout under contention is counted, any other failure reported */ +function attempt(string $name, array &$ops, callable $fn): void +{ + try { + $fn(); + } catch (\RuntimeException $e) { + if ($e->getCode() === noneDB::LOCK_FAILED) { + $ops['lockTimeouts'] = ($ops['lockTimeouts'] ?? 0) + 1; + return; + } + workerError($name, get_class($e) . ' ' . $e->getCode() . ': ' . $e->getMessage()); + } +} + +function workerStats(string $name, array $ops, noneDB $db, float &$lastStats): void +{ + if (microtime(true) - $lastStats < 10) { + return; + } + $lastStats = microtime(true); + $metrics = $db->getMetrics()['process']; + // Counters of this process since it started (a restarted worker starts again at 0): the report sums the processes + appendLine(getenv('SOAK_WORK') . "/stats.$name.jsonl", ['t' => time(), 'pid' => getmypid(), 'ops' => $ops, 'peak' => memory_get_peak_usage(), + 'evictions' => $metrics['indexCache']['evictions'], 'misses' => $metrics['indexCache']['misses'], 'hits' => $metrics['indexCache']['hits'], + 'compactions' => $metrics['compactions']['count'], 'checkpoints' => $metrics['checkpoints']['written']]); +} + +/** An error a worker found (at most 100 lines per process: a failure that repeats on every operation stays readable) */ +function workerError(string $name, string $message): void +{ + static $written = 0; + if (++$written <= 100) { + appendLine(getenv('SOAK_WORK') . '/errors.jsonl', ['t' => time(), 'who' => $name, 'error' => $message]); + } +} + +// ---------------------------------------------------------------------------------------------- the journal + +/** + * A writer's journal replayed: [records => id => value (proven), unknown => id => true (an operation not proven + * applied or not), nextId, nextSeq, pending => seq => op]. Resolutions (R lines, written by the verifier while the + * writer waits) settle pending operations. + */ +function replayJournal(string $file): array +{ + $ops = []; + $done = []; + $records = []; + $nextId = 0; + $nextSeq = 0; + foreach (readLines($file) as $line) { + if ($line['t'] === 'S') { + // the state the verifier proved and wrote in place of the lines before it (compactJournal()) + $records = $line['records']; + $nextId = $line['nextId']; + $nextSeq = $line['nextSeq']; + } elseif ($line['t'] === 'B') { + $ops[$line['seq']] = $line; + } elseif ($line['t'] === 'A') { + $done[$line['seq']] = 'applied'; + } elseif ($line['t'] === 'F') { + // a failure before the commit point leaves nothing; OUTCOME_UNKNOWN (3301) may have committed + $done[$line['seq']] = $line['code'] === 3301 ? null : 'not'; + } elseif ($line['t'] === 'R') { + $done[$line['seq']] = $line['applied'] ? 'applied' : 'not'; + } + } + ksort($ops); + $unknown = []; + $pending = []; + foreach ($ops as $seq => $op) { + $nextSeq = $seq + 1; + foreach ($op['ids'] as $id) { + $nextId = max($nextId, $id + 1); + } + $state = $done[$seq] ?? null; + if ($state === null) { + $pending[$seq] = $op; + foreach ($op['ids'] as $id) { + $unknown[$id] = true; + unset($records[$id]); + } + continue; + } + if ($state === 'not') { + continue; + } + foreach ($op['ids'] as $n => $id) { + if ($op['op'] === 'delete') { + unset($records[$id]); + } elseif ($op['op'] === 'update') { + if (array_key_exists($id, $records)) { + $records[$id] = $op['values'][$n]; + } + } else { + $records[$id] = $op['values'][$n]; + } + } + } + return ['records' => $records, 'unknown' => $unknown, 'nextId' => $nextId, 'nextSeq' => $nextSeq, 'pending' => $pending]; +} + +/** + * Replace a journal whose operations are all settled by one S line holding its state (the writer is paused): the + * journals stay small and quick to replay over a day. Written to a temp file and renamed: a kill leaves either one. + */ +function compactJournal(string $file): void +{ + $state = replayJournal($file); + if (!empty($state['pending'])) { + return; + } + $line = json_encode(['t' => 'S', 'records' => (object)$state['records'], 'nextId' => $state['nextId'], 'nextSeq' => $state['nextSeq']], JSON_UNESCAPED_SLASHES); + file_put_contents("$file.new", $line . "\n"); + rename("$file.new", $file); +} + +// ---------------------------------------------------------------------------------------------- the workers + +/** + * A writer of "crud": inserts (1 to 20 records, while it holds fewer than $cap), updates, deletes, reads its own records + * back, counts them, compacts now and then; every write journaled before it starts and after it returned + */ +function writerMain(string $name, int $w, int $cap): void +{ + $work = getenv('SOAK_WORK'); + $journal = "$work/journal.$name.jsonl"; + $state = replayJournal($journal); + $records = $state['records']; + $unknown = $state['unknown']; + $nextId = $state['nextId']; + $seq = $state['nextSeq']; + $db = engine(['shardSize' => 1000]); + $ops = ['insert' => 0, 'update' => 0, 'delete' => 0, 'read' => 0, 'count' => 0, 'compact' => 0, 'failed' => 0]; + $lastStats = 0.0; + while (true) { + if (is_file("$work/pause")) { + paused($name); + // what the verifier settled while this writer waited + $state = replayJournal($journal); + $records = $state['records']; + $unknown = $state['unknown']; + } + $known = array_keys($records); + $roll = mt_rand(1, 100); + $op = null; + if (($roll <= 40 && count($known) < $cap) || count($known) < 20) { + $n = mt_rand(1, 20); + $ids = range($nextId, $nextId + $n - 1); + $nextId += $n; + $values = array_map(function () { return bin2hex(random_bytes(mt_rand(4, 40))); }, $ids); + $op = ['op' => 'insert', 'ids' => $ids, 'values' => $values]; + } elseif ($roll <= 40 || ($roll > 65 && $roll <= 80)) { + $id = $known[array_rand($known)]; + $op = ['op' => 'delete', 'ids' => [$id], 'values' => [null]]; + } elseif ($roll <= 65) { + $id = $known[array_rand($known)]; + $op = ['op' => 'update', 'ids' => [$id], 'values' => [bin2hex(random_bytes(mt_rand(4, 40)))]]; + } elseif ($roll <= 95) { + $id = $known[array_rand($known)]; + attempt($name, $ops, function () use ($db, $w, $id, $records, $name, &$ops) { + $found = $db->find(CRUD, ['w' => $w, 'id' => $id]); + $ops['read']++; + if (count($found) !== 1 || $found[0]['v'] !== $records[$id]) { + workerError($name, "read-your-writes: id $id is " . json_encode($found) . ', expected ' . json_encode($records[$id])); + } + }); + } elseif ($roll <= 99) { + if (empty($unknown)) { + attempt($name, $ops, function () use ($db, $w, $records, $name, &$ops) { + $count = $db->count(CRUD, ['w' => $w]); + $ops['count']++; + if ($count !== count($records)) { + workerError($name, "count of its records: $count, expected " . count($records)); + } + }); + } + } else { + attempt($name, $ops, function () use ($db, &$ops) { + $db->compact(CRUD); + $ops['compact']++; + }); + } + if ($op !== null) { + appendLine($journal, ['t' => 'B', 'seq' => $seq] + $op); + try { + if ($op['op'] === 'insert') { + $rows = []; + foreach ($op['ids'] as $n => $id) { + $rows[] = ['w' => $w, 'id' => $id, 'v' => $op['values'][$n]]; + } + $result = $db->insert(CRUD, count($rows) === 1 ? $rows[0] : $rows); + if (($result['n'] ?? 0) !== count($rows)) { + workerError($name, 'insert returned ' . json_encode($result)); + } + foreach ($op['ids'] as $n => $id) { + $records[$id] = $op['values'][$n]; + } + } elseif ($op['op'] === 'update') { + $result = $db->update(CRUD, [['w' => $w, 'id' => $op['ids'][0]], ['set' => ['v' => $op['values'][0]]]]); + if (($result['n'] ?? 0) !== 1) { + workerError($name, "update of id {$op['ids'][0]} returned " . json_encode($result)); + } + $records[$op['ids'][0]] = $op['values'][0]; + } else { + $result = $db->delete(CRUD, ['w' => $w, 'id' => $op['ids'][0]]); + if (($result['n'] ?? 0) !== 1) { + workerError($name, "delete of id {$op['ids'][0]} returned " . json_encode($result)); + } + unset($records[$op['ids'][0]]); + } + appendLine($journal, ['t' => 'A', 'seq' => $seq]); + $ops[$op['op']]++; + } catch (\RuntimeException $e) { + appendLine($journal, ['t' => 'F', 'seq' => $seq, 'code' => $e->getCode(), 'message' => $e->getMessage()]); + $ops['failed']++; + if ($e->getCode() === noneDB::OUTCOME_UNKNOWN) { + foreach ($op['ids'] as $id) { + $unknown[$id] = true; + unset($records[$id]); + } + } + if ($e->getCode() !== noneDB::LOCK_FAILED) { + workerError($name, get_class($e) . ' ' . $e->getCode() . ': ' . $e->getMessage()); + } + } + $seq++; + } + workerStats($name, $ops, $db, $lastStats); + } +} + +/** A reader of "big": random keys, now and then a group by the field index and a count; every record checked */ +function readerMain(string $name, int $big): void +{ + $db = engine(); + $ops = ['key' => 0, 'group' => 0, 'count' => 0]; + $lastStats = 0.0; + while (true) { + paused($name); + $roll = mt_rand(1, 1000); + attempt($name, $ops, function () use ($db, $big, $roll, $name, &$ops) { + if ($roll <= 990) { + $i = mt_rand(0, $big - 1); + $found = $db->find(BIG, ['key' => $i]); + $ops['key']++; + if (count($found) !== 1 || $found[0]['key'] !== $i || array_diff_key($found[0], ['u' => 0, 'key' => 0]) !== array_diff_key(bigRecord($i), ['u' => 0])) { + workerError($name, "big key $i: " . json_encode($found)); + } + } elseif ($roll <= 999) { + $g = mt_rand(0, 999); + $found = $db->find(BIG, ['g' => "g$g"]); + $ops['group']++; + if (count($found) !== intdiv($big - 1 - $g, 1000) + 1) { + workerError($name, "big group g$g: " . count($found) . ' records'); + } + foreach ($found as $record) { + if ($record['i'] % 1000 !== $g || $record['h'] !== crc32('big' . $record['i'])) { + workerError($name, "big group g$g holds " . json_encode($record)); + break; + } + } + } else { + $ops['count']++; + if (($count = $db->count(BIG)) !== $big) { + workerError($name, "count of big: $count"); + } + } + }); + workerStats($name, $ops, $db, $lastStats); + } +} + +/** The writer of "big": updates of the field `u` of random keys, a compaction by chance (1 in 50 000 per update) */ +function bigWriterMain(string $name, int $big): void +{ + $db = engine(); + $ops = ['update' => 0, 'compact' => 0]; + $lastStats = 0.0; + while (true) { + paused($name); + attempt($name, $ops, function () use ($db, $big, $name, &$ops) { + $i = mt_rand(0, $big - 1); + $result = $db->update(BIG, [['key' => $i], ['set' => ['u' => mt_rand(1, PHP_INT_MAX)]]]); + $ops['update']++; + if (($result['n'] ?? 0) !== 1) { + workerError($name, "update of big key $i returned " . json_encode($result)); + } + // by chance, not by this process's count: a kill starts that again, and at 128M a process seldom reaches it + if (mt_rand(1, 50000) === 1) { + $db->compact(BIG); + $ops['compact']++; + } + }); + workerStats($name, $ops, $db, $lastStats); + } +} + + +// ---------------------------------------------------------------------------------------------- the verifier + +/** + * Check the store against the journals; settle every pending operation (R lines appended to its journal). + * @return array ['ok' => bool, 'errors' => string[], 'settled' => int, 'records' => int, 'checked' => int] + */ +function verifyMain(int $big, bool $whole, array $writers): array +{ + $work = getenv('SOAK_WORK'); + $errors = []; + noneDB::clearStaticCache(); + $db = engine(); + foreach ([CRUD, BIG] as $name) { + $report = $db->checkIntegrity($name); + if (!$report['ok']) { + $errors[] = "checkIntegrity($name): " . json_encode(array_intersect_key($report, array_flip(['badLines', 'tornTail', 'indexMismatches', 'pendingRecovery']))); + } + } + // crud: the records of every writer, by id + $actual = []; + $all = $db->find(CRUD, 0); + foreach ($all as $record) { + $w = $record['w']; + if (isset($actual[$w][$record['id']])) { + $errors[] = "crud: writer $w id {$record['id']} twice"; + } + $actual[$w][$record['id']] = $record['v']; + } + $settled = 0; + foreach ($writers as $w => $journal) { + $state = replayJournal($journal); + $expected = $state['records']; + $have = $actual[$w] ?? []; + // pending operations: applied completely or not at all, decided by the store (nothing touched their ids since) + foreach ($state['pending'] as $seq => $op) { + $applied = []; + foreach ($op['ids'] as $n => $id) { + if ($op['op'] === 'delete') { + $applied[] = !array_key_exists($id, $have); + } else { + $applied[] = array_key_exists($id, $have) && $have[$id] === $op['values'][$n]; + } + } + $yes = count(array_filter($applied)); + if ($yes !== 0 && $yes !== count($applied)) { + $errors[] = "crud: writer $w operation $seq ({$op['op']}) applied to $yes of " . count($applied) . ' records'; + } + appendLine($journal, ['t' => 'R', 'seq' => $seq, 'applied' => $yes > 0]); + $settled++; + } + $expected = replayJournal($journal)['records']; + foreach ($expected as $id => $value) { + if (!array_key_exists($id, $have)) { + $errors[] = "crud: writer $w id $id lost (expected " . json_encode($value) . ')'; + } elseif ($have[$id] !== $value) { + $errors[] = "crud: writer $w id $id is " . json_encode($have[$id]) . ', expected ' . json_encode($value); + } + } + foreach (array_diff_key($have, $expected) as $id => $value) { + $errors[] = "crud: writer $w id $id exists, not written by it (" . json_encode($value) . ')'; + } + unset($actual[$w]); + } + foreach (array_keys($actual) as $w) { + $errors[] = "crud: records of an unknown writer $w"; + } + // big: every record, or a sample + $checked = 0; + if (($count = $db->count(BIG)) !== $big) { + $errors[] = "big: $count records, expected $big"; + } + $chunks = $whole ? intdiv($big + 4999, 5000) : 4; + for ($c = 0; $c < $chunks; $c++) { + // the whole database 5000 keys at a time, or 4 × 5000 random keys + $chunk = $whole ? range($c * 5000, min($big, ($c + 1) * 5000) - 1) : array_map(function () use ($big) { return mt_rand(0, $big - 1); }, range(1, 5000)); + $found = []; + foreach ($db->find(BIG, ['key' => $chunk]) as $record) { + $found[$record['key']] = $record; + } + foreach ($chunk as $i) { + $checked++; + if (!isset($found[$i]) || array_diff_key($found[$i], ['u' => 0, 'key' => 0]) !== array_diff_key(bigRecord($i), ['u' => 0])) { + $errors[] = "big: key $i is " . json_encode($found[$i] ?? null); + break 2; + } + } + unset($found); + } + // a backup of crud restored into a store of its own holds the same records + $backup = "$work/verify-backup"; + $scratch = "$work/verify-store"; + removeTree($backup); + removeTree($scratch); + try { + $db->backup(CRUD, $backup); + noneDB::clearStaticCache(); + $other = new noneDB(['secretKey' => getenv('SOAK_SECRET') . '-scratch', 'dbDir' => "$scratch/", 'cacheMemoryLimit' => (int)getenv('SOAK_CACHE')]); + $other->restore($backup, CRUD); + $restored = $other->find(CRUD, 0); + $byKey = function (array $records) { + $out = []; + foreach ($records as $record) { + $out[$record['key']] = $record; + } + ksort($out); + return $out; + }; + if ($byKey($restored) !== $byKey($all)) { + $errors[] = 'backup and restore: ' . count($restored) . ' records, the store holds ' . count($all); + } + if (!$other->checkIntegrity(CRUD)['ok']) { + $errors[] = 'backup and restore: the restored database fails checkIntegrity()'; + } + } catch (\Throwable $e) { + $errors[] = 'backup and restore: ' . get_class($e) . ': ' . $e->getMessage(); + } + removeTree($backup); + removeTree($scratch); + foreach ($writers as $journal) { + compactJournal($journal); + } + return ['ok' => empty($errors), 'errors' => array_slice($errors, 0, 50), 'errorCount' => count($errors), 'settled' => $settled, + 'records' => count($all), 'checked' => $checked, 'whole' => $whole, 'peak' => memory_get_peak_usage()]; +} + +// ---------------------------------------------------------------------------------------------- the migration cycle + +/** + * A fixture written by an old release, migrated by noneDB-migrate.php killed at a random moment (always when $kill, + * else two times in three; a run of these fixtures takes about 100 ms), run again until it finishes, compared with the + * old release's own view, then restored byte for byte + */ +function migrationCycle(string $tool, string $fixture, bool $kill): array +{ + $work = getenv('SOAK_WORK'); + $base = "$work/migration"; + removeTree($base); + $src = "$base/db/"; + $backup = "$base/backup/"; + copyTree(root() . "/tests/Migration/fixtures/$fixture/db", $src); + $meta = json_decode((string)file_get_contents(root() . "/tests/Migration/fixtures/$fixture/fixture.json"), true); + $before = snapshot($src); + $target = 'soak-migration-target-' . bin2hex(random_bytes(8)); + $env = getenv(); + unset($env['NONEDB_DEV_MODE']); + $env['NONEDB_SECRET'] = $target; + $env['NONEDB_SOURCE_SECRET'] = $meta['sourceKey']; + $run = function (array $args, ?int $killAfterMs) use ($tool, $env, $base) { + $process = proc_open(array_merge([PHP_BINARY, $tool], $args, ['--json']), [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $base, $env); + if ($killAfterMs !== null) { + usleep($killAfterMs * 1000); + proc_terminate($process, 9); + } + $out = stream_get_contents($pipes[1]); + stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + $status = proc_close($process); + return [$status, json_decode((string)$out, true)]; + }; + $args = ["--dir=$src", "--backup-dir=$backup", '--confirm-stopped']; + $killed = $kill || mt_rand(0, 2) > 0 ? mt_rand(0, 150) : null; + $attempts = 0; + [$status, $out] = $run($args, $killed); + $interrupted = $status !== 0; + $result = function (?string $error) use ($fixture, $killed, &$interrupted, &$attempts) { + return ['ok' => $error === null, 'fixture' => $fixture, 'killedAfterMs' => $killed, 'interrupted' => $interrupted, 'reruns' => $attempts] + + ($error === null ? [] : ['error' => $error]); + }; + while ($status !== 0 && $attempts < 5) { + $attempts++; + [$status, $out] = $run($args, null); + } + // the run of the backup (a kill after it had finished leaves the next run nothing to do: it reports no run), with + // its final verification recorded and its report written (MIGRATION.md §7.8) + $runs = glob($backup . '*/journal.done.json') ?: []; + $runId = count($runs) === 1 ? basename(dirname($runs[0])) : null; + $done = $runId === null ? null : json_decode(trim((string)file_get_contents($runs[0])), true); + if ($status !== 0 || $runId === null || !in_array($out['migrated']['run'] ?? null, [null, $runId], true) + || ($done['state'] ?? null) !== 'FINAL_VERIFIED' || !is_file($backup . "$runId/report.json")) { + return $result("the run did not finish (status $status, " + . count($runs) . ' closed runs, state ' . ($done['state'] ?? 'none') . '): ' . json_encode($out)); + } + $releases = array_keys($meta['oracles']); + $release = end($releases); + noneDB::clearStaticCache(); + $db = new noneDB(['secretKey' => $target, 'dbDir' => $src, 'autoCreateDB' => false]); + foreach ($meta['oracles'][$release] as $name => $oracle) { + $expected = []; + foreach ($oracle['records'] as [$key, $record]) { + $expected[$key] = $record; + } + $got = []; + foreach ($db->find($name, 0) ?: [] as $record) { + $got[$record['key']] = $record; + } + ksort($expected); + ksort($got); + if ($got !== $expected) { + return $result("$name: " . count($got) . ' records, the old release returned ' . count($expected)); + } + } + unset($db); + noneDB::clearStaticCache(); + [$status, $restored] = $run(["--dir=$src", "--backup-dir=$backup", '--restore=' . $runId], null); + $after = snapshot($src); + if ($status !== 0 || $after !== $before) { + $differ = array_keys(array_diff_assoc($before, $after) + array_diff_assoc($after, $before)); + return $result("the restore exited with $status and left " + . count($differ) . ' files different: ' . json_encode(array_slice($differ, 0, 10)) . ' ' . json_encode($restored)); + } + removeTree($base); + return $result(null); +} + +// ---------------------------------------------------------------------------------------------- the supervisor + +function childCommand(array $settings, string $role, array $roleArgs): array +{ + return [PHP_BINARY, '-d', 'memory_limit=' . $settings['memory'], __FILE__, 'child', $role, json_encode($roleArgs)]; +} + +/** Start a child process (no shell: a SIGKILL reaches PHP itself); its output goes to out.$name.log */ +function spawn(array $settings, string $name, string $role, array $roleArgs) +{ + $env = getenv(); + $env['SOAK_WORK'] = $settings['work']; + $env['SOAK_SECRET'] = $settings['secret']; + $env['SOAK_CACHE'] = (string)$settings['cache']; + $env['SOAK_LIB'] = $settings['lib']; + @unlink($settings['work'] . "/paused.$name"); // a killed worker's mark is not the new one's + $log = fopen($settings['work'] . "/out.$name.log", 'ab'); + $process = proc_open(childCommand($settings, $role, $roleArgs + ['name' => $name]), [0 => ['file', '/dev/null', 'r'], 1 => $log, 2 => $log], $pipes, null, $env); + fclose($log); + return $process; +} + +/** Run a child to its end; its result is the file it wrote (none: it failed) */ +function runChildSync(array $settings, string $role, array $roleArgs): array +{ + $file = $settings['work'] . "/result.$role.json"; + @unlink($file); + $status = proc_close(spawn($settings, $role, $role, $roleArgs)); + $result = json_decode((string)@file_get_contents($file), true); + return is_array($result) ? $result : ['ok' => false, 'errors' => ["$role exited with $status and no result (out.$role.log)"], 'errorCount' => 1]; +} + +/** Pause the workers: each finishes its operation and waits (paused.$name) */ +function pauseWorkers(array $settings, array $workers): bool +{ + foreach (array_keys($workers) as $name) { + @unlink($settings['work'] . "/paused.$name"); + } + touch($settings['work'] . '/pause'); + $deadline = microtime(true) + 180; + while (microtime(true) < $deadline) { + $waiting = 0; + foreach (array_keys($workers) as $name) { + if (!is_file($settings['work'] . "/paused.$name")) { + $waiting++; + } + } + if ($waiting === 0) { + return true; + } + usleep(100000); + } + return false; +} + + +function supervise(array $settings, array $options): int +{ + $work = $settings['work']; + $out = $settings['out']; + $hours = (float)($options['hours'] ?? 24); + $writers = (int)($options['writers'] ?? 4); + $readers = (int)($options['readers'] ?? 2); + $big = (int)($options['big'] ?? 2000000); + $cap = (int)($options['cap'] ?? 2500); + $verifyMinutes = (float)($options['verify-minutes'] ?? 10); + [$killMin, $killMax] = array_map('intval', explode(',', (string)($options['kill-seconds'] ?? '10,90'))); + $tool = (string)($options['tool'] ?? root() . '/noneDB-migrate.php'); + $started = time(); + $deadline = $started + (int)($hours * 3600); + $events = "$out/events.jsonl"; + appendLine($events, ['t' => $started, 'event' => 'start', 'hours' => $hours, 'writers' => $writers, 'cap' => $cap, 'readers' => $readers, 'big' => $big, + 'memory' => $settings['memory'], 'cache' => $settings['cache'], 'lib' => $settings['lib'], 'sha256' => hash_file('sha256', $settings['lib'])]); + + $setup = runChildSync($settings, 'setup', ['big' => $big]); + if (empty($setup['ok'])) { + fail('setup failed: ' . json_encode($setup)); + } + appendLine($events, ['t' => time(), 'event' => 'setup', 'seconds' => $setup['seconds']]); + + $roles = []; + for ($w = 1; $w <= $writers; $w++) { + $roles["writer$w"] = ['writer', ['w' => $w, 'cap' => $cap]]; + } + for ($r = 1; $r <= $readers; $r++) { + $roles["reader$r"] = ['reader', ['big' => $big]]; + } + $roles['bigwriter'] = ['bigwriter', ['big' => $big]]; + $journals = []; + for ($w = 1; $w <= $writers; $w++) { + $journals[$w] = "$work/journal.writer$w.jsonl"; + } + $processes = []; + foreach ($roles as $name => [$role, $roleArgs]) { + $processes[$name] = spawn($settings, $name, $role, $roleArgs); + } + + $kills = 0; + $verifications = 0; + $failedChecks = 0; + $migrations = 0; + $nextKill = time() + mt_rand($killMin, $killMax); + $nextVerify = time() + (int)($verifyMinutes * 60); + $final = false; + while (true) { + $now = time(); + foreach ($processes as $name => $process) { + if (!proc_get_status($process)['running']) { + // a worker never stops by itself: one that did crashed + proc_close($process); + appendLine($events, ['t' => $now, 'event' => 'exited', 'who' => $name]); + workerErrorFile($work, $name, 'exited by itself (see out.' . $name . '.log)'); + $processes[$name] = spawn($settings, $name, $roles[$name][0], $roles[$name][1]); + } + } + if ($now >= $deadline) { + $final = true; + } + if ($final || $now >= $nextVerify) { + if (!pauseWorkers($settings, $processes)) { + appendLine($events, ['t' => time(), 'event' => 'pause-timeout']); + workerErrorFile($work, 'supervisor', 'the workers did not pause within 180 s'); + } + $verifications++; + $result = runChildSync($settings, 'verify', ['big' => $big, 'whole' => $final || $verifications % 6 === 0, 'writers' => $journals]); + if (empty($result['ok'])) { + $failedChecks++; + } + appendLine($events, ['t' => time(), 'event' => 'verify', 'n' => $verifications] + $result); + if ($verifications % 3 === 0 || $final) { + $migrations++; + $fixture = FIXTURES[mt_rand(0, count(FIXTURES) - 1)]; + $migration = runChildSync($settings, 'migration', ['tool' => $tool, 'fixture' => $fixture]); + if (empty($migration['ok'])) { + $failedChecks++; + } + appendLine($events, ['t' => time(), 'event' => 'migration', 'n' => $migrations] + $migration); + } + appendLine($events, ['t' => time(), 'event' => 'disk', 'bytes' => (int)trim((string)shell_exec('du -sk ' . escapeshellarg("$work/store") . ' | cut -f1')) * 1024]); + if ($final) { + break; + } + @unlink("$work/pause"); + $nextVerify = time() + (int)($verifyMinutes * 60); + } + if (time() >= $nextKill) { + $names = array_keys($processes); + $name = $names[mt_rand(0, count($names) - 1)]; + proc_terminate($processes[$name], 9); + proc_close($processes[$name]); + $kills++; + appendLine($events, ['t' => time(), 'event' => 'kill', 'who' => $name]); + // the journal of a killed writer may end in an operation it never finished: the next verification settles it + $processes[$name] = spawn($settings, $name, $roles[$name][0], $roles[$name][1]); + $nextKill = time() + mt_rand($killMin, $killMax); + } + usleep(200000); + } + foreach ($processes as $process) { + proc_terminate($process, 9); + proc_close($process); + } + @unlink("$work/pause"); + return report($settings, $started); +} + +function workerErrorFile(string $work, string $who, string $message): void +{ + appendLine("$work/errors.jsonl", ['t' => time(), 'who' => $who, 'error' => $message]); +} + +function report(array $settings, int $started): int +{ + $work = $settings['work']; + $out = $settings['out']; + $events = readLines("$out/events.jsonl"); + $errors = readLines("$work/errors.jsonl"); + $start = $events[0]; + $verifies = array_values(array_filter($events, function ($e) { return $e['event'] === 'verify'; })); + $migrations = array_values(array_filter($events, function ($e) { return $e['event'] === 'migration'; })); + $kills = array_count_values(array_column(array_filter($events, function ($e) { return $e['event'] === 'kill'; }), 'who')); + $disk = array_values(array_filter($events, function ($e) { return $e['event'] === 'disk'; })); + $stats = []; + foreach (glob("$work/stats.*.jsonl") as $file) { + $name = substr(basename($file, '.jsonl'), 6); + $lines = readLines($file); + $last = []; + foreach ($lines as $line) { + $last[$line['pid']] = $line['ops']; // the last report of each process + } + $ops = []; + foreach ($last as $counters) { + foreach ($counters as $op => $count) { + $ops[$op] = ($ops[$op] ?? 0) + $count; + } + } + $stats[$name] = ['processes' => count($last), 'peak' => max(array_column($lines, 'peak') ?: [0]), 'ops' => $ops]; + } + ksort($stats); + $failedVerifies = count(array_filter($verifies, function ($v) { return empty($v['ok']); })); + $failedMigrations = count(array_filter($migrations, function ($m) { return empty($m['ok']); })); + $ok = $failedVerifies === 0 && $failedMigrations === 0 && empty($errors) && !empty($verifies); + $summary = ['ok' => $ok, 'started' => date('c', $started), 'ended' => date('c'), 'hours' => round((time() - $started) / 3600, 2), + 'library' => $start['lib'], 'sha256' => $start['sha256'], 'php' => PHP_VERSION, 'os' => php_uname('s') . ' ' . php_uname('r'), + 'writers' => $start['writers'], 'readers' => $start['readers'], 'big' => $start['big'], 'memory' => $start['memory'], 'cache' => $start['cache'], + 'kills' => $kills, 'verifications' => count($verifies), 'failedVerifications' => $failedVerifies, 'settled' => array_sum(array_column($verifies, 'settled')), + 'migrations' => count($migrations), 'failedMigrations' => $failedMigrations, + 'migrationsInterrupted' => count(array_filter($migrations, function ($m) { return !empty($m['interrupted']); })), + 'migrationFixtures' => array_count_values(array_column($migrations, 'fixture')), + 'errors' => count($errors), 'firstErrors' => array_slice($errors, 0, 20), 'workers' => $stats, + 'diskFirst' => $disk[0]['bytes'] ?? null, 'diskLast' => end($disk)['bytes'] ?? null, + 'crudRecordsLast' => end($verifies)['records'] ?? null]; + file_put_contents("$out/soak.json", json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); + $md = "# noneDB soak test\n\n" + . '- **Result:** ' . ($ok ? 'OK' : '**FAILED**') . "\n" + . "- **Run:** {$summary['started']} → {$summary['ended']} ({$summary['hours']} h), PHP {$summary['php']}, {$summary['os']}\n" + . "- **Library:** `{$summary['library']}` (sha256 `" . substr((string)$summary['sha256'], 0, 12) . "…`)\n" + . "- **Processes:** {$summary['writers']} writers on \"crud\", {$summary['readers']} readers and 1 writer on \"big\" ({$summary['big']} records); " + . "`memory_limit` {$summary['memory']}, `cacheMemoryLimit` {$summary['cache']} bytes\n" + . '- **Kills (SIGKILL) and restarts:** ' . array_sum($kills) . ' (' . implode(', ', array_map(function ($k, $v) { return "$k $v"; }, array_keys($kills), $kills)) . ")\n" + . "- **Verifications:** {$summary['verifications']} by a separate process, {$summary['failedVerifications']} failed; " + . "{$summary['settled']} interrupted operations settled (applied or not, never in part)\n" + . "- **Migrations** killed at a random moment, run again, compared with the old release, restored: {$summary['migrations']} " + . "({$summary['migrationsInterrupted']} interrupted by the kill), {$summary['failedMigrations']} failed\n" + . "- **Errors reported by the workers:** {$summary['errors']}\n" + . '- **Disk of the store:** first verification ' . round(($summary['diskFirst'] ?? 0) / 1048576, 1) . ' MiB, last ' . round(($summary['diskLast'] ?? 0) / 1048576, 1) . " MiB\n\n" + . "| Process | Started | Peak memory | Operations (all its processes, as last reported every 10 s) |\n|---|---|---|---|\n"; + foreach ($stats as $name => $s) { + $md .= "| $name | {$s['processes']} | " . round($s['peak'] / 1048576, 1) . ' MiB | ' . json_encode($s['ops']) . " |\n"; + } + if (!empty($errors)) { + $md .= "\n## First errors\n\n"; + foreach (array_slice($errors, 0, 20) as $e) { + $md .= "- {$e['who']}: {$e['error']}\n"; + } + } + file_put_contents("$out/soak.md", $md); + echo $md; + return $ok ? 0 : 1; +} + +// ---------------------------------------------------------------------------------------------- the self-check + +/** + * The verifier finds what it must. A writer works a few seconds, then, behind its back: + * - lost: a record of it deleted; changed: one changed; phantom: one it never wrote; + * - partial: an insert of two records the journal began, one of them in the store (applied in part); + * - interrupted: an insert the journal began and never ended, in the store completely: settled as applied, no error. + * Each case must give its own error (or none); then every fixture goes through the migration cycle. + */ +function selfCheck(array $settings): int +{ + $work = $settings['work']; + $journal = "$work/journal.writer1.jsonl"; + $setup = runChildSync($settings, 'setup', ['big' => 5000]); + if (empty($setup['ok'])) { + fail('setup failed'); + } + $writer = spawn($settings, 'writer1', 'writer', ['w' => 1, 'cap' => 2500]); + sleep(5); + pauseWorkers($settings, ['writer1' => true]); + proc_terminate($writer, 9); + proc_close($writer); + @unlink("$work/pause"); + putenv("SOAK_WORK=$work"); + putenv('SOAK_SECRET=' . $settings['secret']); + putenv('SOAK_CACHE=' . $settings['cache']); + $verify = function () use ($settings, $journal) { + noneDB::clearStaticCache(); + return runChildSync($settings, 'verify', ['big' => 5000, 'whole' => false, 'writers' => [1 => $journal]]); + }; + $db = function () { + noneDB::clearStaticCache(); + return engine(); + }; + $begin = function (array $op) use ($journal) { + $seq = replayJournal($journal)['nextSeq']; + appendLine($journal, ['t' => 'B', 'seq' => $seq] + $op); + return $seq; + }; + $results = []; + $clean = $verify(); + $results['clean'] = ['pass' => !empty($clean['ok']) && $clean['records'] > 0, 'records' => $clean['records'], 'errors' => $clean['errors']]; + $victim = $db()->find(CRUD, ['w' => 1])[0]; + $cases = [ + 'lost' => [function () use ($db, $victim) { $db()->delete(CRUD, ['key' => $victim['key']]); }, + ["crud: writer 1 id {$victim['id']} lost (expected " . json_encode($victim['v']) . ')'], + function () use ($db, $victim) { $db()->insert(CRUD, ['w' => 1, 'id' => $victim['id'], 'v' => $victim['v']]); }], + 'changed' => [function () use ($db, $victim) { $db()->update(CRUD, [['w' => 1, 'id' => $victim['id']], ['set' => ['v' => 'changed']]]); }, + ["crud: writer 1 id {$victim['id']} is \"changed\", expected " . json_encode($victim['v'])], + function () use ($db, $victim) { $db()->update(CRUD, [['w' => 1, 'id' => $victim['id']], ['set' => ['v' => $victim['v']]]]); }], + 'phantom' => [function () use ($db) { $db()->insert(CRUD, ['w' => 1, 'id' => 900000, 'v' => 'phantom']); }, + ['crud: writer 1 id 900000 exists, not written by it ("phantom")'], + function () use ($db) { $db()->delete(CRUD, ['w' => 1, 'id' => 900000]); }], + 'partial' => [function () use ($db, $begin, &$partialSeq) { + $partialSeq = $begin(['op' => 'insert', 'ids' => [900001, 900002], 'values' => ['p1', 'p2']]); + $db()->insert(CRUD, ['w' => 1, 'id' => 900001, 'v' => 'p1']); + }, + // settled as applied (it was, in part): the record it did not write is then missing too + ['crud: writer 1 operation %d (insert) applied to 1 of 2 records', 'crud: writer 1 id 900002 lost (expected "p2")'], + // the verifier settled it as applied: the store gets the record it lacks + function () use ($db) { $db()->insert(CRUD, ['w' => 1, 'id' => 900002, 'v' => 'p2']); }], + 'interrupted' => [function () use ($db, $begin) { + $begin(['op' => 'insert', 'ids' => [900003, 900004], 'values' => ['i3', 'i4']]); + $db()->insert(CRUD, [['w' => 1, 'id' => 900003, 'v' => 'i3'], ['w' => 1, 'id' => 900004, 'v' => 'i4']]); + }, null, null], + ]; + $partialSeq = null; + foreach ($cases as $case => [$break, $expected, $repair]) { + $break(); + $found = $verify(); + if ($expected === null) { + $model = replayJournal($journal)['records']; + $pass = !empty($found['ok']) && $found['settled'] === 1 && ($model[900003] ?? null) === 'i3' && ($model[900004] ?? null) === 'i4'; + } else { + $expected = array_map(function ($error) use ($partialSeq) { return sprintf($error, (int)$partialSeq); }, $expected); + $pass = empty($found['ok']) && $found['errors'] === $expected && $found['errorCount'] === count($expected); + } + $results[$case] = ['pass' => $pass, 'expected' => $expected, 'errors' => $found['errors'], 'settled' => $found['settled']]; + if ($repair !== null) { + $repair(); + $after = $verify(); + $results[$case]['repaired'] = !empty($after['ok']); + $results[$case]['pass'] = $results[$case]['pass'] && $results[$case]['repaired']; + } + } + // a writer started again after a kill picks its records from the replay: an unfinished operation's id is not among + // them, whether the operation reached the store or not, until the verifier has settled it + [$applied, $notApplied] = array_slice(array_keys(replayJournal($journal)['records']), 0, 2); + $old = replayJournal($journal)['records'][$notApplied]; + $begin(['op' => 'delete', 'ids' => [$applied], 'values' => [null]]); + $db()->delete(CRUD, ['w' => 1, 'id' => $applied]); + $begin(['op' => 'update', 'ids' => [$notApplied], 'values' => ['never written']]); + $pending = replayJournal($journal); + $found = $verify(); + $model = replayJournal($journal)['records']; + $results['unfinished'] = ['pass' => !array_key_exists($applied, $pending['records']) && !array_key_exists($notApplied, $pending['records']) + && isset($pending['unknown'][$applied], $pending['unknown'][$notApplied]) && !empty($found['ok']) && $found['settled'] === 2 + && !array_key_exists($applied, $model) && ($model[$notApplied] ?? null) === $old, 'errors' => $found['errors'], 'settled' => $found['settled']]; + foreach (FIXTURES as $fixture) { + $migration = runChildSync($settings, 'migration', ['tool' => root() . '/noneDB-migrate.php', 'fixture' => $fixture, 'kill' => true]); + $results["migration $fixture"] = ['pass' => !empty($migration['ok'])] + $migration; + } + file_put_contents($settings['out'] . '/self-check.json', json_encode($results, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"); + $failed = 0; + foreach ($results as $case => $result) { + echo ($result['pass'] ? 'ok ' : 'FAILED'), " $case", isset($result['killedAfterMs']) ? " (killed after {$result['killedAfterMs']} ms: " . (!empty($result['interrupted']) ? 'interrupted' : 'already done') . ", {$result['reruns']} reruns)" : '', "\n"; + $failed += $result['pass'] ? 0 : 1; + } + return $failed === 0 ? 0 : 1; +} + +// ---------------------------------------------------------------------------------------------- main + +if ($mode === 'child') { + [$role, $json] = [$args[0], $args[1]]; + $roleArgs = json_decode($json, true); + require getenv('SOAK_LIB'); + mt_srand(random_int(0, PHP_INT_MAX)); + $work = getenv('SOAK_WORK'); + switch ($role) { + case 'setup': + $t = microtime(true); + $db = engine(['shardSize' => 100000]); + $big = $roleArgs['big']; + if ($db->count(BIG) === 0) { + for ($from = 0; $from < $big; $from += 20000) { + $db->insert(BIG, array_map('bigRecord', range($from, min($big, $from + 20000) - 1))); + } + $db->createFieldIndex(BIG, 'g'); + } + noneDB::clearStaticCache(); + $crud = engine(['shardSize' => 1000]); + $crud->createFieldIndex(CRUD, 'w'); + file_put_contents("$work/result.setup.json", json_encode(['ok' => true, 'seconds' => round(microtime(true) - $t, 1)])); + break; + case 'writer': + writerMain($roleArgs['name'], $roleArgs['w'], $roleArgs['cap']); + break; + case 'reader': + readerMain($roleArgs['name'], $roleArgs['big']); + break; + case 'bigwriter': + bigWriterMain($roleArgs['name'], $roleArgs['big']); + break; + case 'verify': + $writers = []; + foreach ($roleArgs['writers'] as $w => $journal) { + $writers[(int)$w] = $journal; + } + file_put_contents("$work/result.verify.json", json_encode(verifyMain($roleArgs['big'], $roleArgs['whole'], $writers))); + break; + case 'migration': + file_put_contents("$work/result.migration.json", json_encode(migrationCycle($roleArgs['tool'], $roleArgs['fixture'], !empty($roleArgs['kill'])))); + break; + } + exit(0); +} + +if ($mode !== 'run' && $mode !== 'self-check') { + fwrite(STDERR, "usage: php tools/soak.php run --out=DIR [--hours=24] ... | self-check --out=DIR\n"); + exit(2); +} +$out = $options['out'] ?? fail('--out=DIR is required'); +@mkdir($out, 0777, true); +$out = realpath($out); +$work = $out . '/work'; +removeTree($work); +mkdir($work, 0700, true); +$settings = ['out' => $out, 'work' => $work, 'secret' => bin2hex(random_bytes(32)), 'memory' => (string)($options['memory'] ?? '128M'), + 'cache' => (int)($options['cache'] ?? 16777216), 'lib' => realpath((string)($options['lib'] ?? root() . '/noneDB.php'))]; +// the run's own throwaway key, kept for opening the store of a failed run (never a key of a real store) +file_put_contents("$work/secret", $settings['secret']); +chmod("$work/secret", 0600); +if (isset($options['seed'])) { + mt_srand((int)$options['seed']); +} +if ($mode === 'self-check') { + require $settings['lib']; + exit(selfCheck($settings)); +} +@unlink("$out/events.jsonl"); +exit(supervise($settings, $options)); From 0a3ce093d48fc3a31dac27ee190f9653b388d5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 19:42:56 +0300 Subject: [PATCH 2/6] Version 4.0.0, migration fixtures whatever php.ini says The fixture tool ran old releases with the error settings of the php.ini in use, and a notice of 1.4 on a key that isn't there stopped CI while it went to a log file here. Notices are off in those runs now, and what is left always reaches the tool. noneDB::VERSION is 4.0.0. PERFORMANCE.md says what was measured without names of earlier builds, and no longer names the disk or the system. --- docs/PERFORMANCE.md | 10 +++++----- noneDB.php | 2 +- src/noneDB/Facade.php | 2 +- tools/make-migration-fixtures.php | 5 +++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 974f3ca..758c69d 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -13,7 +13,7 @@ faster, with no verified regression. ## 1. Summary -- **The performance gate** (§2): two independent runs of 66 workloads on the release candidate. +- **The performance gate** (§2): two independent runs of 66 workloads. - 46 workloads are verified improvements and 13 show no verified change. - 5 are findings, significant in both runs: one insert into a small shard is 3.2–7.0% slower, the others at most 3.1%. - 2 are fixed costs of the fail-closed store, within their budgets. @@ -58,7 +58,7 @@ faster, with no verified regression. A time regression within its budget in both runs is reported as "within budget". - **Settings:** performance mode; child processes with `memory_limit=-1`; the operating system's file cache warm. -**Result** (2026-11-01, release candidates `4.0.0-rc.5`, artifact `3f1b2205…`, 2 × 20 minutes, and `4.0.0-rc.6`, artifact `7b29f3cd…`: rc.6 changed the field index code, so the 15 workloads of that code were measured again on it, 2 × 2.5 minutes, and their rows below are rc.6's): +**Result** (2026-11-01, 2 × 20 minutes; the 15 workloads of the field index code were measured again after its last change, 2 × 2.5 minutes, and their rows below are those): | Workload | 3.3-dev median per run | Ratio run 1 [99% CI] | Ratio run 2 [99% CI] | p95 ratio (1 / 2) | Memory ratio (1) | Verdict | |---|---|---|---|---|---|---| @@ -156,7 +156,7 @@ and with 1000 shards of 10,000. - **Cold:** 40 new processes. - **Warm:** 200 key lookups or updates, or 40 indexed queries, in a process that has already run one of the kind. - **Memory:** `cacheMemoryLimit` is `'auto'` unless a row says otherwise. -- **What was measured:** the release candidate `4.0.0-rc.6` (artifact `7b29f3cd…`, 2026-11-01), and 3.3-dev +- **What was measured:** version 4.0.0 (2026-11-01), and 3.3-dev (`32c3d641…`). | | 4.0, 100 shards | 3.3-dev, 100 shards | 4.0, 1000 shards | 3.3-dev, 1000 shards | @@ -194,7 +194,7 @@ records takes most of the time, in both. **The 1000-shard query holds data files** under the automatic cache limit (§4). That is where its median comes from, and it has a price: a warm process peaks at 1380 MiB, and the queries that load a file show as the p95 and p99. All three stay under -3.3-dev's. The previous release candidate, which held none, had a median of 42.4 ms at a peak of 380 MiB. +3.3-dev's. A build that held none had a median of 42.4 ms at a peak of 380 MiB. ## 4. First use and later use @@ -286,7 +286,7 @@ it created or renamed a file, to reach the device (DURABILITY.md). Its cost is t That is a property of the disk and its cache, from tens of microseconds on an SSD with power-loss protection to milliseconds elsewhere. PHP can't flush the drive cache on macOS, so durable mode isn't measured there. Performance mode, which every number of this document uses, issues no sync on the write path. -The release candidate measured on Linux: PHP 8.5 in a Docker container, a database of 10,000 records, not sharded, 300 runs +Measured on Linux: PHP 8.5 in a Docker container, a database of 10,000 records, not sharded, 300 runs of each single operation. The container's virtual disk syncs at two speeds, about 0.35 and about 1.3 ms, and the median of a run can fall on either; a real device gives its own number. diff --git a/noneDB.php b/noneDB.php index fc558ce..dd77aeb 100644 --- a/noneDB.php +++ b/noneDB.php @@ -19137,7 +19137,7 @@ public function deleteSegmentRecords($dbname, $shardId, array $oldRecords){ namespace { class noneDB { /** This library's version */ - public const VERSION = '4.0.0-rc.6'; + public const VERSION = '4.0.0'; /** The storage format it reads and writes (docs/STORAGE_FORMAT.md) */ public const STORAGE_FORMAT = 4; diff --git a/src/noneDB/Facade.php b/src/noneDB/Facade.php index 633c1e8..e1ccf19 100644 --- a/src/noneDB/Facade.php +++ b/src/noneDB/Facade.php @@ -2,7 +2,7 @@ // The public API: every call goes to the instance's kernel (src/Internal) trait noneDB_Facade { /** This library's version */ - public const VERSION = '4.0.0-rc.6'; + public const VERSION = '4.0.0'; /** The storage format it reads and writes (docs/STORAGE_FORMAT.md) */ public const STORAGE_FORMAT = 4; diff --git a/tools/make-migration-fixtures.php b/tools/make-migration-fixtures.php index d7699ff..0310c3d 100644 --- a/tools/make-migration-fixtures.php +++ b/tools/make-migration-fixtures.php @@ -255,7 +255,7 @@ function runRelease(string $root, array $release, array $settings, string $work, + (isset($settings['shardSize']) ? ['shardSize' => $settings['shardSize']] : []), true) : 'null'; $script = "$work/run-" . bin2hex(random_bytes(4)) . '.php'; file_put_contents($script, 'db; ' . $body . "\n"); - $process = proc_open([$php, $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $work); + // Without notices (1.4 gives one on a key that isn't there), and what is left comes here whatever php.ini says + $process = proc_open([$php, '-d', 'display_errors=1', '-d', 'log_errors=0', $script], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes, $work); $out = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); fclose($pipes[1]); fclose($pipes[2]); From 89b0ca06d0c2ac6df9c3c7bec505bab5e4037891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 20:17:19 +0300 Subject: [PATCH 3/6] Windows: no file is open when it is replaced or moved Windows replaces no file that is open and moves no directory with an open file in it. The writer of the global field index kept the old file open through its own rename, so on Windows the file was never replaced. It closes it first now, also when a damaged row makes the file again. The migration tool reads the files it has locked through their handles, since a lock keeps every other handle out. It drops the engine's lock handles before it publishes, and lets go of a lock before its file or directory moves. Tests: FileHandlesTest looks for an open handle at every rename, on every system. Paths, cd and the tests that need POSIX follow the system they run on. OPERATIONS.md says what a rotation does on Windows under a running process. --- docs/OPERATIONS.md | 3 + migrate/Files.php | 46 +++++++- migrate/Publisher/Migration.php | 17 ++- migrate/Readers/Source.php | 2 +- migrate/Restore/Restore.php | 7 +- noneDB-migrate.php | 72 ++++++++++-- noneDB.php | 13 ++- src/Internal/Index/FieldIndex.php | 13 ++- tests/Crash/GenerationSwitchTest.php | 4 +- tests/Crash/KeyMaintenanceTest.php | 49 ++++++++ tests/Integration/BoundedMemoryTest.php | 2 +- tests/Integration/ConfigurationTest.php | 12 +- tests/Integration/DurabilityConfigTest.php | 3 + tests/Integration/FileHandlesTest.php | 107 ++++++++++++++++++ tests/Integration/HardeningRegressionTest.php | 2 +- tests/Integration/OperationsToolsTest.php | 3 + .../SecurityConfigRegressionTest.php | 12 +- tests/Integration/SpecOperationsTest.php | 36 +++--- tests/Integration/SpecStorageTest.php | 1 + tests/Migration/MigrationProcessTest.php | 2 +- tests/Support/Platform.php | 22 ++++ tests/Unit/ArchitectureTest.php | 2 +- tests/Unit/BuildTest.php | 2 +- tests/bootstrap.php | 8 +- tests/noneDBTestCase.php | 10 ++ 25 files changed, 390 insertions(+), 60 deletions(-) create mode 100644 tests/Integration/FileHandlesTest.php create mode 100644 tests/Support/Platform.php diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index ca591a9..c208217 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -141,6 +141,9 @@ file_put_contents('/secure/place/store-manifest.json', json_encode($r['storeMani **Rotating a key:** 1. Stop the processes that use the store (a database in use makes the rotation `MAINTENANCE_BUSY`, and nothing is written). + On Windows a process that has used a database keeps its `LOCK` open while it runs, and a directory with an open file + in it can't be renamed. The rotation then stops with "run migrateKey() again": the store stays closed, nothing is lost, + and a new run finishes it once that process has ended. 2. Run `migrateKey()`. Every database directory of the old key is renamed to the new key's locator; contents don't change. A process that still runs with the old key gets `MAINTENANCE_IN_PROGRESS` during the rotation and `KEY_NOT_REGISTERED` after it, never an empty database. diff --git a/migrate/Files.php b/migrate/Files.php index 3a49c7d..0bf210d 100644 --- a/migrate/Files.php +++ b/migrate/Files.php @@ -10,6 +10,9 @@ final class Files { public const FILE_MODE = 0640; public const DIR_MODE = 0750; + /** The lock handles of the run by file name: on Windows a lock keeps every other handle from reading the file */ + private static $locked = []; + /** * Write a file atomically: a temp file in the same directory, synced, renamed over the target, the directory * synced; $mode set on it whatever the umask @@ -132,22 +135,55 @@ public static function makeDir(string $dir, int $mode = self::DIR_MODE): void { */ public static function openLock(string $path) { $created = !is_file($path); - $fp = @fopen($path, 'c'); + $fp = @fopen($path, 'c+b'); if($fp !== false && $created){ @chmod($path, self::FILE_MODE); } return $fp; } + /** The run has this file locked: from now on it is read through this handle */ + public static function keepLocked(string $path, $fp): void { + self::$locked[basename($path)] = $fp; + } + + /** The handle of a file the run has locked, at the file's start; null for any other file */ + private static function lockedHandle(string $path) { + $fp = self::$locked[basename($path)] ?? null; + if(!is_resource($fp)){ + return null; + } + rewind($fp); + return $fp; + } + + /** SHA-256 of a file */ + public static function sha256(string $path): string { + $fp = self::lockedHandle($path); + if($fp === null){ + return (string)@hash_file('sha256', $path); + } + $hash = hash_init('sha256'); + hash_update_stream($hash, $fp); + return hash_final($hash); + } + + /** The first bytes of a file, '' when it can't be read */ + public static function head(string $path, int $length): string { + $fp = self::lockedHandle($path); + return (string)($fp === null ? @file_get_contents($path, false, null, 0, $length) : fread($fp, $length)); + } + /** * Copy a file, synced, and return the SHA-256 of what was copied. $mode: set on the copy (else the umask decides) * @throws Stop */ public static function copy(string $from, string $to, ?int $mode = null): string { - $in = @fopen($from, 'rb'); + $held = self::lockedHandle($from); + $in = $held ?? @fopen($from, 'rb'); $out = $in === false ? false : @fopen($to, 'xb'); if($in === false || $out === false){ - if($in !== false){ + if($in !== false && $held === null){ fclose($in); } throw new Stop('FAILED', "cannot copy $from to $to"); @@ -164,7 +200,9 @@ public static function copy(string $from, string $to, ?int $mode = null): string $ok = fwrite($out, $chunk) === strlen($chunk); } $ok = $ok && fflush($out) && self::syncHandle($out); - fclose($in); + if($held === null){ + fclose($in); + } fclose($out); if(!$ok){ throw new Stop('FAILED', "cannot copy $from to $to"); diff --git a/migrate/Publisher/Migration.php b/migrate/Publisher/Migration.php index 1927d9f..e3cf629 100644 --- a/migrate/Publisher/Migration.php +++ b/migrate/Publisher/Migration.php @@ -94,6 +94,7 @@ public static function run(string $dir, string $sourceKey, string $targetKey, ar * @throws Stop */ public static function abort(string $dir): array { + \noneDB::clearStaticCache(); // staging goes: Windows removes no file that is open $run = new self($dir, '', '', []); $journal = Journal::read($run->dir); if($journal === null || ($journal['op'] ?? null) !== 'migrate'){ @@ -317,6 +318,7 @@ private function publish(array $journal): array { }, $journal['targets']); $this->write($journal); } + \noneDB::clearStaticCache(); // the engine's lock handles: Windows moves no directory with an open file in it foreach($journal['published'] as $locator){ $staged = $this->staging() . $locator; if(is_dir($this->dir . $locator) && !is_dir($staged)){ @@ -368,6 +370,7 @@ private function apply(array $journal): array { Files::makeDir($retained . '/files'); foreach(self::prints($journal['files']) as $file => $fingerprint){ if(is_file($this->dir . $file)){ + $this->unlock($this->dir . $file); if(!@rename($this->dir . $file, "$retained/files/$file")){ throw new Stop('MAINTENANCE_IN_PROGRESS', "committed, not completed: cannot move $file into $retained; run again"); } @@ -478,7 +481,8 @@ private function lock(array $targets): void { throw $busy ? new Stop('MAINTENANCE_BUSY', basename($path) . ' is locked by another process; nothing was changed') : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); } - $this->locks[] = $fp; + $this->locks[$path] = $fp; + Files::keepLocked($path, $fp); } } @@ -504,10 +508,19 @@ private function fingerprints(array $groups): array { return $prints; } + /** A file of the run leaves the directory: its lock goes first */ + private function unlock(string $path): void { + if(isset($this->locks[$path])){ + flock($this->locks[$path], LOCK_UN); + fclose($this->locks[$path]); + unset($this->locks[$path]); + } + } + private function fingerprint(string $file): ?array { clearstatcache(true, $this->dir . $file); $stat = @stat($this->dir . $file); - return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], hash_file('sha256', $this->dir . $file)]; + return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], Files::sha256($this->dir . $file)]; } /** Every file of the groups: file => fingerprint */ diff --git a/migrate/Readers/Source.php b/migrate/Readers/Source.php index 7a987f2..59ad69f 100644 --- a/migrate/Readers/Source.php +++ b/migrate/Readers/Source.php @@ -91,7 +91,7 @@ private function readAll(): void { $lock = $this->path('.nonedb.lock'); if(is_file($lock)){ $this->locked = true; - $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', (string)file_get_contents($lock, false, null, 0, 128)) === 1; + $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', \noneDB\Migrate\Files::head($lock, 128)) === 1; } if(is_file($this->path('.nonedb.meta'))){ $this->readSharded(); diff --git a/migrate/Restore/Restore.php b/migrate/Restore/Restore.php index caf6ce3..2bd7f78 100644 --- a/migrate/Restore/Restore.php +++ b/migrate/Restore/Restore.php @@ -48,6 +48,11 @@ public static function restore(string $dir, string $backupDir, string $runId, st Journal::write(rtrim($dir, '/'), ['op' => 'restore-old', 'id' => $runId, 'state' => 'RESTORING']); Files::makeDir("$runDir/v4", 0700); foreach($done['published'] as $locator){ + if(isset($locks[$dir . $locator . '/LOCK'])){ // Windows moves no directory with an open file in it + flock($locks[$dir . $locator . '/LOCK'], LOCK_UN); + fclose($locks[$dir . $locator . '/LOCK']); + unset($locks[$dir . $locator . '/LOCK']); + } if(is_dir($dir . $locator) && !@rename($dir . $locator, "$runDir/v4/$locator")){ throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $locator into the backup; run --restore again"); } @@ -141,7 +146,7 @@ private static function lock(string $dir, array $published): array { throw $busy ? new Stop('MAINTENANCE_BUSY', basename(dirname($path)) . '/' . basename($path) . ' is locked by another process; nothing was changed') : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); } - $locks[] = $fp; + $locks[$path] = $fp; } return $locks; } diff --git a/noneDB-migrate.php b/noneDB-migrate.php index 4176227..8e18ccf 100644 --- a/noneDB-migrate.php +++ b/noneDB-migrate.php @@ -40,6 +40,9 @@ final class Files { public const FILE_MODE = 0640; public const DIR_MODE = 0750; + /** The lock handles of the run by file name: on Windows a lock keeps every other handle from reading the file */ + private static $locked = []; + /** * Write a file atomically: a temp file in the same directory, synced, renamed over the target, the directory * synced; $mode set on it whatever the umask @@ -150,22 +153,55 @@ public static function makeDir(string $dir, int $mode = self::DIR_MODE): void { */ public static function openLock(string $path) { $created = !is_file($path); - $fp = @fopen($path, 'c'); + $fp = @fopen($path, 'c+b'); if($fp !== false && $created){ @chmod($path, self::FILE_MODE); } return $fp; } + /** The run has this file locked: from now on it is read through this handle */ + public static function keepLocked(string $path, $fp): void { + self::$locked[basename($path)] = $fp; + } + + /** The handle of a file the run has locked, at the file's start; null for any other file */ + private static function lockedHandle(string $path) { + $fp = self::$locked[basename($path)] ?? null; + if(!is_resource($fp)){ + return null; + } + rewind($fp); + return $fp; + } + + /** SHA-256 of a file */ + public static function sha256(string $path): string { + $fp = self::lockedHandle($path); + if($fp === null){ + return (string)@hash_file('sha256', $path); + } + $hash = hash_init('sha256'); + hash_update_stream($hash, $fp); + return hash_final($hash); + } + + /** The first bytes of a file, '' when it can't be read */ + public static function head(string $path, int $length): string { + $fp = self::lockedHandle($path); + return (string)($fp === null ? @file_get_contents($path, false, null, 0, $length) : fread($fp, $length)); + } + /** * Copy a file, synced, and return the SHA-256 of what was copied. $mode: set on the copy (else the umask decides) * @throws Stop */ public static function copy(string $from, string $to, ?int $mode = null): string { - $in = @fopen($from, 'rb'); + $held = self::lockedHandle($from); + $in = $held ?? @fopen($from, 'rb'); $out = $in === false ? false : @fopen($to, 'xb'); if($in === false || $out === false){ - if($in !== false){ + if($in !== false && $held === null){ fclose($in); } throw new Stop('FAILED', "cannot copy $from to $to"); @@ -182,7 +218,9 @@ public static function copy(string $from, string $to, ?int $mode = null): string $ok = fwrite($out, $chunk) === strlen($chunk); } $ok = $ok && fflush($out) && self::syncHandle($out); - fclose($in); + if($held === null){ + fclose($in); + } fclose($out); if(!$ok){ throw new Stop('FAILED', "cannot copy $from to $to"); @@ -410,7 +448,7 @@ private function readAll(): void { $lock = $this->path('.nonedb.lock'); if(is_file($lock)){ $this->locked = true; - $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', (string)file_get_contents($lock, false, null, 0, 128)) === 1; + $this->epoch = preg_match('/^\d+:[0-9a-f]{8}(?: |$)/', \noneDB\Migrate\Files::head($lock, 128)) === 1; } if(is_file($this->path('.nonedb.meta'))){ $this->readSharded(); @@ -1419,6 +1457,7 @@ public static function run(string $dir, string $sourceKey, string $targetKey, ar * @throws Stop */ public static function abort(string $dir): array { + \noneDB::clearStaticCache(); // staging goes: Windows removes no file that is open $run = new self($dir, '', '', []); $journal = Journal::read($run->dir); if($journal === null || ($journal['op'] ?? null) !== 'migrate'){ @@ -1641,6 +1680,7 @@ private function publish(array $journal): array { }, $journal['targets']); $this->write($journal); } + \noneDB::clearStaticCache(); // the engine's lock handles: Windows moves no directory with an open file in it foreach($journal['published'] as $locator){ $staged = $this->staging() . $locator; if(is_dir($this->dir . $locator) && !is_dir($staged)){ @@ -1690,6 +1730,7 @@ private function apply(array $journal): array { Files::makeDir($retained . '/files'); foreach(self::prints($journal['files']) as $file => $fingerprint){ if(is_file($this->dir . $file)){ + $this->unlock($this->dir . $file); if(!@rename($this->dir . $file, "$retained/files/$file")){ throw new Stop('MAINTENANCE_IN_PROGRESS', "committed, not completed: cannot move $file into $retained; run again"); } @@ -1796,7 +1837,8 @@ private function lock(array $targets): void { throw $busy ? new Stop('MAINTENANCE_BUSY', basename($path) . ' is locked by another process; nothing was changed') : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); } - $this->locks[] = $fp; + $this->locks[$path] = $fp; + Files::keepLocked($path, $fp); } } @@ -1822,10 +1864,19 @@ private function fingerprints(array $groups): array { return $prints; } + /** A file of the run leaves the directory: its lock goes first */ + private function unlock(string $path): void { + if(isset($this->locks[$path])){ + flock($this->locks[$path], LOCK_UN); + fclose($this->locks[$path]); + unset($this->locks[$path]); + } + } + private function fingerprint(string $file): ?array { clearstatcache(true, $this->dir . $file); $stat = @stat($this->dir . $file); - return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], hash_file('sha256', $this->dir . $file)]; + return $stat === false ? null : [$stat['size'], $stat['mtime'], $stat['ino'], Files::sha256($this->dir . $file)]; } /** Every file of the groups: file => fingerprint */ @@ -1959,6 +2010,11 @@ public static function restore(string $dir, string $backupDir, string $runId, st Journal::write(rtrim($dir, '/'), ['op' => 'restore-old', 'id' => $runId, 'state' => 'RESTORING']); Files::makeDir("$runDir/v4", 0700); foreach($done['published'] as $locator){ + if(isset($locks[$dir . $locator . '/LOCK'])){ // Windows moves no directory with an open file in it + flock($locks[$dir . $locator . '/LOCK'], LOCK_UN); + fclose($locks[$dir . $locator . '/LOCK']); + unset($locks[$dir . $locator . '/LOCK']); + } if(is_dir($dir . $locator) && !@rename($dir . $locator, "$runDir/v4/$locator")){ throw new Stop('MAINTENANCE_IN_PROGRESS', "cannot move $locator into the backup; run --restore again"); } @@ -2052,7 +2108,7 @@ private static function lock(string $dir, array $published): array { throw $busy ? new Stop('MAINTENANCE_BUSY', basename(dirname($path)) . '/' . basename($path) . ' is locked by another process; nothing was changed') : new Stop('FAILED', "cannot open $path (permissions of the directory?); nothing was changed"); } - $locks[] = $fp; + $locks[$path] = $fp; } return $locks; } diff --git a/noneDB.php b/noneDB.php index dd77aeb..dbad4a3 100644 --- a/noneDB.php +++ b/noneDB.php @@ -4509,7 +4509,7 @@ public function getDurability(): array { array_map, array_merge, array_multisort, array_search, array_slice, array_sum, array_unique, array_values, basename, bin2hex, chr, clearstatcache, count, end, fclose, fgets, file_exists, file_get_contents, filesize, fopen, fread, fseek, fstat, hash, hex2bin, in_array, intdiv, is_array, is_file, is_float, is_int, is_nan, is_null, is_numeric, - is_scalar, is_string, json_decode, ksort, max, memory_get_usage, min, preg_match, random_bytes, sort, + is_resource, is_scalar, is_string, json_decode, ksort, max, memory_get_usage, min, preg_match, random_bytes, sort, str_repeat, strcmp, strlen, strncmp, strpos, substr, time, unpack; use const PHP_SAPI, SORT_NUMERIC, SORT_STRING; @@ -5857,13 +5857,20 @@ public function writeGlobalFieldIndex($dbname, $field, array $global){ } yield ShardFilters::seal($rows); } + fclose($source); // before the rename: Windows doesn't replace a file that is open }; + $damaged = false; try { $this->writeGlobalFile($dbname, $field, $path, $global['sh'], $global['bits'], $global['width'], $pages()); } catch(\UnexpectedValueException $e){ - $this->rebuildGlobalFieldIndex($dbname, $field); + $damaged = true; } finally { - fclose($source); + if(is_resource($source)){ + fclose($source); + } + } + if($damaged){ + $this->rebuildGlobalFieldIndex($dbname, $field); } } diff --git a/src/Internal/Index/FieldIndex.php b/src/Internal/Index/FieldIndex.php index 2f2fb00..fc4765e 100644 --- a/src/Internal/Index/FieldIndex.php +++ b/src/Internal/Index/FieldIndex.php @@ -8,7 +8,7 @@ array_map, array_merge, array_multisort, array_search, array_slice, array_sum, array_unique, array_values, basename, bin2hex, chr, clearstatcache, count, end, fclose, fgets, file_exists, file_get_contents, filesize, fopen, fread, fseek, fstat, hash, hex2bin, in_array, intdiv, is_array, is_file, is_float, is_int, is_nan, is_null, is_numeric, - is_scalar, is_string, json_decode, ksort, max, memory_get_usage, min, preg_match, random_bytes, sort, + is_resource, is_scalar, is_string, json_decode, ksort, max, memory_get_usage, min, preg_match, random_bytes, sort, str_repeat, strcmp, strlen, strncmp, strpos, substr, time, unpack; use const PHP_SAPI, SORT_NUMERIC, SORT_STRING; @@ -1365,13 +1365,20 @@ public function writeGlobalFieldIndex($dbname, $field, array $global){ } yield ShardFilters::seal($rows); } + fclose($source); // before the rename: Windows doesn't replace a file that is open }; + $damaged = false; try { $this->writeGlobalFile($dbname, $field, $path, $global['sh'], $global['bits'], $global['width'], $pages()); } catch(\UnexpectedValueException $e){ - $this->rebuildGlobalFieldIndex($dbname, $field); + $damaged = true; } finally { - fclose($source); + if(is_resource($source)){ + fclose($source); + } + } + if($damaged){ + $this->rebuildGlobalFieldIndex($dbname, $field); } } diff --git a/tests/Crash/GenerationSwitchTest.php b/tests/Crash/GenerationSwitchTest.php index 5749ea0..ebaf81e 100644 --- a/tests/Crash/GenerationSwitchTest.php +++ b/tests/Crash/GenerationSwitchTest.php @@ -113,8 +113,8 @@ public function theSwitchInterruptedAtEveryStepOpensOneConsistentGeneration(): v */ public function anObsoleteGenerationThatCannotBeRemovedIsLoggedAndReported(): void { - if (function_exists('posix_geteuid') && posix_geteuid() === 0) { - $this->markTestSkipped('root ignores file permissions'); + if (PHP_OS_FAMILY === 'Windows' || (function_exists('posix_geteuid') && posix_geteuid() === 0)) { + $this->markTestSkipped('no permissions that keep an entry from being removed'); } $this->assertSame('', $this->interruptSwitchAt('generation:published')); $this->assertContains('g=1>2', $this->lockState('m')['flags']); diff --git a/tests/Crash/KeyMaintenanceTest.php b/tests/Crash/KeyMaintenanceTest.php index 854ed48..d694676 100644 --- a/tests/Crash/KeyMaintenanceTest.php +++ b/tests/Crash/KeyMaintenanceTest.php @@ -287,9 +287,58 @@ public function aBusyDatabaseStopsMaintenanceBeforeAnythingIsWritten(): void $this->assertRotated('after the lock was released'); } + /** + * @test docs/OPERATIONS.md: on Windows no directory with an open file in it can move, and a process keeps the handle of + * the LOCK it used. The rotation stops before its commit and says so, the old key stays out, and it finishes once that + * process has ended + */ + public function aRotationWaitsForAProcessThatKeepsItsLockHandle(): void + { + if (PHP_OS_FAMILY !== 'Windows') { + $this->markTestSkipped('a directory with an open file in it moves everywhere else'); + } + $this->seed(); + $go = $this->testDbDir . '../rot-go'; + $ready = $this->testDbDir . '../rot-ready'; + @unlink($go); + @unlink($ready); + $proc = proc_open([PHP_BINARY, $this->childScript(' + $db = new noneDB(["secretKey" => ' . var_export(self::SECRET, true) . ', "dbDir" => ' . var_export($this->testDbDir, true) . ']); + $db->find("users", 0); + file_put_contents(' . var_export($ready, true) . ', "1"); + $deadline = microtime(true) + 30; + while (!is_file(' . var_export($go, true) . ') && microtime(true) < $deadline) { + usleep(5000); + } + ')], [1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + try { + $deadline = microtime(true) + 10; + while (!is_file($ready) && microtime(true) < $deadline) { + usleep(5000); + } + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertFalse($result['success'], 'the directory can not move under the process'); + $this->assertStringContainsString('run migrateKey() again', (string)$result['error']); + } finally { + file_put_contents($go, '1'); + stream_get_contents($pipes[1]); + stream_get_contents($pipes[2]); + proc_close($proc); + @unlink($go); + @unlink($ready); + } + \noneDB::clearStaticCache(); + $result = \noneDB::migrateKey($this->testDbDir, self::SECRET, self::NEW); + $this->assertTrue($result['success'], 'the process has ended: ' . (string)($result['error'] ?? '')); + $this->assertCount(4, $this->with(self::NEW)->find('users', 0)); + } + /** @test */ public function aRunningProcessNeverSeesAnEmptyDatabaseDuringARotation(): void { + if (PHP_OS_FAMILY === 'Windows') { + $this->markTestSkipped('a running process keeps its lock handle: aRotationWaitsForAProcessThatKeepsItsLockHandle'); + } $this->seed(); $go = $this->testDbDir . '../rot-go'; $ready = $this->testDbDir . '../rot-ready'; diff --git a/tests/Integration/BoundedMemoryTest.php b/tests/Integration/BoundedMemoryTest.php index 14e1627..18d70de 100644 --- a/tests/Integration/BoundedMemoryTest.php +++ b/tests/Integration/BoundedMemoryTest.php @@ -286,7 +286,7 @@ public function aFieldOfDistinctValuesCostsNoArrayPerValue(): void unset($db); \noneDB::clearStaticCache(); $batch = self::batchBytes(); - $maps = self::LARGE * self::TWO_MAPS; + $maps = self::LARGE * self::TWO_MAPS + 2 * 1048576; // and what differs between systems // A flat list costs 32 bytes per entry on PHP 8.1, 16 later (60 000 entries: 2.1 and 1.05 MB): a range's ids, strings and bucket ids $lists = PHP_VERSION_ID < 80200 ? 3 * 16 * self::LARGE : 0; $steps = [ diff --git a/tests/Integration/ConfigurationTest.php b/tests/Integration/ConfigurationTest.php index ebc1a08..269fa48 100644 --- a/tests/Integration/ConfigurationTest.php +++ b/tests/Integration/ConfigurationTest.php @@ -9,6 +9,8 @@ */ class ConfigurationTest extends TestCase { + use \noneDB\Tests\Support\Platform; + private $testDbDir; private $hadScript = false; private $savedScript = null; @@ -76,7 +78,7 @@ private function runIsolated(string $code, string $env = ''): string $dir = $this->scratch('iso'); copy(__DIR__ . '/../../noneDB.php', $dir . '/noneDB.php'); file_put_contents($dir . '/run.php', "&1')); } @@ -166,7 +168,7 @@ public function configExistsChecksMultiplePaths(): void $this->entryScriptIn($script); file_put_contents($script . '/.nonedb', json_encode(['secretKey' => 'script-dir-key-0123', 'dbDir' => 'here'])); $this->assertTrue(\noneDB::configExists()); - $this->assertSame($script . '/here/', (new \noneDB())->__debugInfo()['dbDir'], 'the file next to the entry script is used'); + $this->assertSame(self::resolvedDir($script . '/here/'), (new \noneDB())->__debugInfo()['dbDir'], 'the file next to the entry script is used'); // In a process where no file exists anywhere: false; then one in noneDB.php's directory, then in the cwd $out = $this->runIsolated('require __DIR__ . "/noneDB.php"; @@ -206,7 +208,7 @@ public function getConfigTemplateReturnsPathToExampleFile(): void $this->assertArrayHasKey('lockRetryDelay', $template); // "the .nonedb.example next to noneDB.php" (the library that runs: the test build) - $this->assertSame(realpath(dirname(NONEDB_TEST_LIBRARY) . '/.nonedb.example'), $templatePath); + $this->assertSame(realpath(dirname(NONEDB_TEST_LIBRARY) . '/.nonedb.example'), realpath($templatePath)); // Its key is a placeholder: "replace the placeholder secretKey (it is rejected)" try { new \noneDB(['dbDir' => $this->testDbDir] + $template); @@ -329,7 +331,7 @@ public function relativeDbDirIsResolved(): void ]; $db = new \noneDB($config); - $this->assertSame($dir . '/test_db/', $db->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir($dir . '/test_db/'), $db->__debugInfo()['dbDir']); $result = $db->insert('relative-test', ['data' => 'test']); $this->assertSame(['n' => 1], $result); @@ -539,7 +541,7 @@ public function missingConfigInProductionThrowsException(): void try { new noneDB(); echo "NO_EXCEPTION"; } catch (RuntimeException $e) { echo get_class($e), ":", $e->getMessage(); }'); - $cmd = 'cd ' . escapeshellarg($tempDir) . ' && env -u NONEDB_DEV_MODE ' + $cmd = self::cd($tempDir) . 'env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1'; $output = shell_exec($cmd); diff --git a/tests/Integration/DurabilityConfigTest.php b/tests/Integration/DurabilityConfigTest.php index aad50cc..b56d83e 100644 --- a/tests/Integration/DurabilityConfigTest.php +++ b/tests/Integration/DurabilityConfigTest.php @@ -123,6 +123,7 @@ public function theReportHasTheDocumentedShape(): void /** @test §6: lastOperation of a committed write, in both modes */ public function aCommittedWriteIsReported(): void { + $this->needsDirectorySyncs(); $db = $this->db(); $db->insert('rep', ['v' => 1]); $last = $db->getDurability()['lastOperation']; @@ -186,6 +187,7 @@ public function operationsAreNamed(): void /** @test §6: "Paths are relative to dbDir. No key material appears." */ public function theLastSyncErrorHasNoAbsolutePathAndNoKey(): void { + $this->needsDirectorySyncs(); $this->onPlatform('Linux'); $db = $this->db(['durability' => 'durable']); $db->insert('err', ['v' => 1]); @@ -223,6 +225,7 @@ public function theModeBelongsToTheInstance(): void /** @test maintenance takes the same durability option (docs/DURABILITY.md §1) */ public function maintenanceTakesTheDurabilityOption(): void { + $this->needsDirectorySyncs(); $this->db()->insert('m', ['v' => 1]); $other = 'second_registered_key_0123'; $this->onPlatform('Darwin'); diff --git a/tests/Integration/FileHandlesTest.php b/tests/Integration/FileHandlesTest.php new file mode 100644 index 0000000..b75c53c --- /dev/null +++ b/tests/Integration/FileHandlesTest.php @@ -0,0 +1,107 @@ +open[] = basename((string)$uri); + } + } + return rename($from, $to); + }); + } + + private function records(int $n): array + { + $records = []; + for ($i = 0; $i < $n; $i++) { + $records[] = ['g' => 'g' . ($i % 7)]; + } + return $records; + } + + /** The keys of a value in a shard of 100, as records() makes them */ + private static function keysOf(string $value, int $shard): array + { + return array_values(array_filter(range(100 * $shard, 100 * $shard + 99), function ($key) use ($value) { return 'g' . ($key % 7) === $value; })); + } + + /** @test a shard's filter goes into the global file: the old file, which its rows are copied from, is closed by then */ + public function aGlobalFileIsNotOpenWhenItIsReplaced(): void + { + $config = ['shardSize' => 100, 'autoCompactThreshold' => 1000]; // no compaction: it would replace the shard's data + $db = $this->db($config); + $db->insert('gf', $this->records(300)); + $this->assertTrue($db->createFieldIndex('gf', 'g')['success']); + $gfx = $this->dbPath('gf', 'gfx', 'g'); + $before = file_get_contents($gfx); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(self::keysOf('g3', 1), $db->getKeysFromFieldIndex('gf', 'g', 'g3', 1)); + $this->assertSame(100, $db->update('gf', [['key' => range(100, 199)], ['set' => ['g' => 'new', 'pad' => str_repeat('y', 1000)]]])['n']); + $this->assertNotSame($before, file_get_contents($gfx), 'written again'); + $this->assertSame([], $this->open); + } + + /** @test the same when a row of the old file is damaged and the file is made again */ + public function aGlobalFileWithADamagedRowIsNotOpenWhenItIsMadeAgain(): void + { + $config = ['shardSize' => 100, 'autoCompactThreshold' => 1000]; + $db = $this->db($config); + $db->insert('gd', $this->records(300)); + $this->assertTrue($db->createFieldIndex('gd', 'g')['success']); + $gfx = $this->dbPath('gd', 'gfx', 'g'); + $raw = (string)file_get_contents($gfx); + $end = strlen($raw) - 2; + $raw[$end] = chr(ord($raw[$end]) ^ 1); + file_put_contents($gfx, $raw); + \noneDB::clearStaticCache(); + $db = $this->db($config); + $this->assertSame(self::keysOf('g3', 1), $db->getKeysFromFieldIndex('gd', 'g', 'g3', 1)); + $this->assertSame(100, $db->update('gd', [['key' => range(100, 199)], ['set' => ['pad' => str_repeat('y', 1000)]]])['n']); + $this->assertNotSame($raw, file_get_contents($gfx), 'made again'); + $this->assertSame([], $this->open); + } + + /** @test the writes that replace files or move directories: compaction, the switch to shards, a rotation, a restore */ + public function nothingIsOpenWhenMaintenanceReplacesAFile(): void + { + $db = $this->db(['shardingEnabled' => false]); + $db->insert('mt', $this->records(200)); + $this->assertTrue($db->createFieldIndex('mt', 'g')['success']); + $this->assertSame(10, $db->delete('mt', ['key' => range(0, 9)])['n']); + $db->compact('mt'); + $this->assertTrue($db->rebuildIndex('mt')['success']); + $db->convertToSharded('mt'); + $backup = $this->testDbDir . '../handles-backup'; + $this->removeDirectoryTree($backup); + try { + $db->backup('mt', $backup); + $db->restore($backup, 'mt'); + } finally { + $this->removeDirectoryTree($backup); + } + $this->assertCount(190, $db->find('mt', 0)); + unset($db); + $this->noneDB = null; + $this->assertTrue(\noneDB::migrateKey($this->testDbDir, 'test_secret_key_for_unit_tests', 'another_secret_key_for_handles')['success']); + $this->assertSame([], $this->open); + } +} diff --git a/tests/Integration/HardeningRegressionTest.php b/tests/Integration/HardeningRegressionTest.php index 4b79a32..e2e93a7 100644 --- a/tests/Integration/HardeningRegressionTest.php +++ b/tests/Integration/HardeningRegressionTest.php @@ -328,7 +328,7 @@ public function interruptedDropCanBeRepeatedAndNothingComesBack(): void public function dbDirWithGlobCharactersWorks(): void { // A-7: glob() read [ab] as a pattern: interrupted flushes were never replayed, dropDB deleted nothing - $dir = $this->testDbDir . '[ab]*?/'; + $dir = $this->testDbDir . (PHP_OS_FAMILY === 'Windows' ? '[ab]/' : '[ab]*?/'); // no * or ? in a name on Windows @mkdir($dir, 0777, true); try { $config = ['secretKey' => self::SECRET, 'dbDir' => $dir, 'bufferEnabled' => true, 'shardSize' => 4]; diff --git a/tests/Integration/OperationsToolsTest.php b/tests/Integration/OperationsToolsTest.php index cd18ce0..8d4d7a8 100644 --- a/tests/Integration/OperationsToolsTest.php +++ b/tests/Integration/OperationsToolsTest.php @@ -28,6 +28,7 @@ protected function tearDown(): void /** @test DURABILITY.md §6: the report says what the last write needed and did, in each mode */ public function durabilityReportDescribesWhatWasDone(): void { + $this->needsDirectorySyncs(); $db = $this->db(); $report = $db->getDurability(); $this->assertSame(['mode', 'platform', 'lastOperation', 'lastSyncError', 'syncs'], array_keys($report)); @@ -56,6 +57,7 @@ public function durabilityReportDescribesWhatWasDone(): void /** @test DURABILITY.md §3: a dbDir the first write creates is synced into its parent like any new directory */ public function aDatabaseDirectoryCreatedByTheFirstWriteIsSynced(): void { + $this->needsDirectorySyncs(); // dbDir is used by its canonical path (STORAGE_FORMAT.md §3): the temp directory may be a link (macOS) $parent = realpath(sys_get_temp_dir()) . '/nonedb-dirsync-' . bin2hex(random_bytes(6)); $dir = $parent . '/db/'; @@ -86,6 +88,7 @@ public function aDatabaseDirectoryCreatedByTheFirstWriteIsSynced(): void /** @test DURABILITY.md §2: a failed sync fails the write (never success), and is never carried over */ public function aFailedSyncIsReportedAndNeverCarriedOver(): void { + $this->needsDirectorySyncs(); $this->setStatic('osFamily', 'Linux'); $db = $this->db(['durability' => 'durable']); $db->insert('sf', ['v' => 1]); diff --git a/tests/Integration/SecurityConfigRegressionTest.php b/tests/Integration/SecurityConfigRegressionTest.php index 8f901a4..be1cadd 100644 --- a/tests/Integration/SecurityConfigRegressionTest.php +++ b/tests/Integration/SecurityConfigRegressionTest.php @@ -130,7 +130,7 @@ public function configFileResolution(): void $php = escapeshellarg(PHP_BINARY); $web = trim(shell_exec("$php " . escapeshellarg("$p/public/index.php") . ' 2>&1')); $cron = trim(shell_exec("$php " . escapeshellarg("$p/bin/cron.php") . ' 2>&1')); - $fromParent = trim(shell_exec('cd ' . escapeshellarg($p) . " && $php bin/cron.php 2>&1")); + $fromParent = trim(shell_exec(self::cd($p) . "$php bin/cron.php 2>&1")); $mixed = trim(shell_exec("$php " . escapeshellarg("$p/bin/mixed.php") . ' 2>&1')); $this->assertSame('[{"from":"web","key":0}]', $web); $this->assertSame($web, $cron, 'web and cron share one database'); @@ -155,13 +155,13 @@ public function plainRelativeDbDirFollowsTheConfigFile(): void // Array config: relative to the entry script (bin/), not to the working directory file_put_contents("$p/bin/array.php", ' "other-key-123", "dbDir" => "store/"]); $db->insert("a", ["x" => 1]); echo "ok";'); $php = escapeshellarg(PHP_BINARY); - $web = trim(shell_exec('cd ' . escapeshellarg("$p/public") . " && $php index.php 2>&1")); - $cron = trim(shell_exec('cd ' . escapeshellarg(sys_get_temp_dir()) . " && $php " . escapeshellarg("$p/bin/cron.php") . ' 2>&1')); + $web = trim(shell_exec(self::cd("$p/public") . "$php index.php 2>&1")); + $cron = trim(shell_exec(self::cd(sys_get_temp_dir()) . "$php " . escapeshellarg("$p/bin/cron.php") . ' 2>&1')); $this->assertSame('[{"from":"web","key":0}]', $web); $this->assertSame($web, $cron, 'a cwd elsewhere does not matter'); $this->assertSame([$p . '/data/' . $this->locatorOf('orders', 's3cr3t-long-random') . '.ndb4'], glob("$p/data/*.ndb4")); $this->assertSame([], glob("$p/public/data") ?: []); - $this->assertSame('ok', trim(shell_exec('cd ' . escapeshellarg($p) . " && $php bin/array.php 2>&1"))); + $this->assertSame('ok', trim(shell_exec(self::cd($p) . "$php bin/array.php 2>&1"))); $this->assertSame([$p . '/bin/store/' . $this->locatorOf('a', 'other-key-123') . '.ndb4'], glob("$p/bin/store/*.ndb4")); $this->assertSame([], glob("$p/store") ?: []); } @@ -183,7 +183,7 @@ public function givenInvalidKeyIsRejectedWithoutConfigFileAndInDevMode(): void // Dev mode: an invalid key is not silently replaced by the predictable dev mode key. foreach (['' => 'NOKEY:RuntimeException', 'define("NONEDB_DEV_MODE", true);' => 'NOKEY:dev'] as $prefix => $noKey) { file_put_contents("$dir/run.php", sprintf($script, $prefix)); - $out = trim((string)shell_exec('cd ' . escapeshellarg($dir) . ' && env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1')); + $out = trim((string)shell_exec(self::cd($dir) . 'env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1')); $this->assertSame("REJECTED REJECTED REJECTED $noKey", $out, $prefix === '' ? 'no config file' : 'dev mode'); } } @@ -281,7 +281,7 @@ public function devModeInstancesAreConfigured(): void copy(__DIR__ . '/../../noneDB.php', "$dir/noneDB.php"); file_put_contents("$dir/run.php", 'insert("t", ["a" => 1]); echo count($a->find("t", 0));'); - $out = trim(shell_exec('cd ' . escapeshellarg(sys_get_temp_dir()) . ' && env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg("$dir/run.php") . ' 2>&1')); + $out = trim(shell_exec(self::cd(sys_get_temp_dir()) . 'env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg("$dir/run.php") . ' 2>&1')); $this->assertSame('1', $out); $databases = glob("$dir/db/*.ndb4"); $this->assertCount(1, $databases, 'data next to noneDB.php, not in the working directory'); diff --git a/tests/Integration/SpecOperationsTest.php b/tests/Integration/SpecOperationsTest.php index ac74f8a..e1e8218 100644 --- a/tests/Integration/SpecOperationsTest.php +++ b/tests/Integration/SpecOperationsTest.php @@ -196,7 +196,8 @@ private function directorySnapshot(?string $dir = null): array $files[substr($path . $entry, strlen($root)) . '/'] = 'dir'; $walk($path . $entry . '/'); } else { - $files[substr($path . $entry, strlen($root))] = sha1_file($path . $entry); + // On Windows a lock keeps every other handle from reading the file + $files[substr($path . $entry, strlen($root))] = PHP_OS_FAMILY === 'Windows' && $entry === 'LOCK' ? filesize($path . $entry) : sha1_file($path . $entry); } } }; @@ -280,7 +281,7 @@ public function defaultsAreTheDocumentedOnes(): void $out["durability"] = $db->getDurability()["mode"]; $out["cacheMemoryLimit"] = noneDB\Tests\Support\Testing::get("cacheMemoryLimit"); echo json_encode($out);'); - $out = trim((string)shell_exec('cd ' . escapeshellarg($dir) . ' && env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1')); + $out = trim((string)shell_exec(self::cd($dir) . 'env -u NONEDB_DEV_MODE ' . escapeshellarg(PHP_BINARY) . ' run.php 2>&1')); $this->assertSame([ 'autoCreateDB' => true, // "autoCreateDB | bool | true" 'shardingEnabled' => true, // "shardingEnabled | bool | true" @@ -421,7 +422,7 @@ public function configFileNextToTheEntryScriptIsUsedOncePerProcess(): void $this->assertTrue(\noneDB::configExists()); $db = new \noneDB(); - $this->assertSame($app . '/data/', $db->__debugInfo()['dbDir'], 'relative to the .nonedb file'); + $this->assertSame(self::resolvedDir($app . '/data/'), $db->__debugInfo()['dbDir'], 'relative to the .nonedb file'); $this->assertSame('***', $db->__debugInfo()['secretKey']); $this->assertFalse($db->__debugInfo()['devMode']); $this->assertSame(7, $db->getShardSize()); @@ -435,7 +436,7 @@ public function configFileNextToTheEntryScriptIsUsedOncePerProcess(): void $merged = new \noneDB(['shardSize' => 9, 'bufferEnabled' => false]); $this->assertSame(9, $merged->getShardSize()); $this->assertFalse($merged->isBufferingEnabled()); - $this->assertSame($app . '/data/', $merged->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir($app . '/data/'), $merged->__debugInfo()['dbDir']); $this->assertSame(2.0, $this->prop($merged, 'lockTimeout')); $this->assertSame([1], array_column($merged->find('fromfile', 0), 'v'), 'same key and directory as the file'); @@ -445,7 +446,7 @@ public function configFileNextToTheEntryScriptIsUsedOncePerProcess(): void \noneDB::clearConfigCache(); $reloaded = new \noneDB(); $this->assertSame(11, $reloaded->getShardSize()); - $this->assertSame($app . '/other/', $reloaded->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir($app . '/other/'), $reloaded->__debugInfo()['dbDir']); } /** @test a .nonedb that isn't a JSON object can't configure anything (unless the array gives secretKey and dbDir) */ @@ -535,7 +536,7 @@ public function relativeDbDirOfAnArrayFollowsTheEntryScript(): void chdir($root . '/cwd'); foreach (['store' => $root . '/app/store/', './store/' => $root . '/app/store/', '../shared' => $root . '/shared/'] as $dbDir => $expected) { $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => $dbDir]); - $this->assertSame($expected, $db->__debugInfo()['dbDir'], $dbDir); + $this->assertSame(self::resolvedDir($expected), $db->__debugInfo()['dbDir'], $dbDir); } $db = new \noneDB(['secretKey' => self::SECRET, 'dbDir' => '../shared']); $db->insert('rel', ['v' => 1]); @@ -548,15 +549,15 @@ public function relativeDbDirOfAnArrayFollowsTheEntryScript(): void // Entry script that doesn't exist (any more): its directory still counts; no entry script: the working directory $this->entryScriptIn($root . '/app', 'gone.php', false); - $this->assertSame($root . '/app/x/', (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir($root . '/app/x/'), (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); $_SERVER['SCRIPT_FILENAME'] = ''; - $this->assertSame(getcwd() . '/x/', (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir(getcwd() . '/x/'), (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'x']))->__debugInfo()['dbDir']); // Many different entry scripts in one process (a long-running worker): each resolves on its own for ($i = 0; $i < 20; $i++) { mkdir($root . "/w$i"); $this->entryScriptIn($root . "/w$i"); - $this->assertSame($root . "/w$i/db/", (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'db']))->__debugInfo()['dbDir']); + $this->assertSame(self::resolvedDir($root . "/w$i/db/"), (new \noneDB(['secretKey' => self::SECRET, 'dbDir' => 'db']))->__debugInfo()['dbDir']); } } @@ -632,7 +633,7 @@ public function devModeFillsInWhatIsMissing(): void $this->removeDirectoryTree($lib . '/db'); $this->removeDirectoryTree($own); file_put_contents($lib . '/run.php', strtr($script, ['PREFIX' => $prefix, 'OWNDIR' => var_export($own . '/', true)])); - $out = trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + $out = trim((string)shell_exec(self::cd($cwd) . 'env -u NONEDB_DEV_MODE ' . $env . ' ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($lib . '/run.php') . ' 2>&1')); $this->assertSame([ 'none' => 'dev ' . $lib . '/db/', @@ -654,7 +655,7 @@ public function devModeFillsInWhatIsMissing(): void $body = $label === 'setDevMode(false)' ? 'noneDB::setDevMode(false); ' : ''; file_put_contents($lib . '/run.php', 'getMessage(), "\n"); }'); - $out = trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + $out = trim((string)shell_exec(self::cd($cwd) . 'env -u NONEDB_DEV_MODE ' . $env . ' ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($lib . '/run.php') . ' 2>&1')); $this->assertSame('RuntimeException: noneDB: Configuration file not found!', $out, $label); } @@ -678,7 +679,7 @@ public function configSearchOrderAndComposerProjectRoot(): void try { $db = new noneDB(); echo $db->__debugInfo()["dbDir"], " ", $db->__debugInfo()["devMode"] ? "dev" : "file"; } catch (Throwable $e) { echo get_class($e); }'); $run = function (string $env = '') use ($proj, $cwd) { - return trim((string)shell_exec('cd ' . escapeshellarg($cwd) . ' && env -u NONEDB_DEV_MODE ' . $env . ' ' + return trim((string)shell_exec(self::cd($cwd) . 'env -u NONEDB_DEV_MODE ' . $env . ' ' . escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($proj . '/public/index.php') . ' 2>&1')); }; $write = function (string $dir, string $dbDir) { @@ -686,12 +687,12 @@ public function configSearchOrderAndComposerProjectRoot(): void }; $this->assertSame('RuntimeException', $run(), 'no file anywhere'); - $this->assertSame($proj . '/db/ dev', $run('DEV=1'), 'dev mode: db/ next to vendor/'); + $this->assertSame(self::resolvedDir($proj . '/db/') . ' dev', $run('DEV=1'), 'dev mode: db/ next to vendor/'); $write($cwd, 'from-cwd'); // 4. the working directory - $this->assertSame($cwd . '/from-cwd/ file', $run()); + $this->assertSame(self::resolvedDir($cwd . '/from-cwd/') . ' file', $run()); $write($proj, 'from-root'); // 3. the Composer project root - $this->assertSame($proj . '/from-root/ file', $run()); + $this->assertSame(self::resolvedDir($proj . '/from-root/') . ' file', $run()); $write($vendor, 'from-lib'); // 2. noneDB.php's directory $this->assertSame($vendor . '/from-lib/ file', $run()); $write($proj . '/public', 'from-script'); // 1. the entry script's directory @@ -702,7 +703,7 @@ public function configSearchOrderAndComposerProjectRoot(): void /** @test the template is the .nonedb.example next to noneDB.php, null when it isn't there (docs "Configuration Helpers") */ public function configTemplateIsTheExampleNextToNoneDb(): void { - $this->assertSame(realpath(dirname(NONEDB_TEST_LIBRARY) . '/.nonedb.example'), \noneDB::getConfigTemplate()); // next to the library that runs + $this->assertSame(realpath(dirname(NONEDB_TEST_LIBRARY) . '/.nonedb.example'), realpath(\noneDB::getConfigTemplate())); // next to the library that runs $lib = $this->scratch('notemplate'); copy(__DIR__ . '/../../noneDB.php', $lib . '/noneDB.php'); $out = trim((string)shell_exec(escapeshellarg(PHP_BINARY) . ' -r ' . escapeshellarg('require ' . var_export($lib . '/noneDB.php', true) @@ -941,7 +942,7 @@ public function uncreatableDbDirFailsWrites(): void $this->fail('the directory can not exist'); } catch (\RuntimeException $e) { $this->assertStringContainsString('cannot create database directory', $e->getMessage()); - $this->assertStringContainsString($dir, $e->getMessage()); + $this->assertStringContainsString(self::resolvedDir($dir), $e->getMessage()); } $this->assertSame('a file, not a directory', file_get_contents($blocker)); } @@ -1634,6 +1635,7 @@ public function durabilityReportFieldsInPerformanceMode(): void */ public function lockStateRecordIsSyncedWhenItProtectsData(): void { + $this->needsDirectorySyncs(); $this->setStatic('osFamily', 'Linux'); // durable mode's rules, on every platform $synced = []; $this->setStatic('syncFn', function ($handle, string $kind) use (&$synced) { diff --git a/tests/Integration/SpecStorageTest.php b/tests/Integration/SpecStorageTest.php index a221ecd..6dfdb81 100644 --- a/tests/Integration/SpecStorageTest.php +++ b/tests/Integration/SpecStorageTest.php @@ -946,6 +946,7 @@ public function anInterruptedBufferFlushIsReplayedExactlyOnce(): void */ public function aBufferedInsertWhoseSyncFailsIsRefusedAndUndone(): void { + $this->needsDirectorySyncs(); $this->setStatic('osFamily', 'Linux'); // durable mode's rules, on every platform $db = $this->db(['bufferEnabled' => true, 'shardingEnabled' => false, 'durability' => 'durable']); $db->insert('bf', ['v' => 'a']); diff --git a/tests/Migration/MigrationProcessTest.php b/tests/Migration/MigrationProcessTest.php index 261bf6d..4ca6e38 100644 --- a/tests/Migration/MigrationProcessTest.php +++ b/tests/Migration/MigrationProcessTest.php @@ -347,7 +347,7 @@ public function anInterruptedRunNeedsItsOwnKeys(): void public function aSettingNoneDB4DoesNotKnowStopsTheRunFirst(): void { $dir = $this->stage(self::FIXTURE); - $cwd = sys_get_temp_dir() . '/nonedb_migrate_cwd_' . bin2hex(random_bytes(4)); + $cwd = realpath(sys_get_temp_dir()) . '/nonedb_migrate_cwd_' . bin2hex(random_bytes(4)); mkdir($cwd); try { file_put_contents("$cwd/.nonedb", json_encode(['secretKey' => self::TARGET, 'dbDir' => './db/', 'autoMigrate' => true])); diff --git a/tests/Support/Platform.php b/tests/Support/Platform.php new file mode 100644 index 0000000..e1bb921 --- /dev/null +++ b/tests/Support/Platform.php @@ -0,0 +1,22 @@ +markTestSkipped('durable mode needs directory syncs'); + } + } + /** * Remove all files from test database directory */ From d50f7e25b41fe2bffa05cd8fbf2d9b2e1eb6ef60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 20:26:02 +0300 Subject: [PATCH 4/6] Windows: the backup directory check, tests of what Windows can't do The migration tool compared paths with / only and took D:\... for a relative path, so on Windows a --backup-dir inside --dir wasn't refused. It compares them as the system does now. A file that is open can't be replaced there, nor can a new LOCK appear while a process has one: those tests run where that can happen. Under a lock a test reads the state record through the lock's handle. --- migrate/Cli.php | 14 ++++++++++---- noneDB-migrate.php | 14 ++++++++++---- tests/Concurrency/LockEpochTest.php | 3 +++ tests/Concurrency/LockStateRegressionTest.php | 14 +++++++++++--- tests/Crash/RollbackScopeTest.php | 3 ++- tests/Integration/HardeningRegressionTest.php | 1 + tests/Integration/SpecOperationsTest.php | 2 +- tests/Migration/MigrationProcessTest.php | 1 + 8 files changed, 39 insertions(+), 13 deletions(-) diff --git a/migrate/Cli.php b/migrate/Cli.php index 0614c0f..d39862a 100644 --- a/migrate/Cli.php +++ b/migrate/Cli.php @@ -111,19 +111,25 @@ private static function dispatch(array $o): array { } $real = realpath($dir); $backup = rtrim($o['backup-dir'], '/') . '/'; - if($real === false || strncmp(self::absolute($backup), $real . '/', strlen($real) + 1) === 0){ + if($real === false || strncmp(self::absolute($backup), self::comparable($real) . '/', strlen($real) + 1) === 0){ throw new Stop('FAILED', '--backup-dir must be outside --dir'); } return ['migrated' => Migration::run($dir, $source, $target, $options + ['backupDir' => $backup])]; } - /** A path made absolute without requiring it to exist */ + /** A path made absolute without requiring it to exist, to compare */ private static function absolute(string $path): string { $real = realpath($path); if($real !== false){ - return $real . '/'; + return self::comparable($real) . '/'; } - return ($path[0] === '/' ? '' : getcwd() . '/') . $path; + $path = self::comparable($path); + return preg_match('~^([a-z]:)?/~i', $path) === 1 ? $path : self::comparable((string)getcwd()) . '/' . $path; + } + + /** A path as paths of the system compare: on Windows / and \ are one and case doesn't count */ + private static function comparable(string $path): string { + return DIRECTORY_SEPARATOR === '\\' ? strtolower(str_replace('\\', '/', $path)) : $path; } private static function output(bool $json, array $result): void { diff --git a/noneDB-migrate.php b/noneDB-migrate.php index 8e18ccf..48c0cf7 100644 --- a/noneDB-migrate.php +++ b/noneDB-migrate.php @@ -2248,19 +2248,25 @@ private static function dispatch(array $o): array { } $real = realpath($dir); $backup = rtrim($o['backup-dir'], '/') . '/'; - if($real === false || strncmp(self::absolute($backup), $real . '/', strlen($real) + 1) === 0){ + if($real === false || strncmp(self::absolute($backup), self::comparable($real) . '/', strlen($real) + 1) === 0){ throw new Stop('FAILED', '--backup-dir must be outside --dir'); } return ['migrated' => Migration::run($dir, $source, $target, $options + ['backupDir' => $backup])]; } - /** A path made absolute without requiring it to exist */ + /** A path made absolute without requiring it to exist, to compare */ private static function absolute(string $path): string { $real = realpath($path); if($real !== false){ - return $real . '/'; + return self::comparable($real) . '/'; } - return ($path[0] === '/' ? '' : getcwd() . '/') . $path; + $path = self::comparable($path); + return preg_match('~^([a-z]:)?/~i', $path) === 1 ? $path : self::comparable((string)getcwd()) . '/' . $path; + } + + /** A path as paths of the system compare: on Windows / and \ are one and case doesn't count */ + private static function comparable(string $path): string { + return DIRECTORY_SEPARATOR === '\\' ? strtolower(str_replace('\\', '/', $path)) : $path; } private static function output(bool $json, array $result): void { diff --git a/tests/Concurrency/LockEpochTest.php b/tests/Concurrency/LockEpochTest.php index 503ea6c..158e029 100644 --- a/tests/Concurrency/LockEpochTest.php +++ b/tests/Concurrency/LockEpochTest.php @@ -70,6 +70,9 @@ public function writesWithoutTheExclusiveLockAreRefused(): void /** @test */ public function aRecreatedLockFileNeverValidatesAnOldCache(): void { + if (PHP_OS_FAMILY === 'Windows') { + $this->markTestSkipped('a file that is open keeps its name until it is closed: no new LOCK while this process has one'); + } $db = $this->db(['shardingEnabled' => false]); $db->insert('epo', [['n' => 0], ['n' => 1]]); $db->insert('epo', ['n' => 2]); // a generation the child can reach again however a new lock file starts diff --git a/tests/Concurrency/LockStateRegressionTest.php b/tests/Concurrency/LockStateRegressionTest.php index dd0bcc4..c4cc233 100644 --- a/tests/Concurrency/LockStateRegressionTest.php +++ b/tests/Concurrency/LockStateRegressionTest.php @@ -50,7 +50,13 @@ public function stateFlagsFollowTheDatabase(): void private function rawRecord(string $dbName): string { clearstatcache(); - return (string)file_get_contents($this->dbPath($dbName, 'lock')); + $path = $this->dbPath($dbName, 'lock'); + $held = \noneDB\Tests\Support\Testing::get('lockHandles')[$path]['fp'] ?? null; + if (PHP_OS_FAMILY === 'Windows' && is_resource($held)) { + rewind($held); // a lock keeps every other handle from reading the file + return (string)fread($held, 8192); + } + return (string)file_get_contents($path); } /** Name of the database (of $names) a file belongs to */ @@ -140,8 +146,7 @@ public function aJournalTooLongForTheRecordBecomesJ(): void $states = []; $this->setFaultHook(function ($point) use (&$states) { if ($point === 'append:after') { - clearstatcache(); - $states[] = (string)file_get_contents($this->dbPath('many', 'lock')); + $states[] = $this->rawRecord('many'); } }); // keys 2-121: shards 1-60; "j=s1,s2,...,s60" alone is 232 bytes @@ -250,6 +255,9 @@ public function generationMovesOncePerWriteAndNeverOnReads(): void /** @test the path is locked, not a deleted file a kept handle still points to */ public function replacedLockFileIsNotTrusted(): void { + if (PHP_OS_FAMILY === 'Windows') { + $this->markTestSkipped('a file that is open keeps its name until it is closed: no new LOCK while this process has one'); + } $db = $this->db(['lockTimeout' => 1]); $db->insert('rl', ['a' => 1]); // the lock handle stays open in this process $lock = $this->dbPath('rl', 'lock'); diff --git a/tests/Crash/RollbackScopeTest.php b/tests/Crash/RollbackScopeTest.php index 7f7e6fe..acc5e38 100644 --- a/tests/Crash/RollbackScopeTest.php +++ b/tests/Crash/RollbackScopeTest.php @@ -548,7 +548,8 @@ public function failedDeferredFlushAndSwitchToShardsAreLoggedAndRetried(): void */ public function rollbackOfAFileReplacedOrRemovedMeanwhileFails(): void { - foreach (['replaced', 'removed'] as $what) { + // Windows replaces no file that is open + foreach (PHP_OS_FAMILY === 'Windows' ? ['removed'] : ['replaced', 'removed'] as $what) { $this->cleanTestDirectory(); $db = $this->db(['shardingEnabled' => false]); $db->insert('gone', ['v' => 'base']); diff --git a/tests/Integration/HardeningRegressionTest.php b/tests/Integration/HardeningRegressionTest.php index e2e93a7..8ac253f 100644 --- a/tests/Integration/HardeningRegressionTest.php +++ b/tests/Integration/HardeningRegressionTest.php @@ -348,6 +348,7 @@ public function dbDirWithGlobCharactersWorks(): void $this->assertTrue($db->dropDB('g')['success']); $this->assertSame([], $this->filesOf('g', $dir)); } finally { + \noneDB::clearStaticCache(); // its lock handles: Windows removes no directory with an open file in it $remove = function (string $path) use (&$remove) { if (is_dir($path) && !is_link($path)) { foreach (scandir($path) as $entry) { diff --git a/tests/Integration/SpecOperationsTest.php b/tests/Integration/SpecOperationsTest.php index e1e8218..fee7734 100644 --- a/tests/Integration/SpecOperationsTest.php +++ b/tests/Integration/SpecOperationsTest.php @@ -687,7 +687,7 @@ public function configSearchOrderAndComposerProjectRoot(): void }; $this->assertSame('RuntimeException', $run(), 'no file anywhere'); - $this->assertSame(self::resolvedDir($proj . '/db/') . ' dev', $run('DEV=1'), 'dev mode: db/ next to vendor/'); + $this->assertSame($proj . '/db/ dev', $run('DEV=1'), 'dev mode: db/ next to vendor/'); $write($cwd, 'from-cwd'); // 4. the working directory $this->assertSame(self::resolvedDir($cwd . '/from-cwd/') . ' file', $run()); diff --git a/tests/Migration/MigrationProcessTest.php b/tests/Migration/MigrationProcessTest.php index 4ca6e38..b34d622 100644 --- a/tests/Migration/MigrationProcessTest.php +++ b/tests/Migration/MigrationProcessTest.php @@ -550,6 +550,7 @@ public function aRestoreAfterARunStoppedBeforeItsJournalLeavesNoStoreLock(): voi $this->assertSame(0, $status, "$stop, run again: $stdout"); $this->assertMigrated($dir, "$stop, run again"); $run = $out['migrated']['run']; + \noneDB::clearStaticCache(); // assertMigrated() read it here: Windows moves no directory with an open file in it [$status, $out, $stdout] = $this->tool(["--dir=$dir", '--backup-dir=' . $this->testDbDir . 'backup/', "--restore=$run"], $this->key()); $this->assertSame(0, $status, "$stop, restore: $stdout"); $this->assertSame($before, $this->snapshot($dir), "$stop: restored byte for byte, STORE.LOCK gone"); From 24b521f35c5ffccc639ee7dbc095c539be4dd187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 20:32:53 +0300 Subject: [PATCH 5/6] Windows: the last two test paths Under a lock a test finds the handle in the lock registry, where a held lock is kept. The two paths of the config search order that were still compared with / go through resolvedDir(). --- tests/Concurrency/LockStateRegressionTest.php | 2 +- tests/Integration/SpecOperationsTest.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Concurrency/LockStateRegressionTest.php b/tests/Concurrency/LockStateRegressionTest.php index c4cc233..ccd05b2 100644 --- a/tests/Concurrency/LockStateRegressionTest.php +++ b/tests/Concurrency/LockStateRegressionTest.php @@ -51,7 +51,7 @@ private function rawRecord(string $dbName): string { clearstatcache(); $path = $this->dbPath($dbName, 'lock'); - $held = \noneDB\Tests\Support\Testing::get('lockHandles')[$path]['fp'] ?? null; + $held = \noneDB\Tests\Support\Testing::get('lockRegistry')[$path]['fp'] ?? null; if (PHP_OS_FAMILY === 'Windows' && is_resource($held)) { rewind($held); // a lock keeps every other handle from reading the file return (string)fread($held, 8192); diff --git a/tests/Integration/SpecOperationsTest.php b/tests/Integration/SpecOperationsTest.php index fee7734..5f40532 100644 --- a/tests/Integration/SpecOperationsTest.php +++ b/tests/Integration/SpecOperationsTest.php @@ -694,10 +694,10 @@ public function configSearchOrderAndComposerProjectRoot(): void $write($proj, 'from-root'); // 3. the Composer project root $this->assertSame(self::resolvedDir($proj . '/from-root/') . ' file', $run()); $write($vendor, 'from-lib'); // 2. noneDB.php's directory - $this->assertSame($vendor . '/from-lib/ file', $run()); + $this->assertSame(self::resolvedDir($vendor . '/from-lib/') . ' file', $run()); $write($proj . '/public', 'from-script'); // 1. the entry script's directory - $this->assertSame($proj . '/public/from-script/ file', $run()); - $this->assertSame($proj . '/public/from-script/ file', $run('DEV=1'), 'a file found: dev mode is not used'); + $this->assertSame(self::resolvedDir($proj . '/public/from-script/') . ' file', $run()); + $this->assertSame(self::resolvedDir($proj . '/public/from-script/') . ' file', $run('DEV=1'), 'a file found: dev mode is not used'); } /** @test the template is the .nonedb.example next to noneDB.php, null when it isn't there (docs "Configuration Helpers") */ From 670cc2748f63ff541ed81273922d0dab530b4f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orhan=20AYDO=C4=9EDU?= Date: Sun, 1 Nov 2026 22:11:15 +0300 Subject: [PATCH 6/6] Windows is a required job, v4.0.0 dated --- .github/workflows/tests.yml | 3 +-- CHANGES.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f0a884..96143d5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -126,11 +126,10 @@ jobs: run: rm -rf tests/test_db examples/db windows: - # Windows (file locking, rename retries): informational until it has been seen green + # Windows (file locking, rename retries) name: PHP ${{ matrix.php-version }} (Windows) needs: fixtures runs-on: windows-latest - continue-on-error: true strategy: fail-fast: false diff --git a/CHANGES.md b/CHANGES.md index 964e6fd..683d8ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # noneDB Changelog -## v4.0.0 (unreleased) +## v4.0.0 (2026-11-01) ### A new engine and a new storage format