From 63fd11e642c3bacd050c1447adbd9a29c6f15787 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Sun, 4 Oct 2026 21:34:32 -0700 Subject: [PATCH] Expose semantic models through standard database catalogs --- .../workflows/duckdb-extension-release.yml | 80 ++++- sidemantic-duckdb/CMakeLists.txt | 4 +- sidemantic-duckdb/README.md | 25 +- .../src/include/sidemantic_catalog.hpp | 4 + .../src/sidemantic_discovery.cpp | 325 ++++++++++++++++++ .../src/sidemantic_extension.cpp | 1 + sidemantic-duckdb/src/sidemantic_parser.cpp | 70 +++- sidemantic-duckdb/test/test_catalog_host.py | 179 +++++++++- .../test/test_duckflight_host.py | 136 +++++++- .../test/test_release_workflow.py | 85 +++++ sidemantic-rs/include/sidemantic.h | 2 + sidemantic-rs/src/ffi/catalog/api.rs | 51 ++- tests/db/test_adbc_adapter.py | 2 +- 13 files changed, 910 insertions(+), 54 deletions(-) create mode 100644 sidemantic-duckdb/src/sidemantic_discovery.cpp diff --git a/.github/workflows/duckdb-extension-release.yml b/.github/workflows/duckdb-extension-release.yml index 9e0d051e..6a6242d7 100644 --- a/.github/workflows/duckdb-extension-release.yml +++ b/.github/workflows/duckdb-extension-release.yml @@ -9,7 +9,7 @@ on: default: "v1.5.6" type: string create_github_release: - description: "Create or update a GitHub release with extension artifacts" + description: "Publish a new release; the matching extension version tag must already point to this commit" required: false default: false type: boolean @@ -27,15 +27,41 @@ permissions: env: GEN: ninja + CMAKE_BUILD_PARALLEL_LEVEL: 2 + CARGO_BUILD_JOBS: 2 + MACOSX_DEPLOYMENT_TARGET: "11.0" jobs: build: - name: Build source extension package - runs-on: ubuntu-latest + name: Build ${{ matrix.platform }} extension package + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux_amd64 + runner: ubuntu-24.04 + container: quay.io/pypa/manylinux_2_28_x86_64:latest + - platform: linux_arm64 + runner: ubuntu-24.04-arm + container: quay.io/pypa/manylinux_2_28_aarch64:latest + - platform: osx_amd64 + runner: macos-15-intel + - platform: osx_arm64 + runner: macos-15 steps: - uses: actions/checkout@v4 + - name: Install Linux build dependencies + if: runner.os == 'Linux' + run: dnf install -y cmake ninja-build gcc gcc-c++ git make + + - name: Install macOS build dependencies + if: runner.os == 'macOS' + run: command -v ninja >/dev/null || brew install ninja + - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -72,11 +98,10 @@ jobs: DUCKDB_VERSION: ${{ steps.duckdb.outputs.version }} run: make deps DUCKDB_VERSION="$DUCKDB_VERSION" - - name: Install build dependencies - run: sudo apt-get update && sudo apt-get install -y ninja-build - - name: Build extension working-directory: sidemantic-duckdb + env: + DUCKDB_PLATFORM: ${{ matrix.platform }} run: make - name: Run extension tests @@ -87,16 +112,24 @@ jobs: working-directory: sidemantic-duckdb env: DUCKDB_VERSION: ${{ steps.duckdb.outputs.version }} + PLATFORM: ${{ matrix.platform }} run: | mkdir -p dist DUCKDB_VERSION="${DUCKDB_VERSION#v}" - cp build/release/extension/sidemantic/sidemantic.duckdb_extension "dist/sidemantic-duckdb-${DUCKDB_VERSION}-linux-amd64.duckdb_extension" + ASSET="sidemantic-duckdb-${DUCKDB_VERSION}-${PLATFORM}.duckdb_extension" + cp build/release/extension/sidemantic/sidemantic.duckdb_extension "dist/$ASSET" + cd dist + if command -v sha256sum >/dev/null; then + sha256sum "$ASSET" > "$ASSET.sha256" + else + shasum -a 256 "$ASSET" > "$ASSET.sha256" + fi - name: Upload extension artifact uses: actions/upload-artifact@v4 with: - name: sidemantic-duckdb-extension-linux-amd64 - path: sidemantic-duckdb/dist/*.duckdb_extension + name: sidemantic-duckdb-extension-${{ matrix.platform }} + path: sidemantic-duckdb/dist/* if-no-files-found: error - name: Save completed build caches @@ -119,6 +152,7 @@ jobs: - name: Download extension artifact uses: actions/download-artifact@v4 with: + pattern: sidemantic-duckdb-extension-* path: dist merge-multiple: true @@ -135,15 +169,33 @@ jobs: TAG="sidemantic-duckdb-v${VERSION}" fi + if [[ ! "$TAG" =~ ^sidemantic-duckdb-v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid extension release tag" >&2 + exit 1 + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" - - name: Create or update GitHub release + - name: Verify and publish immutable artifacts env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ steps.release.outputs.tag }} + RELEASE_COMMIT: ${{ github.sha }} run: | - if gh release view "$TAG" >/dev/null 2>&1; then - gh release upload "$TAG" dist/* --clobber - else - gh release create "$TAG" dist/* --title "$TAG" --generate-notes + set -euo pipefail + git fetch --no-tags origin "refs/tags/$TAG:refs/tags/$TAG" + if [ "$(git rev-parse "$TAG^{commit}")" != "$RELEASE_COMMIT" ]; then + echo "Release tag does not identify the built commit" >&2 + exit 1 fi + cd dist + for PLATFORM in linux_amd64 linux_arm64 osx_amd64 osx_arm64; do + ASSET="sidemantic-duckdb-1.5.6-${PLATFORM}.duckdb_extension" + test -s "$ASSET" + sha256sum --check "$ASSET.sha256" + done + cat *.sha256 > SHA256SUMS + cp ../LICENSE . + # Creation fails if the tag already has a release; never overwrite published bytes. + gh release create "$TAG" *.duckdb_extension SHA256SUMS LICENSE \ + --verify-tag --target "$RELEASE_COMMIT" --title "$TAG" --generate-notes diff --git a/sidemantic-duckdb/CMakeLists.txt b/sidemantic-duckdb/CMakeLists.txt index e521acae..07b1cf91 100644 --- a/sidemantic-duckdb/CMakeLists.txt +++ b/sidemantic-duckdb/CMakeLists.txt @@ -118,7 +118,7 @@ if(NOT CARGO_EXECUTABLE) message(FATAL_ERROR "cargo is required to build the sidemantic DuckDB extension") endif() -set(SIDEMANTIC_CARGO_BUILD_ARGS build --manifest-path "${SIDEMANTIC_RS_DIR}/Cargo.toml" --lib) +set(SIDEMANTIC_CARGO_BUILD_ARGS build --locked --manifest-path "${SIDEMANTIC_RS_DIR}/Cargo.toml" --lib) if(SIDEMANTIC_CARGO_TARGET) list(APPEND SIDEMANTIC_CARGO_BUILD_ARGS --target "${SIDEMANTIC_CARGO_TARGET}") endif() @@ -145,7 +145,7 @@ add_custom_target(sidemantic_rust_staticlib DEPENDS "${SIDEMANTIC_LIB}") # Include Rust library headers include_directories(${SIDEMANTIC_INCLUDE}) -set(EXTENSION_SOURCES src/sidemantic_extension.cpp src/sidemantic_parser.cpp src/sidemantic_grammar.cpp src/sidemantic_catalog.cpp src/sidemantic_api.cpp) +set(EXTENSION_SOURCES src/sidemantic_extension.cpp src/sidemantic_parser.cpp src/sidemantic_grammar.cpp src/sidemantic_catalog.cpp src/sidemantic_api.cpp src/sidemantic_discovery.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/sidemantic-duckdb/README.md b/sidemantic-duckdb/README.md index b1ba7afb..7f95e9ee 100644 --- a/sidemantic-duckdb/README.md +++ b/sidemantic-duckdb/README.md @@ -18,6 +18,12 @@ A DuckDB extension that adds a SQL-first semantic layer. Define metrics and dime Current builds are loaded from a local build or GitHub release artifact: +The release workflow builds DuckDB 1.5.6 packages for Linux and macOS on AMD64 and +ARM64, with a `SHA256SUMS` manifest. Linux builds use a glibc 2.28 baseline. Choose +the matching version and platform, verify the checksum, and rename the downloaded +file to `sidemantic.duckdb_extension` before loading it. Published assets are +immutable; corrections receive a new release tag. + Start DuckDB with unsigned-extension loading enabled because these artifacts are not signed yet: ```bash @@ -114,6 +120,18 @@ use `EXPLAIN SELECT` to validate a complete combination of metrics and dimension `SHOW SEMANTIC METRICS`, and the corresponding forms for other kinds, are also accepted. +Standard SQL clients can also discover loaded models as virtual views in the +current database's `semantic` schema through `information_schema.tables`, +`information_schema.columns`, and the corresponding `pg_catalog` relations. +Query them with ordinary SQL, for example +`select status, revenue from semantic.orders_model`. Dimension and metric column +types come from binding their compiled expressions against the current database. +If a source cannot bind, its fields remain visible with `NULL` data types and +`duckdb_views().is_bound = false`. An existing physical `semantic.orders_model` +table or view takes precedence. Discovery observes transactions and creates no +persistent schema or views; `SET schema = 'semantic'` requires an actual schema +created by the user. + Qualified definitions work independently of the connection's active model: ```sql @@ -547,9 +565,8 @@ rows, including equal-valued metrics on distinct customers. - DuckDB 1.5.6 uses the compatibility frontend. Native PEG support is tested against the pinned 2.x commit, not an arbitrary future 2.x build. Rust still parses model properties and compiles semantic queries. -- Distribution currently targets unsigned Linux amd64 release packages. Signed - community installation and a published multi-platform binary matrix are not - provided by this repository. +- The release workflow packages unsigned Linux and macOS builds on AMD64 and ARM64. + Signed community installation is not yet provided by this repository. ## Utility Functions @@ -603,7 +620,7 @@ CI also builds `make deps DUCKDB_VERSION=v2.0-cyanoptera`, pinned in the Makefil commit `80e17fc252edd6d9e9b090ae00a1100daef4876a`. Run its tests with `SIDEMANTIC_NATIVE_PEG=1 make test`. Use separate build directories/checkouts when switching DuckDB versions. CMake probes host APIs rather than assuming C++ ABI -compatibility across releases. Release packaging targets unsigned Linux amd64 +compatibility across releases. Release packaging targets unsigned Linux and macOS builds for 1.5.6; development-version CI does not publish stable artifacts. ## Architecture diff --git a/sidemantic-duckdb/src/include/sidemantic_catalog.hpp b/sidemantic-duckdb/src/include/sidemantic_catalog.hpp index 278f65fe..2ae25023 100644 --- a/sidemantic-duckdb/src/include/sidemantic_catalog.hpp +++ b/sidemantic-duckdb/src/include/sidemantic_catalog.hpp @@ -14,6 +14,10 @@ struct SidemanticCatalogSnapshot { // sidecars are read only when no native snapshot exists and are never modified. SidemanticCatalogSnapshot ReadSidemanticCatalog(ClientContext &context); +// Virtual semantic relations are exposed through the host's standard catalogs. +bool SidemanticPhysicalRelationExists(ClientContext &context, const string &model); +void RegisterSidemanticDiscovery(DatabaseInstance &db); + void RegisterSidemanticCatalogRead(ClientContext &context, StatementProperties &properties); void RegisterSidemanticCatalogWrite(ClientContext &context, StatementProperties &properties); diff --git a/sidemantic-duckdb/src/sidemantic_discovery.cpp b/sidemantic-duckdb/src/sidemantic_discovery.cpp new file mode 100644 index 00000000..3a13d641 --- /dev/null +++ b/sidemantic-duckdb/src/sidemantic_discovery.cpp @@ -0,0 +1,325 @@ +#include "sidemantic_catalog.hpp" +#include "sidemantic_compat.hpp" +#include "sidemantic.h" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/common/error_data.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/database_manager.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_set_operation.hpp" +#include "duckdb/planner/planner_extension.hpp" +#include + +namespace duckdb { +namespace { + +constexpr const char *SEMANTIC_SCHEMA = "semantic"; + +const string &ScanName(const LogicalGet &get) { +#if SIDEMANTIC_NEW_IDENTIFIER_API + return SidemanticName(get.function.GetName()); +#else + return get.function.name; +#endif +} + +struct CatalogEntriesOwner { + explicit CatalogEntriesOwner(const string &snapshot) + : result(sidemantic_snapshot_catalog(snapshot.c_str(), "", "", nullptr)) { + if (result.error) { + string error(result.error); + sidemantic_free_catalog_entries(result); + throw InvalidInputException("Sidemantic catalog: %s", error); + } + } + ~CatalogEntriesOwner() { sidemantic_free_catalog_entries(result); } + SidemanticCatalogEntries result; +}; + +string QuoteName(const string &name) { + return "\"" + StringUtil::Replace(name, "\"", "\"\"") + "\""; +} + +// Synthetic OIDs remain within PostgreSQL's positive int32 range. Hash only +// stable names, never process-local DuckDB OIDs or std::hash implementation data. +int64_t StableOid(const string &key, unordered_set &used) { + uint32_t hash = 2166136261U; + for (unsigned char byte : StringUtil::Lower(key)) hash = (hash ^ byte) * 16777619U; + int64_t oid = 0x40000000U | (hash & 0x3fffffffU); + while (used.count(oid)) oid = 0x40000000U | ((oid + 1) & 0x3fffffffU); + used.insert(oid); + return oid; +} + +struct SemanticColumn { + string name; + idx_t ordinal = 0; + Value comment; + LogicalType type = LogicalTypeId::INVALID; +}; + +struct SemanticRelation { + string name; + Value comment; + int64_t oid = 0; + vector columns; +}; + +struct SemanticMetadata { + string database; + int64_t database_oid = 0; + int64_t schema_oid = 0; + bool physical_schema = false; + vector relations; +}; + +LogicalType BindFieldType(ClientContext &context, const string &snapshot, const string &model, const string &field) { + auto sql = "SELECT " + QuoteName(model) + "." + QuoteName(field) + " FROM " + QuoteName(model); + auto result = sidemantic_snapshot_rewrite(snapshot.c_str(), sql.c_str()); + string compiled = result.sql ? result.sql : ""; + string error = result.error ? result.error : ""; + sidemantic_free_result(result); + if (!error.empty()) throw BinderException("%s", error); + Parser parser(SidemanticBuiltinParserOptions()); + parser.ParseQuery(compiled); + if (parser.statements.size() != 1) throw BinderException("Semantic field must compile to one query"); + auto binder = Binder::CreateBinder(context); + auto bound = binder->Bind(*parser.statements[0]); + if (bound.types.size() != 1) throw BinderException("Semantic field must produce one column"); + return bound.types[0]; +} + +SemanticMetadata ReadMetadata(ClientContext &context) { + SemanticMetadata metadata; + auto &catalog = Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); + metadata.database = SidemanticName(catalog.GetName()); + metadata.database_oid = NumericCast(catalog.GetOid()); + auto snapshot = ReadSidemanticCatalog(context); + if (snapshot.payload.empty()) return metadata; + CatalogEntriesOwner entries(snapshot.payload); + unordered_set used; + case_insensitive_set_t physical_relations; + for (auto &schema_ref : Catalog::GetAllSchemas(context)) { + auto &schema = schema_ref.get(); + used.insert(NumericCast(schema.oid)); + bool same_schema = &schema.catalog == &catalog && StringUtil::CIEquals(SidemanticName(schema.name), SEMANTIC_SCHEMA); + if (same_schema) { + metadata.physical_schema = true; + metadata.schema_oid = NumericCast(schema.oid); + } + for (auto type : {CatalogType::TABLE_ENTRY, CatalogType::SEQUENCE_ENTRY, CatalogType::INDEX_ENTRY}) { + schema.Scan(context, type, [&](CatalogEntry &entry) { + used.insert(NumericCast(entry.oid)); + if (same_schema && type == CatalogType::TABLE_ENTRY) physical_relations.insert(SidemanticName(entry.name)); + }); + } + } + if (!metadata.physical_schema) metadata.schema_oid = StableOid(metadata.database + ".semantic", used); + // The Rust catalog is deterministically sorted, including across restarts. + for (idx_t i = 0; i < entries.result.count; ++i) { + auto &entry = entries.result.entries[i]; + if (string(entry.kind) != "model" || physical_relations.count(entry.name)) continue; + SemanticRelation relation; + relation.name = entry.name; + relation.comment = entry.description ? Value(entry.description) : Value(); + relation.oid = StableOid(metadata.database + ".semantic." + relation.name, used); + for (idx_t j = 0; j < entries.result.count; ++j) { + auto &field = entries.result.entries[j]; + if (!field.model_name || relation.name != field.model_name) continue; + auto kind = string(field.kind); + if (kind != "dimension" && kind != "metric") continue; + SemanticColumn column; + column.name = field.name; + column.ordinal = field.column_index; + column.comment = field.description ? Value(field.description) : Value(); + try { + column.type = BindFieldType(context, snapshot.payload, relation.name, column.name); + } catch (const std::exception &exception) { + // Like DuckDB's unbound views, keep the relation discoverable + // without inventing a physical type for an unavailable source. + // Do not hide cancellation or an engine invariant failure. + auto error = ErrorData(exception); + if (error.Type() == ExceptionType::INTERRUPT || error.Type() == ExceptionType::INTERNAL || + error.Type() == ExceptionType::FATAL) throw; + } + relation.columns.push_back(std::move(column)); + } + std::sort(relation.columns.begin(), relation.columns.end(), + [](const SemanticColumn &left, const SemanticColumn &right) { return left.ordinal < right.ordinal; }); + metadata.relations.push_back(std::move(relation)); + } + return metadata; +} + +struct DiscoveryRows : public TableFunctionData { + vector> rows; +}; + +struct DiscoveryState : public GlobalTableFunctionState { + idx_t offset = 0; +}; + +unique_ptr InitDiscovery(ClientContext &, TableFunctionInitInput &) { + return make_uniq(); +} + +void ScanDiscovery(ClientContext &, TableFunctionInput &input, DataChunk &output) { + auto &data = input.bind_data->Cast(); + auto &state = input.global_state->Cast(); + auto count = MinValue(STANDARD_VECTOR_SIZE, data.rows.size() - state.offset); + for (idx_t row = 0; row < count; ++row) { + for (idx_t column = 0; column < output.ColumnCount(); ++column) { + output.SetValue(column, row, data.rows[state.offset + row][column]); + } + } + state.offset += count; + output.SetCardinality(count); +} + +void NumericMetadata(const LogicalType &type, Value &precision, Value &radix, Value &scale) { + int bits = 0; + switch (type.id()) { + case LogicalTypeId::DECIMAL: + precision = Value::INTEGER(DecimalType::GetWidth(type)); + radix = Value::INTEGER(10); + scale = Value::INTEGER(DecimalType::GetScale(type)); + return; + case LogicalTypeId::TINYINT: bits = 8; break; + case LogicalTypeId::SMALLINT: bits = 16; break; + case LogicalTypeId::INTEGER: bits = 32; break; + case LogicalTypeId::BIGINT: bits = 64; break; + case LogicalTypeId::HUGEINT: bits = 128; break; + case LogicalTypeId::FLOAT: bits = 24; break; + case LogicalTypeId::DOUBLE: bits = 53; break; + default: return; + } + precision = Value::INTEGER(bits); + radix = Value::INTEGER(2); + scale = Value::INTEGER(0); +} + +unique_ptr BuildRows(const SemanticMetadata &metadata, const string &function, + const vector &names, const vector &types) { + auto result = make_uniq(); + auto append = [&](const unordered_map &values) { + vector row; + for (idx_t i = 0; i < names.size(); ++i) { + auto entry = values.find(SidemanticName(names[i])); + row.push_back(entry == values.end() ? Value(types[i]) : entry->second); + } + result->rows.push_back(std::move(row)); + }; + unordered_map common { + {"database_name", Value(metadata.database)}, {"database_oid", Value::BIGINT(metadata.database_oid)}, + {"schema_name", Value(SEMANTIC_SCHEMA)}, {"schema_oid", Value::BIGINT(metadata.schema_oid)}, + {"internal", Value::BOOLEAN(false)}, + {"tags", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, {}, {})} + }; + if (function == "duckdb_schemas") { + if (!metadata.relations.empty() && !metadata.physical_schema) { + common["oid"] = Value::BIGINT(metadata.schema_oid); + append(common); + } + return result; + } + for (auto &relation : metadata.relations) { + auto values = common; + if (function == "duckdb_views") { + values["view_name"] = Value(relation.name); + values["view_oid"] = Value::BIGINT(relation.oid); + values["comment"] = relation.comment; + values["temporary"] = Value::BOOLEAN(false); + values["column_count"] = Value::BIGINT(relation.columns.size()); + values["is_bound"] = Value::BOOLEAN(std::all_of(relation.columns.begin(), relation.columns.end(), + [](const SemanticColumn &column) { return column.type.id() != LogicalTypeId::INVALID; })); + append(values); + } else { + values["table_name"] = Value(relation.name); + values["table_oid"] = Value::BIGINT(relation.oid); + values["is_nullable"] = Value::BOOLEAN(true); + values["is_generated"] = Value::BOOLEAN(false); + for (idx_t i = 0; i < relation.columns.size(); ++i) { + auto &column = relation.columns[i]; + auto row = values; + row["column_name"] = Value(column.name); + row["column_index"] = Value::INTEGER(i + 1); + row["comment"] = column.comment; + if (column.type.id() != LogicalTypeId::INVALID) { + row["data_type"] = Value(column.type.ToString()); + row["data_type_id"] = Value::BIGINT(static_cast(column.type.id())); + NumericMetadata(column.type, row["numeric_precision"], row["numeric_precision_radix"], row["numeric_scale"]); + } + append(row); + } + } + } + return result; +} + +void OverlayScans(PlannerExtensionInput &input, unique_ptr &plan, + unique_ptr &metadata) { + // PREPARE has a nested planner. Its child can already contain this overlay. + if (plan->type == LogicalOperatorType::LOGICAL_UNION) { + for (auto &child : plan->children) { + if (child->type == LogicalOperatorType::LOGICAL_GET && + ScanName(child->Cast()) == "sidemantic_catalog_scan") return; + } + } + for (auto &child : plan->children) OverlayScans(input, child, metadata); + if (plan->type != LogicalOperatorType::LOGICAL_GET) return; + auto &get = plan->Cast(); + auto name = ScanName(get); + if (name != "duckdb_columns" && name != "duckdb_views" && name != "duckdb_schemas") return; + RegisterSidemanticCatalogRead(input.context, input.binder.GetStatementProperties()); + if (!metadata) metadata = make_uniq(ReadMetadata(input.context)); + if (metadata->relations.empty()) return; + auto rows = BuildRows(*metadata, name, get.names, get.returned_types); + if (rows->rows.empty()) return; + TableFunction function("sidemantic_catalog_scan", {}, ScanDiscovery, nullptr, InitDiscovery); +#if SIDEMANTIC_NEW_IDENTIFIER_API + auto extra = make_uniq(input.binder.GenerateTableIndex(), BoundTableFunction(function), std::move(rows), + get.returned_types, get.names); +#else + auto extra = make_uniq(input.binder.GenerateTableIndex(), function, std::move(rows), + get.returned_types, get.names); +#endif + auto columns = get.GetColumnIds(); + extra->SetColumnIds(std::move(columns)); + auto union_index = get.table_index; + get.table_index = input.binder.GenerateTableIndex(); + auto column_count = get.GetColumnBindings().size(); + plan = make_uniq(union_index, column_count, std::move(plan), std::move(extra), + LogicalOperatorType::LOGICAL_UNION, true); +} + +void DiscoveryPostBind(PlannerExtensionInput &input, BoundStatement &statement) { + if (!statement.plan) return; + unique_ptr metadata; + OverlayScans(input, statement.plan, metadata); +} + +} // namespace + +bool SidemanticPhysicalRelationExists(ClientContext &context, const string &model) { + auto &catalog = Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); +#if SIDEMANTIC_NEW_IDENTIFIER_API + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, + QualifiedName(catalog.GetName(), Identifier(SEMANTIC_SCHEMA), Identifier(model))); + return Catalog::GetEntry(context, lookup, OnEntryNotFound::RETURN_NULL) != nullptr; +#else + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, model); + return Catalog::GetEntry(context, catalog.GetName(), SEMANTIC_SCHEMA, lookup, OnEntryNotFound::RETURN_NULL) != nullptr; +#endif +} + +void RegisterSidemanticDiscovery(DatabaseInstance &db) { + PlannerExtension extension; + extension.post_bind_function = DiscoveryPostBind; + PlannerExtension::Register(DBConfig::GetConfig(db), extension); +} + +} // namespace duckdb diff --git a/sidemantic-duckdb/src/sidemantic_extension.cpp b/sidemantic-duckdb/src/sidemantic_extension.cpp index e1c0cbac..86dcd3a3 100644 --- a/sidemantic-duckdb/src/sidemantic_extension.cpp +++ b/sidemantic-duckdb/src/sidemantic_extension.cpp @@ -161,6 +161,7 @@ static void LoadInternal(ExtensionLoader &loader) { ParserExtension::Register(config, parser); OperatorExtension::Register(config, make_shared_ptr()); RegisterSidemanticRouting(db); + RegisterSidemanticDiscovery(db); loader.RegisterFunction(TableFunction("sidemantic_load", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); loader.RegisterFunction(TableFunction("sidemantic_load_file", {LogicalType::VARCHAR}, LoadFunction, LoadBind, LoadInit)); diff --git a/sidemantic-duckdb/src/sidemantic_parser.cpp b/sidemantic-duckdb/src/sidemantic_parser.cpp index 8a396cab..48b62719 100644 --- a/sidemantic-duckdb/src/sidemantic_parser.cpp +++ b/sidemantic-duckdb/src/sidemantic_parser.cpp @@ -6,6 +6,7 @@ #include "duckdb/common/error_data.hpp" #include "duckdb/main/config.hpp" #include "duckdb/main/connection_manager.hpp" +#include "duckdb/main/database_manager.hpp" #include "duckdb/main/prepared_statement_data.hpp" #include "duckdb/main/valid_checker.hpp" #include "duckdb/parser/expression/columnref_expression.hpp" @@ -233,14 +234,21 @@ struct ModelListOwner { // A null binding shadows a semantic model name (e.g. a CTE or a physical table // aliased to that name). Each query has its own scope, including nested SELECTs. -using ModelScope = case_insensitive_map_t; +struct ModelBinding { + ModelBinding(const SidemanticModelInfo *model_p = nullptr, bool semantic_schema_p = false) + : model(model_p), semantic_schema(semantic_schema_p) { + } + const SidemanticModelInfo *model; + bool semantic_schema; +}; +using ModelScope = case_insensitive_map_t; using CteNames = case_insensitive_set_t; class SemanticReferences { public: - explicit SemanticReferences(const SidemanticModelList &list) { + SemanticReferences(ClientContext &context, const SidemanticModelList &list) : context(context) { for (idx_t i = 0; i < list.count; ++i) { - models[list.models[i].name] = &list.models[i]; + models[list.models[i].name] = {&list.models[i], false}; } } @@ -272,7 +280,7 @@ class SemanticReferences { if (node.type != QueryNodeType::SELECT_NODE) return found; auto &select = node.Cast(); auto scope = std::move(inherited); - for (auto &name : ctes) scope[name] = nullptr; + for (auto &name : ctes) scope[name] = {}; if (select.from_table) CollectTables(*select.from_table, scope, ctes, found); auto inspect = [&](unique_ptr &expression) { found |= Expression(*expression, scope, ctes); @@ -288,6 +296,7 @@ class SemanticReferences { } private: + ClientContext &context; ModelScope models; bool CanonicalField(const SidemanticModelInfo *model, string &name) { @@ -329,29 +338,38 @@ class SemanticReferences { found |= Query(*ref.Cast().subquery->node, scope, ctes); } const SidemanticModelInfo *model = nullptr; + bool virtual_relation = false; string name; if (ref.type == TableReferenceType::BASE_TABLE) { auto &table = ref.Cast(); #if SIDEMANTIC_QUALIFIED_TABLE_API name = SidemanticName(table.Table()); - bool qualified = !table.GetQualifiedName().Schema().empty() || !table.GetQualifiedName().Catalog().empty(); + auto schema = SidemanticName(table.GetQualifiedName().Schema()); + auto catalog = SidemanticName(table.GetQualifiedName().Catalog()); #else name = table.table_name; - bool qualified = !table.schema_name.empty() || !table.catalog_name.empty(); + auto schema = table.schema_name; + auto catalog = table.catalog_name; #endif + bool qualified = !schema.empty() || !catalog.empty(); + bool semantic_schema = StringUtil::CIEquals(schema, "semantic") && + (catalog.empty() || StringUtil::CIEquals(catalog, SidemanticName(DatabaseManager::GetDefaultDatabase(context)))) && + !SidemanticPhysicalRelationExists(context, name); auto entry = models.find(name); - if (!qualified && !ctes.count(name) && entry != models.end()) model = entry->second; + if (((!qualified && !ctes.count(name)) || semantic_schema) && entry != models.end()) model = entry->second.model; if (model) { + virtual_relation = semantic_schema; name = model->name; #if SIDEMANTIC_QUALIFIED_TABLE_API table.SetTable(Identifier(name)); #else table.table_name = name; #endif + found |= semantic_schema; } } auto alias = ref.alias.empty() ? name : SidemanticName(ref.alias); - if (!alias.empty()) scope[alias] = model; + if (!alias.empty()) scope[alias] = {model, virtual_relation}; } bool TableExpressions(TableRef &ref, const ModelScope &scope, const CteNames &ctes) { @@ -369,17 +387,39 @@ class SemanticReferences { #else auto &names = column.column_names; #endif + if (names.size() >= 3) { + auto qualifier = SidemanticName(names[names.size() - 2]); + auto local = scope.find(qualifier); + // Explicit physical qualifiers retain native resolution even + // when the unqualified FROM name also names a semantic model. + if (!StringUtil::CIEquals(SidemanticName(names[names.size() - 3]), "semantic")) { + if (local != scope.end() && local->second.semantic_schema) { + throw BinderException("Column qualifier does not match the semantic relation"); + } + return false; + } + bool semantic_model = local != scope.end() ? local->second.model != nullptr : models.count(qualifier) != 0; + if (semantic_model) { + bool current_catalog = names.size() == 3 || + (names.size() == 4 && StringUtil::CIEquals(SidemanticName(names[0]), + SidemanticName(DatabaseManager::GetDefaultDatabase(context)))); + if (!current_catalog) { + throw BinderException("Semantic column qualifier must name the current database's semantic schema"); + } + names.erase(names.begin(), names.end() - 2); + } + } if (names.size() == 2) { auto qualifier = SidemanticName(names[0]); auto local = scope.find(qualifier); const SidemanticModelInfo *model = nullptr; if (local != scope.end()) { - model = local->second; + model = local->second.model; qualifier = local->first; } else { auto entry = models.find(qualifier); if (entry != models.end()) { - model = entry->second; + model = entry->second.model; qualifier = entry->first; } } @@ -393,7 +433,7 @@ class SemanticReferences { if (names.size() == 1) { for (auto &binding : scope) { auto field = SidemanticName(names[0]); - if (CanonicalField(binding.second, field)) { + if (CanonicalField(binding.second.model, field)) { names[0] = SidemanticIdentifier(field); return true; } @@ -461,14 +501,14 @@ SelectStatement *QueryInStatement(SQLStatement &statement, bool include_insert_c return nullptr; } -unique_ptr CompileSemanticStatement(SQLStatement &statement, +unique_ptr CompileSemanticStatement(ClientContext &context, SQLStatement &statement, const SidemanticCatalogSnapshot &snapshot, bool explicit_semantic = false) { auto rewritten = statement.Copy(); auto query = QueryInStatement(*rewritten, true); if (!query) return nullptr; ModelListOwner models(snapshot.payload); - if (!SemanticReferences(models.list).Query(*query->node) && !explicit_semantic) return nullptr; + if (!SemanticReferences(context, models.list).Query(*query->node) && !explicit_semantic) return nullptr; auto sql = query->ToString(); auto result = sidemantic_snapshot_rewrite(snapshot.payload.c_str(), sql.c_str()); if (result.error) { @@ -658,7 +698,7 @@ class SidemanticRoutingState : public ClientContextState { // that semantic definitions were absent or returning a partial rewrite. if (native_failed && catalog_error.HasError() && QueryInStatement(statement)) catalog_error.Throw(); if (!has_models) return nullptr; - return CompileSemanticStatement(statement, snapshot); + return CompileSemanticStatement(context, statement, snapshot); } ClientContext &context; @@ -955,7 +995,7 @@ BoundStatement sidemantic_bind(ClientContext &context, Binder &binder, OperatorE if (routing) routing->probing = false; RegisterSidemanticCatalogRead(context, binder.GetStatementProperties()); auto snapshot = ReadSidemanticCatalog(context); - auto original = CompileSemanticStatement(*data.statement, snapshot, data.explicit_semantic); + auto original = CompileSemanticStatement(context, *data.statement, snapshot, data.explicit_semantic); if (!original) original = data.statement->Copy(); auto child_binder = Binder::CreateBinder(context, &binder); return child_binder->Bind(*original); diff --git a/sidemantic-duckdb/test/test_catalog_host.py b/sidemantic-duckdb/test/test_catalog_host.py index b6c5d222..0b387826 100644 --- a/sidemantic-duckdb/test/test_catalog_host.py +++ b/sidemantic-duckdb/test/test_catalog_host.py @@ -78,7 +78,7 @@ def execute(sql, *, setup=SETUP, database=":memory:", readonly=False, error=None args.append(str(database)) result = subprocess.run( args, - input=f".output /dev/null\nload {literal(str(extension))};\n{setup}\n.output stdout\n{sql}\n", + input=f".output /dev/null\nset threads = 2;\nload {literal(str(extension))};\n{setup}\n.output stdout\n{sql}\n", text=True, capture_output=True, timeout=30, @@ -216,5 +216,182 @@ def test_prefixless_physical_collision_and_prepared_catalog_changes(): assert totals == [40, 2, 40] +def test_standard_catalog_discovery(): + tables, columns, attributes, schema, rows, stars = execute(""" + select table_name, table_type from information_schema.tables + where table_schema = 'semantic' order by table_name; + select column_name, data_type, numeric_precision, numeric_scale + from information_schema.columns where table_schema = 'semantic' and table_name = 'orders' + order by ordinal_position; + select c.relname, a.attname, a.atttypid, a.attnum + from pg_catalog.pg_class c join pg_catalog.pg_namespace n on c.relnamespace = n.oid + join pg_catalog.pg_attribute a on c.oid = a.attrelid + where n.nspname = 'semantic' and c.relname = 'orders' order by a.attnum; + select schema_name from information_schema.schemata where schema_name = 'semantic'; + select status, revenue from semantic.orders order by status; + select * from semantic.orders order by status; + """) + assert tables == [{"table_name": name, "table_type": "VIEW"} for name in ("customers", "orders", "unrelated")] + assert columns == [ + {"column_name": "status", "data_type": "VARCHAR", "numeric_precision": None, "numeric_scale": None}, + {"column_name": "revenue", "data_type": "DECIMAL(38,2)", "numeric_precision": 38, "numeric_scale": 2}, + ] + assert [(row["attname"], row["attnum"]) for row in attributes] == [("status", 1), ("revenue", 2)] + assert all(row["atttypid"] is not None for row in attributes) + assert schema == [{"schema_name": "semantic"}] + assert [(row["status"], Decimal(str(row["revenue"]))) for row in rows] == [ + ("complete", Decimal("40")), + ("pending", Decimal("5")), + ] + assert list(stars[0]) == [column["column_name"] for column in columns] + assert stars == rows + + +def test_standard_catalog_subset_counts_unknown_types_and_prepared_changes(): + results = execute(""" + select count(*) as n from information_schema.tables where table_schema = 'semantic'; + select column_name, data_type from information_schema.columns + where table_schema = 'semantic' and table_name = 'unrelated'; + select is_bound from duckdb_views() where schema_name = 'semantic' and view_name = 'unrelated'; + prepare inspect_fields as select column_name, data_type from information_schema.columns + where table_schema = 'semantic' and table_name = 'orders' order by ordinal_position; + execute inspect_fields; + begin; + create metric orders.order_count as count(*); + execute inspect_fields; + create or replace metric orders.revenue as count(*); + execute inspect_fields; + rollback; + execute inspect_fields; + """) + assert results[0] == [{"n": 3}] + assert results[1] == [{"column_name": "name", "data_type": None}] + assert results[2] == [{"is_bound": False}] + fields = [rows for rows in results[3:] if rows and "column_name" in rows[0]] + assert [[row["column_name"] for row in rows] for rows in fields] == [ + ["status", "revenue"], + ["status", "revenue", "order_count"], + ["status", "revenue", "order_count"], + ["status", "revenue"], + ] + assert next(row["data_type"] for row in fields[2] if row["column_name"] == "revenue") == "BIGINT" + assert fields[0] == fields[-1] + assert execute(""" + select a.attname, a.atttypid from pg_catalog.pg_attribute a + join pg_catalog.pg_class c on a.attrelid = c.oid + join pg_catalog.pg_namespace n on c.relnamespace = n.oid + where n.nspname = 'semantic' and c.relname = 'unrelated'; + """)[0] == [{"attname": "name", "atttypid": None}] + + +def test_standard_catalog_physical_namespace_collision_and_rollback(): + results = execute(""" + create schema semantic; + select count(*) as n from information_schema.schemata where schema_name = 'semantic'; + begin; + create table semantic.orders(marker integer); + insert into semantic.orders values (123); + select column_name from information_schema.columns + where table_schema = 'semantic' and table_name = 'orders'; + select * from semantic.orders; + select table_type from information_schema.tables + where table_schema = 'semantic' and table_name = 'orders'; + rollback; + select column_name from information_schema.columns + where table_schema = 'semantic' and table_name = 'orders' order by ordinal_position; + select revenue from semantic.orders; + """) + assert results[:-1] == [ + [{"n": 1}], + [{"column_name": "marker"}], + [{"marker": 123}], + [{"table_type": "BASE TABLE"}], + [{"column_name": "status"}, {"column_name": "revenue"}], + ] + assert Decimal(str(results[-1][0]["revenue"])) == Decimal("45") + + +def test_standard_catalog_persistent_oids_and_readonly(tmp_path): + database = tmp_path / "metadata.duckdb" + sql = """ + select c.oid, c.relname, n.oid as namespace_oid from pg_catalog.pg_class c + join pg_catalog.pg_namespace n on c.relnamespace = n.oid + where n.nspname = 'semantic' order by c.relname; + """ + first = execute(sql, database=database) + second = execute(sql, setup="", database=database, readonly=True) + assert first == second + assert len(first[0]) == 3 + assert all(0 < row["oid"] <= 2**31 - 1 and 0 < row["namespace_oid"] <= 2**31 - 1 for row in first[0]) + + +def test_semantic_namespace_qualification_and_cte_shadowing(): + results = execute(""" + select semantic.orders.revenue from semantic.orders; + with orders as (select 999 as revenue) select revenue from semantic.orders; + with orders as (select 999 as revenue) select revenue from orders; + select o.revenue from semantic.orders o; + """) + assert [Decimal(str(rows[0]["revenue"])) for rows in results] == [45, 45, 999, 45] + execute("select wrong_db.semantic.orders.revenue from semantic.orders;", error="Semantic column qualifier") + execute("select wrong_schema.orders.revenue from semantic.orders;", error="Column qualifier does not match") + + +def test_physical_column_qualifiers_preserve_native_resolution(): + results = execute( + """ + select main.orders.amount from orders order by id; + select main.orders.status from orders order by id; + """, + setup=""" + create table orders(id integer, status varchar, amount integer); + insert into orders values (1, 'paid', 10), (2, 'paid', 30), (3, 'pending', 5); + MODEL orders FROM orders ( + primary key(id) + status + sum(amount) AS revenue + ); + """, + ) + assert results == [ + [{"amount": 10}, {"amount": 30}, {"amount": 5}], + [{"status": "paid"}, {"status": "paid"}, {"status": "pending"}], + ] + + +def test_standard_catalog_numeric_metadata_matches_host(): + columns = [ + ("u8", "utinyint"), + ("u16", "usmallint"), + ("u32", "uinteger"), + ("u64", "ubigint"), + ("u128", "uhugeint"), + ("i8", "tinyint"), + ("i16", "smallint"), + ("i32", "integer"), + ("i64", "bigint"), + ("i128", "hugeint"), + ("f32", "float"), + ("f64", "double"), + ("dec", "decimal(12,3)"), + ] + fields = ", ".join(f"{name} {kind}" for name, kind in columns) + dimensions = "\n".join(f" - name: {name}\n type: numeric" for name, _ in columns) + definitions = f"models:\n- name: numbers\n table: raw_numbers\n primary_key: i32\n dimensions:\n{dimensions}\n" + setup = f"create table raw_numbers({fields}); select * from sidemantic_load({literal(definitions)});" + native, semantic = execute( + """ + select column_name, data_type, numeric_precision, numeric_precision_radix, numeric_scale + from information_schema.columns where table_schema = 'main' and table_name = 'raw_numbers' + order by column_name; + select column_name, data_type, numeric_precision, numeric_precision_radix, numeric_scale + from information_schema.columns where table_schema = 'semantic' and table_name = 'numbers' + order by column_name; + """, + setup=setup, + ) + assert native == semantic + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v", "-o", "addopts="])) diff --git a/sidemantic-duckdb/test/test_duckflight_host.py b/sidemantic-duckdb/test/test_duckflight_host.py index eb182315..77a683cd 100644 --- a/sidemantic-duckdb/test/test_duckflight_host.py +++ b/sidemantic-duckdb/test/test_duckflight_host.py @@ -46,7 +46,7 @@ def postgres(tmp_path): address = f"127.0.0.1:{port}" binary = Path(os.environ["SIDEMANTIC_DUCKDB_BINARY"]).resolve() assert binary.is_file(), binary - setup = "" + setup = "set threads = 2;\n" for name in required: path = Path(os.environ[name]).resolve() assert path.is_file(), path @@ -68,7 +68,7 @@ def postgres(tmp_path): try: host.stdin.write(setup) host.stdin.flush() - deadline = time.monotonic() + 30 + deadline = time.monotonic() + float(os.environ.get("DUCKFLIGHT_TEST_STARTUP_TIMEOUT", "30")) while time.monotonic() < deadline: if host.poll() is not None: log.seek(0) @@ -126,7 +126,8 @@ def test_semantic_queries_and_postgres_metadata(orders): assert [column.name for column in result.description] == ["status", "revenue"] assert result.fetchall() == [("complete", Decimal("150.00")), ("pending", Decimal("75.00"))] assert postgres.execute( - "select column_name from information_schema.columns where table_name = 'orders' order by ordinal_position" + "select column_name from information_schema.columns " + "where table_schema = 'public' and table_name = 'orders' order by ordinal_position" ).fetchall() == [("order_id",), ("status",), ("amount",)] @@ -188,5 +189,134 @@ def test_native_model_prepare_and_describe_do_not_mutate(orders): assert orders.execute("select revenue from prepared_orders").fetchall() == [(Decimal("225.00"),)] +def semantic_attributes(connection, model="orders", *, prepared=False): + return connection.execute( + """ + select n.oid, c.oid, c.relkind, c.relnatts, + a.attname, a.attnum, a.atttypid, a.atttypmod, t.typname + from pg_catalog.pg_namespace n + join pg_catalog.pg_class c on c.relnamespace = n.oid + join pg_catalog.pg_attribute a on a.attrelid = c.oid + left join pg_catalog.pg_type t on t.oid = a.atttypid + where n.nspname = 'semantic' and c.relname = %s + and a.attnum > 0 and not a.attisdropped + order by a.attnum + """, + (model,), + prepare=prepared, + ).fetchall() + + +@pytest.mark.parametrize("prepared", [False, True]) +def test_standard_semantic_catalogs_and_qualified_query(orders, prepared): + assert orders.execute( + """ + select table_schema, table_name, table_type, is_insertable_into + from information_schema.tables + where table_name = 'orders' and table_schema in ('public', 'semantic') + order by table_schema + """, + prepare=prepared, + ).fetchall() == [("public", "orders", "BASE TABLE", "YES"), ("semantic", "orders", "VIEW", "NO")] + assert orders.execute( + """ + select column_name, ordinal_position, data_type, udt_name, numeric_precision, numeric_scale + from information_schema.columns + where table_schema = 'semantic' and table_name = 'orders' + order by ordinal_position + """, + prepare=prepared, + ).fetchall() == [("status", 1, "text", "text", None, None), ("revenue", 2, "numeric", "numeric", 38, 2)] + attributes = semantic_attributes(orders, prepared=prepared) + assert len(attributes) == 2 + schema_oid, relation_oid = attributes[0][:2] + assert 0 < schema_oid < 2**31 + assert 0 < relation_oid < 2**31 + assert schema_oid != relation_oid + assert attributes == [ + (schema_oid, relation_oid, "v", 2, "status", 1, 25, -1, "text"), + (schema_oid, relation_oid, "v", 2, "revenue", 2, 1700, (38 << 16) + 2 + 4, "numeric"), + ] + assert orders.execute("select 'semantic.orders'::regclass::bigint", prepare=prepared).fetchone() == (relation_oid,) + result = orders.execute( + "select status, revenue from semantic.orders where status = %s", + ("pending",), + prepare=prepared, + ) + assert [column.name for column in result.description] == ["status", "revenue"] + assert [column.type_code for column in result.description] == [25, 1700] + assert result.fetchall() == [("pending", Decimal("75.00"))] + star = orders.execute("select * from semantic.orders order by status", prepare=prepared) + assert [column.name for column in star.description] == [row[4] for row in attributes] + assert star.fetchall() == [("complete", Decimal("150.00")), ("pending", Decimal("75.00"))] + # Introspection must preserve the physical table's original schema and types. + assert orders.execute( + """ + select column_name, ordinal_position, data_type, numeric_precision, numeric_scale + from information_schema.columns + where table_schema = 'public' and table_name = 'orders' + order by ordinal_position + """ + ).fetchall() == [ + ("order_id", 1, "integer", 32, 0), + ("status", 2, "text", None, None), + ("amount", 3, "numeric", 10, 2), + ] + + +def test_semantic_catalog_visibility_across_connections_and_rollback(orders): + original = semantic_attributes(orders, prepared=True) + assert len(original) == 2 + with psycopg.connect(orders.info.dsn, password=orders.info.password, autocommit=True) as observer: + assert semantic_attributes(observer, prepared=True) == original + orders.execute("begin") + try: + orders.execute("drop model orders") + assert semantic_attributes(orders, prepared=True) == [] + assert semantic_attributes(observer, prepared=True) == original + finally: + orders.execute("rollback") + assert semantic_attributes(orders, prepared=True) == original + assert semantic_attributes(observer, prepared=True) == original + orders.execute("begin read only") + try: + assert semantic_attributes(orders, prepared=True) == original + assert orders.execute("select revenue from semantic.orders").fetchall() == [(Decimal("225.00"),)] + finally: + orders.execute("rollback") + orders.execute("drop model orders") + assert semantic_attributes(observer, prepared=True) == [] + + +def test_unavailable_semantic_source_has_unknown_catalog_type(postgres): + postgres.execute("MODEL offline_model FROM missing_source (sum(amount) AS revenue)") + assert postgres.execute( + """ + select column_name, data_type, udt_name + from information_schema.columns + where table_schema = 'semantic' and table_name = 'offline_model' + """ + ).fetchall() == [("revenue", None, None)] + attributes = semantic_attributes(postgres, "offline_model") + assert len(attributes) == 1 + assert attributes[0][2:] == ("v", 1, "revenue", 1, None, -1, None) + + +def test_physical_relation_in_semantic_schema_shadows_virtual_model(orders): + orders.execute("create schema semantic") + orders.execute("create table semantic.orders(physical_value integer)") + orders.execute("insert into semantic.orders values (42)") + assert orders.execute( + """ + select column_name, data_type from information_schema.columns + where table_schema = 'semantic' and table_name = 'orders' + """ + ).fetchall() == [("physical_value", "integer")] + attributes = semantic_attributes(orders) + assert len(attributes) == 1 + assert attributes[0][2:] == ("r", 1, "physical_value", 1, 23, -1, "int4") + assert orders.execute("select * from semantic.orders").fetchall() == [(42,)] + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v", "-o", "addopts="])) diff --git a/sidemantic-duckdb/test/test_release_workflow.py b/sidemantic-duckdb/test/test_release_workflow.py index cd3357fa..f817b58c 100644 --- a/sidemantic-duckdb/test/test_release_workflow.py +++ b/sidemantic-duckdb/test/test_release_workflow.py @@ -1,5 +1,6 @@ """Execute the release input boundary without running builds or publishing.""" +import hashlib import os import subprocess from pathlib import Path @@ -8,6 +9,12 @@ import yaml WORKFLOW = Path(__file__).resolve().parents[2] / ".github/workflows/duckdb-extension-release.yml" +PLATFORMS = ["linux_amd64", "linux_arm64", "osx_amd64", "osx_arm64"] + + +def workflow_step(job, name): + workflow = yaml.safe_load(WORKFLOW.read_text()) + return next(step for step in workflow["jobs"][job]["steps"] if step.get("name") == name) @pytest.mark.parametrize( @@ -50,3 +57,81 @@ def test_release_accepts_supported_version(tmp_path, event): ) assert result.returncode == 0, result.stderr assert output.read_text() == "version=v1.5.6\n" + + +@pytest.mark.parametrize("platform", PLATFORMS) +def test_release_packages_matching_platform_with_verifiable_checksum(tmp_path, platform): + extension = tmp_path / "build/release/extension/sidemantic/sidemantic.duckdb_extension" + extension.parent.mkdir(parents=True) + extension.write_bytes(b"extension payload") + step = workflow_step("build", "Package extension artifact") + result = subprocess.run( + ["bash", "-e", "-c", step["run"]], + cwd=tmp_path, + env={**os.environ, "DUCKDB_VERSION": "v1.5.6", "PLATFORM": platform}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + asset = tmp_path / "dist" / f"sidemantic-duckdb-1.5.6-{platform}.duckdb_extension" + assert asset.read_bytes() == extension.read_bytes() + assert asset.with_suffix(".duckdb_extension.sha256").read_text() == ( + f"{hashlib.sha256(extension.read_bytes()).hexdigest()} {asset.name}\n" + ) + + +@pytest.mark.parametrize("failure", ["mismatched_tag", "missing_platform", "corrupt_payload", "existing_release", None]) +def test_publication_checks_provenance_and_all_payloads_before_creating_release(tmp_path, failure): + dist = tmp_path / "dist" + dist.mkdir() + for platform in PLATFORMS: + asset = dist / f"sidemantic-duckdb-1.5.6-{platform}.duckdb_extension" + payload = platform.encode() + asset.write_bytes(payload) + asset.with_suffix(".duckdb_extension.sha256").write_text( + f"{hashlib.sha256(payload).hexdigest()} {asset.name}\n" + ) + if failure == "missing_platform": + asset.unlink() + elif failure == "corrupt_payload": + asset.write_bytes(b"changed") + (tmp_path / "LICENSE").write_text("Test license\n") + commands = tmp_path / "bin" + commands.mkdir() + git = commands / "git" + git.write_text('#!/bin/sh\nif [ "$1" = rev-parse ]; then echo "$TAG_COMMIT"; fi\n') + git.chmod(0o755) + gh = commands / "gh" + gh.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$GH_ARGS"\nexit "$GH_STATUS"\n') + gh.chmod(0o755) + args = tmp_path / "gh-args" + step = workflow_step("github-release", "Verify and publish immutable artifacts") + result = subprocess.run( + ["bash", "-e", "-c", step["run"]], + cwd=tmp_path, + env={ + **os.environ, + "PATH": f"{commands}{os.pathsep}{os.environ['PATH']}", + "TAG": "sidemantic-duckdb-v0.1.0", + "RELEASE_COMMIT": "built-commit", + "TAG_COMMIT": "other-commit" if failure == "mismatched_tag" else "built-commit", + "GH_ARGS": str(args), + "GH_STATUS": "1" if failure == "existing_release" else "0", + }, + capture_output=True, + text=True, + ) + if failure: + assert result.returncode != 0 + else: + assert result.returncode == 0, result.stderr + assert len((dist / "SHA256SUMS").read_text().splitlines()) == len(PLATFORMS) + assert (dist / "LICENSE").read_text() == "Test license\n" + if failure in {"mismatched_tag", "missing_platform", "corrupt_payload"}: + assert not args.exists() + else: + arguments = args.read_text().splitlines() + assert arguments[:3] == ["release", "create", "sidemantic-duckdb-v0.1.0"] + assert "--verify-tag" in arguments + assert "--clobber" not in arguments + assert sum(arg.endswith(".duckdb_extension") for arg in arguments) == len(PLATFORMS) diff --git a/sidemantic-rs/include/sidemantic.h b/sidemantic-rs/include/sidemantic.h index 917a7ace..ca480f95 100644 --- a/sidemantic-rs/include/sidemantic.h +++ b/sidemantic-rs/include/sidemantic.h @@ -90,6 +90,8 @@ typedef struct { char *granularity; bool is_public; char *definition; + /* One-based SELECT * position for model dimensions/metrics; zero otherwise. */ + size_t column_index; } SidemanticCatalogEntry; typedef struct { diff --git a/sidemantic-rs/src/ffi/catalog/api.rs b/sidemantic-rs/src/ffi/catalog/api.rs index 2323e6a6..d476412f 100644 --- a/sidemantic-rs/src/ffi/catalog/api.rs +++ b/sidemantic-rs/src/ffi/catalog/api.rs @@ -30,6 +30,7 @@ struct CatalogEntry { name: String, qualified_name: String, definition: Value, + column_index: usize, } impl CatalogEntry { @@ -46,6 +47,7 @@ impl CatalogEntry { qualified_name: model .map_or_else(|| name.to_owned(), |model| format!("{model}.{name}")), definition: serde_json::to_value(definition).map_err(|error| error.to_string())?, + column_index: 0, }) } } @@ -84,21 +86,16 @@ fn entries(snapshot: &Snapshot, kind: &str, model: &str) -> CatalogResult = rows + .iter() + .filter(|row| row.column_index > 0) + .map(|row| (row.name.as_str(), row.column_index)) + .collect(); + assert_eq!( + columns, + [("account", 2), ("status", 1), ("revenue", 3), ("total", 4)] + ); + } } diff --git a/tests/db/test_adbc_adapter.py b/tests/db/test_adbc_adapter.py index 02f237e1..35ca4dd2 100644 --- a/tests/db/test_adbc_adapter.py +++ b/tests/db/test_adbc_adapter.py @@ -225,7 +225,7 @@ def test_adbc_adapter_from_url(): @pytest.mark.parametrize("url", ["sqlite:///:memory:", "adbc://sqlite/:memory:"]) def test_adbc_url_resolves_packaged_driver_without_system_install(monkeypatch, url): - import adbc_driver_manager.dbapi as adbc + adbc = pytest.importorskip("adbc_driver_manager.dbapi") sqlite_package = pytest.importorskip("adbc_driver_sqlite") from sidemantic.db.adbc import ADBCAdapter