Keep same-name tables physical and compose semantic relations

.github/actions/setup-uv/action.yml

@@ -39,3 +39,10 @@ runs:
           crates/dax-pyo3/python/**/*.py
           crates/dax-pyo3/python/**/*.pyi
           crates/dax-pyo3/python/**/py.typed
+    # PyPI connections occasionally time out on hosted runners; the default
+    # three retries is not enough to ride those out.
+    - name: Retry dependency downloads
+      shell: bash
+      run: |
+        echo "UV_HTTP_RETRIES=8" >> "$GITHUB_ENV"
+        echo "UV_HTTP_TIMEOUT=60" >> "$GITHUB_ENV"

.gitignore

@@ -57,3 +57,6 @@ package.json
 # GitHub Actions compiler cache and Cargo shim
 /.ci-mbx/
 /.ci-cache-bin/
+
+# Editor indexes
+**/.cache/clangd/

docs/native-format.md

@@ -201,7 +201,7 @@ dimensions:
 | `name` | Yes | Unique within model. |
 | `type` | No | `categorical`, `time`, `boolean`, or `numeric`. Defaults to `categorical`. |
 | `sql` | No | SQL expression. If omitted, the dimension name is used. |
-| `granularity` | No | Default grain for time dimensions. |
+| `granularity` | No | Default grain for time dimensions. On DuckDB, PostgreSQL and Redshift, `day` and coarser grains return `DATE` values; `hour`, `minute` and `second` return timestamps. |
 | `supported_granularities` | No | Allowed grains for time dimensions. |
 | `description` | No | Human-readable description. |
 | `label` | No | Display label. |

sidemantic-duckdb/CMakeLists.txt

@@ -21,14 +21,15 @@ function(sidemantic_detect_apis)
             parser/expression/columnref_expression.hpp parser/tableref/basetableref.hpp
             parser/parser_options.hpp parser/expression/constant_expression.hpp
             parser/statement/insert_statement.hpp
-            parser/grammar_extension.hpp parser/peg/compiled_grammar.hpp)
+            parser/grammar_extension.hpp parser/peg/compiled_grammar.hpp
+            planner/operator/logical_get.hpp)
         if(EXISTS "${CMAKE_SOURCE_DIR}/src/include/duckdb/${header}")
             file(SHA256 "${CMAKE_SOURCE_DIR}/src/include/duckdb/${header}" header_hash)
             string(APPEND host_api ";${header_hash}")
         endif()
     endforeach()
     if(NOT SIDEMANTIC_HOST_API STREQUAL host_api)
-        foreach(api TOKEN_PARSE_FN NEW_IDENTIFIER_API NEW_EXPRESSION_API QUALIFIED_TABLE_API BUILTIN_PARSER_OPTIONS LITERAL_API GRAMMAR_EXTENSION INSERT_QUERY_NODE)
+        foreach(api TOKEN_PARSE_FN NEW_IDENTIFIER_API NEW_EXPRESSION_API QUALIFIED_TABLE_API BUILTIN_PARSER_OPTIONS LITERAL_API GRAMMAR_EXTENSION INSERT_QUERY_NODE BOUND_TABLE_FUNCTION GRAMMAR_EXTENSION_LIST)
             unset(SIDEMANTIC_${api} CACHE)
         endforeach()
         set(SIDEMANTIC_HOST_API "${host_api}" CACHE INTERNAL "DuckDB headers used for compatibility probes" FORCE)
@@ -75,6 +76,17 @@ function(sidemantic_detect_apis)
             duckdb::GrammarChange::PrependChoice(\"Statement\", \"SidemanticStatement\");
         }
     " SIDEMANTIC_GRAMMAR_EXTENSION)
+    check_cxx_source_compiles("
+        #include <duckdb/planner/operator/logical_get.hpp>
+        void probe(duckdb::TableFunction &function) { (void)duckdb::BoundTableFunction(function); }
+    " SIDEMANTIC_BOUND_TABLE_FUNCTION)
+    check_cxx_source_compiles("
+        #include <duckdb/parser/grammar_extension.hpp>
+        #include <duckdb/parser/peg/compiled_grammar.hpp>
+        void probe(duckdb::GrammarExtension &extension) {
+            duckdb::CompiledGrammar::Create(duckdb::vector<duckdb::reference<duckdb::GrammarExtension>> {extension});
+        }
+    " SIDEMANTIC_GRAMMAR_EXTENSION_LIST)
     configure_file(src/include/sidemantic_compat.hpp.in sidemantic_compat.hpp)
     cmake_pop_check_state()
 endfunction()

sidemantic-duckdb/README.md

@@ -79,18 +79,22 @@ METRIC avg_order_value AS AVG(amount);
 -- 4. Define dimensions (grouping attributes)
 DIMENSION (name status, type categorical);
 
--- 5. Query using semantic layer
-SELECT orders_model.status, orders_model.revenue FROM orders_model;
--- Automatically rewrites to:
--- SELECT status, SUM(amount) FROM orders GROUP BY 1
-
--- Result:
--- ┌───────────┬────────────────────┐
--- │  status   │ sum(orders.amount) │
--- ├───────────┼────────────────────┤
--- │ pending   │              75.00 │
--- │ completed │             250.00 │
--- └───────────┴────────────────────┘
+-- 5. Query using semantic layer: revenue is summed per status
+SELECT orders_model.status, orders_model.revenue FROM orders_model ORDER BY 1;
+
+-- ┌───────────┬───────────────┐
+-- │  status   │    revenue    │
+-- │  varchar  │ decimal(38,2) │
+-- ├───────────┼───────────────┤
+-- │ completed │        250.00 │
+-- │ pending   │         75.00 │
+-- └───────────┴───────────────┘
+
+-- The same query through the model's relation in the semantic schema
+SELECT status, revenue FROM semantic.orders_model;
+
+-- Inspect the generated SQL
+SELECT sidemantic_rewrite_sql('SELECT orders_model.status, orders_model.revenue FROM orders_model');
 ```
 
 ## Discover and manage definitions
@@ -112,9 +116,12 @@ FROM orders_model WHERE orders_model.status = 'completed';
 ```
 
 Discovery returns the definition kind, model and qualified name, label,
-description, semantic and declared data types, SQL, aggregation, relationship
-target and cardinality, granularity, visibility, and full JSON definition.
-Absent metadata is SQL `NULL`. `DESCRIBE MODEL` includes the model and all its
+description, semantic type, data type, SQL, aggregation, relationship
+target and cardinality, granularity, visibility, and the JSON definition.
+`data_type` is the declared type or, when none is declared, the type the field
+produces in the current database. Absent metadata is SQL `NULL` and omitted from
+the JSON definition. Definition statements and file loads report what they
+created, replaced or dropped. `DESCRIBE MODEL` includes the model and all its
 fields. `SHOW DIMENSIONS FOR` checks each dimension with the query compiler;
 use `EXPLAIN SELECT` to validate a complete combination of metrics and dimensions.
 `SHOW SEMANTIC METRICS`, and the corresponding forms for other kinds, are also
@@ -132,6 +139,31 @@ table or view takes precedence. Discovery observes transactions and creates no
 persistent schema or views; `SET schema = 'semantic'` requires an actual schema
 created by the user.
 
+A `semantic.<model>` relation is grouped by the model dimensions the surrounding
+query uses, so it can be joined, filtered, aggregated, described and summarized
+like a view:
+
+```sql
+SELECT s.status, s.revenue, t.target
+FROM semantic.orders_model s JOIN status_targets t USING (status);
+
+SELECT sum(revenue) FROM semantic.orders_model WHERE status = 'completed';
+
+DESCRIBE semantic.orders_model;
+```
+
+- Dimensions in `SELECT`, `GROUP BY`, `ORDER BY`, join conditions and `USING`
+  set the grain. `SELECT *` selects every dimension and metric, so it returns
+  one row per distinct combination of all dimensions.
+- `WHERE` predicates that read only one relation's dimensions filter the model
+  before grouping and do not add to the grain.
+- Aggregating a metric again is allowed when each group holds one row of the
+  relation, or when the aggregation composes: `sum` of a `sum` or `count`
+  metric, `min` of `min`, `max` of `max`. Other cases, such as `avg` of a ratio
+  over a coarser grain, are rejected because they compute a different number.
+- A relation needs at least one field: `SELECT count(*) FROM semantic.orders_model`
+  is rejected.
+
 Qualified definitions work independently of the connection's active model:
 
 ```sql
@@ -271,8 +303,17 @@ SELECT "o"."revenue" FROM "orders_model" AS "o";
 ```
 
 Routing inspects DuckDB's parsed table and field references. Comments and strings
-do not trigger rewriting; physical tables, CTEs and subqueries can shadow model
-names. Once a semantic field is detected, compiler errors are reported directly.
+do not trigger rewriting; CTEs, subqueries and schema-qualified tables such as
+`main.orders` are never models.
+
+A model may share its name with a table or view, as in `name: orders` with
+`table: orders`. A query keeps its ordinary SQL meaning while every reference
+binds to the table, including `SELECT *` and raw columns. It becomes semantic
+when it references a field only the model provides, such as a metric or a time
+grain like `created_at__month`. Mixing table-only columns and model-only fields
+through one name is an error; query the model as `semantic.orders` or the table
+as `main.orders`. Once a semantic field is detected, compiler errors are reported
+directly, and unknown fields list the closest matches.
 Use `SEMANTIC SELECT` to explicitly request semantic compilation. Unsupported
 semantic SQL still raises an error; the extension does not implement every DuckDB
 query construct in the Rust compiler.
@@ -524,11 +565,7 @@ models:
 -- Query order revenue by customer country (auto-JOIN)
 SELECT orders.revenue, customers.country FROM orders;
 
--- Automatically rewrites to:
--- SELECT SUM(orders.amount), c.country
--- FROM orders
--- LEFT JOIN customers AS c ON orders.customers_id = c.id
--- GROUP BY 2
+-- Joins customers through the relationship and groups revenue by country
 ```
 
 ### Relationship Types
@@ -623,6 +660,19 @@ switching DuckDB versions. CMake probes host APIs rather than assuming C++ ABI
 compatibility across releases. Release packaging targets unsigned Linux and macOS
 builds for 1.5.6; development-version CI does not publish stable artifacts.
 
+A preview artifact for a published 2.0 alpha runtime must be built from that
+runtime's exact commit and carry its version string, because the C++ extension
+API only loads in the identical build. For example, for `v2.0.0-alpha41489`:
+
+```bash
+make deps DUCKDB_VERSION=v2.0-cyanoptera DUCKDB_NEXT_COMMIT=10de9573794001c649621013bdd93553b54e00c9
+DUCKDB_VERSION=v2.0.0-alpha41489 make
+SIDEMANTIC_NATIVE_PEG=1 make test
+```
+
+Check `pragma version` in the target runtime first: `library_version` is the
+version string and `source_id` the commit prefix.
+
 ## Architecture
 
 ```

sidemantic-duckdb/src/include/sidemantic_catalog.hpp

@@ -16,17 +16,37 @@ SidemanticCatalogSnapshot ReadSidemanticCatalog(ClientContext &context);
 
 // Virtual semantic relations are exposed through the host's standard catalogs.
 bool SidemanticPhysicalRelationExists(ClientContext &context, const string &model);
+
+struct SidemanticPhysicalRelation {
+    string catalog;
+    string schema;
+    case_insensitive_set_t columns;
+    // Columns are unknown (e.g. an unbound view); treat every name as physical.
+    bool all_columns = false;
+
+    bool HasColumn(const string &name) const { return all_columns || columns.count(name); }
+};
+
+// Rust errors arrive as "Error: [context: ]Validation error: ..."; drop the labels.
+string SidemanticErrorText(string error);
+
+// The result type of one semantic field, bound against the current database.
+LogicalType SidemanticFieldType(ClientContext &context, const string &snapshot, const string &model,
+                                const string &field);
+
+// The table or view an unqualified name binds to natively, if any.
+bool SidemanticFindPhysicalRelation(ClientContext &context, const string &name, SidemanticPhysicalRelation &relation);
 void RegisterSidemanticDiscovery(DatabaseInstance &db);
 
 void RegisterSidemanticCatalogRead(ClientContext &context, StatementProperties &properties);
 void RegisterSidemanticCatalogWrite(ClientContext &context, StatementProperties &properties);
 
-// Supported operations: model, item, use, yaml, file.
-void ExecuteSidemanticMutation(ClientContext &context, const string &operation,
-                              const string &content, bool replace);
+// Supported operations: model, item, use, yaml, file. Returns a summary of
+// the definitions it created, replaced or dropped.
+string ExecuteSidemanticMutation(ClientContext &context, const string &operation,
+                                 const string &content, bool replace);
 
 ParserExtensionPlanResult PlanSidemanticMutation(ClientContext &context, const string &operation,
-                                                const string &content, bool replace,
-                                                const string &result_message);
+                                                const string &content, bool replace);
 
 } // namespace duckdb

sidemantic-duckdb/src/include/sidemantic_compat.hpp.in

@@ -8,6 +8,8 @@
 #cmakedefine01 SIDEMANTIC_LITERAL_API
 #cmakedefine01 SIDEMANTIC_GRAMMAR_EXTENSION
 #cmakedefine01 SIDEMANTIC_INSERT_QUERY_NODE
+#cmakedefine01 SIDEMANTIC_BOUND_TABLE_FUNCTION
+#cmakedefine01 SIDEMANTIC_GRAMMAR_EXTENSION_LIST
 
 #include "duckdb/parser/parser_options.hpp"
 #if SIDEMANTIC_NEW_IDENTIFIER_API

sidemantic-duckdb/src/sidemantic_api.cpp

@@ -3,6 +3,7 @@
 #include "sidemantic_compat.hpp"
 #include "sidemantic.h"
 
+#include "duckdb/common/error_data.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/planner/binder.hpp"
 
@@ -13,13 +14,41 @@ struct CatalogBindData : TableFunctionData {
     string kind;
     string model;
     string metric;
+    // Result types of fields without a declared data type, by qualified name.
+    case_insensitive_map_t<string> result_types;
 };
 
 struct CatalogState : GlobalTableFunctionState {
     vector<vector<Value>> rows;
     idx_t offset = 0;
 };
 
+// Report the type a field produces, as information_schema does, when its
+// definition declares none. Fields whose source cannot bind stay NULL.
+static void BindResultTypes(ClientContext &context, CatalogBindData &data) {
+    auto snapshot = ReadSidemanticCatalog(context);
+    if (snapshot.payload.empty()) return;
+    struct CatalogResult {
+        SidemanticCatalogEntries result;
+        ~CatalogResult() { sidemantic_free_catalog_entries(result); }
+    } result {sidemantic_snapshot_catalog(snapshot.payload.c_str(), data.kind.c_str(), data.model.c_str(),
+                                          data.metric.empty() ? nullptr : data.metric.c_str())};
+    if (result.result.error) return;
+    for (idx_t i = 0; i < result.result.count; ++i) {
+        auto &entry = result.result.entries[i];
+        string kind(entry.kind);
+        if (entry.data_type || !entry.model_name || (kind != "dimension" && kind != "metric")) continue;
+        try {
+            data.result_types[entry.qualified_name] =
+                SidemanticFieldType(context, snapshot.payload, entry.model_name, entry.name).ToString();
+        } catch (const std::exception &exception) {
+            auto error = ErrorData(exception);
+            if (error.Type() == ExceptionType::INTERRUPT || error.Type() == ExceptionType::INTERNAL ||
+                error.Type() == ExceptionType::FATAL) throw;
+        }
+    }
+}
+
 static unique_ptr<FunctionData> CatalogBind(ClientContext &context, TableFunctionBindInput &input,
                                           vector<LogicalType> &types, vector<SidemanticIdentifier> &names) {
     if (input.binder) RegisterSidemanticCatalogRead(context, input.binder->GetStatementProperties());
@@ -41,6 +70,7 @@ static unique_ptr<FunctionData> CatalogBind(ClientContext &context, TableFunctio
     types.emplace_back(LogicalType::BOOLEAN);
     names.emplace_back("definition");
     types.emplace_back(LogicalType::VARCHAR);
+    BindResultTypes(context, *data);
     return std::move(data);
 }
 
@@ -53,7 +83,7 @@ static unique_ptr<GlobalTableFunctionState> CatalogInit(ClientContext &context,
         if (result.error) {
             string error(result.error);
             sidemantic_free_result(result);
-            throw InvalidInputException("Sidemantic: %s", error);
+            throw InvalidInputException("Sidemantic: %s", SidemanticErrorText(error));
         }
         string definition(result.sql);
         sidemantic_free_result(result);
@@ -65,7 +95,7 @@ static unique_ptr<GlobalTableFunctionState> CatalogInit(ClientContext &context,
         ~CatalogResult() { sidemantic_free_catalog_entries(result); }
     } result {sidemantic_snapshot_catalog(snapshot.payload.c_str(), data.kind.c_str(), data.model.c_str(),
                                           data.metric.empty() ? nullptr : data.metric.c_str())};
-    if (result.result.error) throw InvalidInputException("Sidemantic: %s", result.result.error);
+    if (result.result.error) throw InvalidInputException("Sidemantic: %s", SidemanticErrorText(result.result.error));
     for (idx_t i = 0; i < result.result.count; ++i) {
         auto &entry = result.result.entries[i];
         vector<Value> row;
@@ -74,6 +104,8 @@ static unique_ptr<GlobalTableFunctionState> CatalogInit(ClientContext &context,
                           entry.target_model, entry.relationship_type, entry.granularity}) {
             row.push_back(value ? Value(value) : Value(LogicalType::VARCHAR));
         }
+        auto result_type = data.result_types.find(entry.qualified_name);
+        if (!entry.data_type && result_type != data.result_types.end()) row[7] = Value(result_type->second);
         row.push_back(Value::BOOLEAN(entry.is_public));
         row.push_back(Value(entry.definition));
         state->rows.push_back(std::move(row));