Fixes 35274: Check test parameters against their definitions and fix open-window and boolean-set runs

.github/playwright/impact-map.generated.json

@@ -804,6 +804,7 @@
         "playwright/e2e/Features/DataQuality/Profiler.spec.ts",
         "playwright/e2e/Features/DataQuality/ProfilerIngestionForm.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseImportExportBasic.spec.ts",
@@ -1721,6 +1722,7 @@
         "playwright/e2e/Features/DataQuality/ProfilerSettingsAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
         "playwright/e2e/Features/DataQuality/TableLevelTests.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseCrudAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
@@ -2035,6 +2037,7 @@
         "playwright/e2e/Features/DataAssetRulesEnabled.spec.ts",
         "playwright/e2e/Features/DataQuality/ProfilerIngestionForm.spec.ts",
         "playwright/e2e/Features/DataQuality/TableLevelTests.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/ExploreQuickFilters.spec.ts",
         "playwright/e2e/Features/IngestionListNameSorting.spec.ts",
         "playwright/e2e/Features/LandingPageWidgets/LandingPagePlatformHealth.spec.ts",
@@ -3085,6 +3088,7 @@
         "playwright/e2e/Features/DataQuality/Profiler.spec.ts",
         "playwright/e2e/Features/DataQuality/ProfilerIngestionForm.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseImportExportBasic.spec.ts",
@@ -3307,6 +3311,7 @@
         "playwright/e2e/Features/AdvancedSearch.spec.ts",
         "playwright/e2e/Features/ContextCenterDashboard.spec.ts",
         "playwright/e2e/Features/DataQuality/IncidentManagerSqlQuery.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DomainFilterQueryFilter.spec.ts",
         "playwright/e2e/Features/LandingPageWidgets/LandingPageContextCenter.spec.ts",
         "playwright/e2e/Features/LandingPageWidgets/LandingPageCriticalAnnouncements.spec.ts",
@@ -3497,6 +3502,7 @@
         "playwright/e2e/Features/DataQuality/ProfilerSettingsAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
         "playwright/e2e/Features/DataQuality/TableLevelTests.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseCrudAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
@@ -4113,6 +4119,7 @@
         "playwright/e2e/Features/DataQuality/DataQualityListAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/DataQualityPermissions.spec.ts",
         "playwright/e2e/Features/DataQuality/IncidentManagerDateFilter.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseImportExportBasic.spec.ts",
@@ -4845,6 +4852,7 @@
         "playwright/e2e/Features/AIMode/ConnectionServiceDetails.spec.ts",
         "playwright/e2e/Features/Announcements/SystemAnnouncements.spec.ts",
         "playwright/e2e/Features/BulkEditEntity.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/OnlineUsers.spec.ts",
         "playwright/e2e/Features/TestSuitePipelineRedeploy.spec.ts",
         "playwright/e2e/Pages/AIProfile/ProfileMembers.spec.ts",
@@ -5173,6 +5181,7 @@
         "playwright/e2e/Features/DataQuality/Profiler.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
         "playwright/e2e/Features/DataQuality/TableLevelTests.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseCrudAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",
@@ -5281,6 +5290,7 @@
         "playwright/e2e/Features/DataQuality/DataQuality.spec.ts",
         "playwright/e2e/Features/DataQuality/IncidentManagerDateFilter.spec.ts",
         "playwright/e2e/Features/DataQuality/IncidentManagerSqlQuery.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/EntitySuggestions.spec.ts",
         "playwright/e2e/Features/FailedTestCaseSampleData.spec.ts",
@@ -6794,6 +6804,14 @@
         "playwright/e2e/Features/DataQuality/ProfilerSettingsAfterReindex.spec.ts"
       ]
     },
+    {
+      "sources": [
+        "openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TableProfiler/QualityTab/QualityTab.component.tsx"
+      ],
+      "specs": [
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts"
+      ]
+    },
     {
       "sources": [
         "openmetadata-ui/src/main/resources/ui/src/components/Database/Profiler/TestSummary/RunSummaryTiles/RunSummaryTiles.tsx"
@@ -12000,6 +12018,7 @@
         "playwright/e2e/Features/DataQuality/ProfilerSettingsAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/RunTestCase.spec.ts",
         "playwright/e2e/Features/DataQuality/TableLevelTests.spec.ts",
+        "playwright/e2e/Features/DataQuality/TableTestCaseExportFilters.spec.ts",
         "playwright/e2e/Features/DataQuality/TableTestCasePagination.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseCrudAfterReindex.spec.ts",
         "playwright/e2e/Features/DataQuality/TestCaseDetailsPage.spec.ts",

bootstrap/sql/migrations/native/2.1.0/mysql/postDataMigrationSQLScript.sql

@@ -650,3 +650,36 @@ SET json = JSON_SET(json, '$.sourceConfig.config.sampleDataCount', 1000)
 WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.sourceConfig.config.type')) = 'AutoClassification'
   AND JSON_TYPE(JSON_EXTRACT(json, '$.sourceConfig.config.sampleDataCount')) = 'INTEGER'
   AND JSON_EXTRACT(json, '$.sourceConfig.config.sampleDataCount') > 1000;
+
+-- Say how to choose tableCustomSQLQuery.strategy and what columnValuesToBeInSet.matchEnum changes,
+-- so users and test generation read it from the definition. Seed data only creates missing definitions.
+UPDATE test_definition
+SET json = JSON_SET(
+    json,
+    REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', 'strategy', NULL, '$.parameterDefinition[*].name')), '.name', '.description'),
+    'How the query result is compared to the threshold. ROWS (default): the query returns the rows that break the rule, such as `SELECT * FROM t WHERE amount < 0`, and the number of rows returned is compared. COUNT: the query returns a single integer, such as `SELECT COUNT(*) FROM t WHERE amount < 0`, and that number is compared. Under ROWS a `SELECT COUNT(*)` query always returns one row. Do not use COUNT with a query that can return NULL or a non-integer, such as SUM or AVG.'
+)
+WHERE name = 'tableCustomSQLQuery'
+  AND JSON_SEARCH(json, 'one', 'strategy', NULL, '$.parameterDefinition[*].name') IS NOT NULL;
+
+UPDATE test_definition
+SET json = JSON_SET(
+    json,
+    REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', 'matchEnum', NULL, '$.parameterDefinition[*].name')), '.name', '.description'),
+    'If true, every observed value must be allowed. If false or omitted, the test passes when at least one allowed value occurs, even if other values are outside the set.'
+)
+WHERE name = 'columnValuesToBeInSet'
+  AND JSON_SEARCH(json, 'one', 'matchEnum', NULL, '$.parameterDefinition[*].name') IS NOT NULL;
+
+UPDATE test_definition
+SET json = JSON_SET(
+    json,
+    REPLACE(JSON_UNQUOTE(JSON_SEARCH(json, 'one', 'allowedValues', NULL, '$.parameterDefinition[*].name')), '.name', '.description'),
+    'Values used for matching. Neither mode requires every allowed value to occur in the column.'
+)
+WHERE name = 'columnValuesToBeInSet'
+  AND JSON_SEARCH(json, 'one', 'allowedValues', NULL, '$.parameterDefinition[*].name') IS NOT NULL;
+
+UPDATE test_definition
+SET json = JSON_SET(json, '$.description', 'Checks column values against an allowed set according to the matchEnum mode.')
+WHERE name = 'columnValuesToBeInSet';

bootstrap/sql/migrations/native/2.1.0/postgres/postDataMigrationSQLScript.sql

@@ -623,3 +623,55 @@ SET json = jsonb_set(json::jsonb, '{sourceConfig,config,sampleDataCount}', '1000
 WHERE json::jsonb -> 'sourceConfig' -> 'config' ->> 'type' = 'AutoClassification'
   AND jsonb_typeof(json::jsonb -> 'sourceConfig' -> 'config' -> 'sampleDataCount') = 'number'
   AND (json::jsonb -> 'sourceConfig' -> 'config' ->> 'sampleDataCount')::numeric > 1000;
+
+-- Say how to choose tableCustomSQLQuery.strategy and what columnValuesToBeInSet.matchEnum changes,
+-- so users and test generation read it from the definition. Seed data only creates missing definitions.
+UPDATE test_definition
+SET json = jsonb_set(
+    json,
+    '{parameterDefinition}',
+    (
+        SELECT jsonb_agg(
+            CASE
+                WHEN param->>'name' = 'strategy' THEN param || jsonb_build_object(
+                    'description',
+                    'How the query result is compared to the threshold. ROWS (default): the query returns the rows that break the rule, such as `SELECT * FROM t WHERE amount < 0`, and the number of rows returned is compared. COUNT: the query returns a single integer, such as `SELECT COUNT(*) FROM t WHERE amount < 0`, and that number is compared. Under ROWS a `SELECT COUNT(*)` query always returns one row. Do not use COUNT with a query that can return NULL or a non-integer, such as SUM or AVG.'
+                )
+                ELSE param
+            END
+            ORDER BY position
+        )
+        FROM jsonb_array_elements(json->'parameterDefinition') WITH ORDINALITY AS params(param, position)
+    )
+)
+WHERE name = 'tableCustomSQLQuery'
+  AND json->'parameterDefinition' @> '[{"name": "strategy"}]'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json,
+    '{parameterDefinition}',
+    (
+        SELECT jsonb_agg(
+            CASE
+                WHEN param->>'name' = 'matchEnum' THEN param || jsonb_build_object(
+                    'description',
+                    'If true, every observed value must be allowed. If false or omitted, the test passes when at least one allowed value occurs, even if other values are outside the set.'
+                )
+                WHEN param->>'name' = 'allowedValues' THEN param || jsonb_build_object(
+                    'description',
+                    'Values used for matching. Neither mode requires every allowed value to occur in the column.'
+                )
+                ELSE param
+            END
+            ORDER BY position
+        )
+        FROM jsonb_array_elements(json->'parameterDefinition') WITH ORDINALITY AS params(param, position)
+    )
+)
+WHERE name = 'columnValuesToBeInSet'
+  AND json->'parameterDefinition' @> '[{"name": "matchEnum"}]'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(json, '{description}', to_jsonb('Checks column values against an allowed set according to the matchEnum mode.'::text))
+WHERE name = 'columnValuesToBeInSet';

docs/decisions/2026-10-12-test-cases-are-checked-against-their-definition-at-create.md

@@ -0,0 +1,57 @@
+# A new test case is checked against its definition's options and column types
+
+- **Status:** Accepted
+- **Revisions:** v1 2026-10-12 (initial); v2 2026-10-16 (checked before the basic test suite is created, and an update checked for what it changes)
+- **Deciders:** Shreyansh Verma
+- **Guard:** `TestCaseDefinitionConformanceTest`, `TestCaseResourceIT` (`post_testWithUnofferedOptionValue_4xx`, `post_testOnUnsupportedColumnType_4xx`, `patch_toUnofferedOptionValue_4xx`, `createMany_withOneUnofferedOptionValue_createsNone`, `patch_keepsATestOnARetypedColumnEditable`, `post_rejectedForConformance_leavesNoTestSuite`)
+- **Related:** open-metadata/ai-platform#1650, #35274, #35276 (2.0 line)
+
+## Context
+
+`validateTestParameters` checked parameter names, required parameters and min/max rules. Nothing
+compared a value with the parameter's optionValues, or the column's type with the definition's
+supportedDataTypes, which no test case was checked against. Test cases that ingestion then
+aborted on (`'SUM' is not a valid Strategy`) or that measured nothing were stored as sent.
+
+## Decision
+
+A new test case is rejected when it:
+
+- gives a parameter with optionValues a value that is none of them (compared case-insensitively,
+  as `TestCaseThresholdValidator` reads thresholdUnit; a null, empty or whitespace value is unset);
+- targets a column whose dataType is not in a non-empty supportedDataTypes (an empty list means
+  every type, as the listing filter reads it; a column with no dataType is accepted). The column is
+  found case-insensitively, as `validateColumnTestCase` finds it.
+
+`prepare()` checks a test case when `update` is false, before it creates the table's basic test
+suite, the gate the disabled-definition check uses. PUT-create and the CSV import miss that gate
+for both checks: `createOrUpdate` prepares with `update` true, and the CSV import does not call
+`prepare()`. #35667 fixes those callers.
+
+An update is checked only for what it changes (`TestUpdater`, through
+`TestCaseDefinitionConformance.validateUpdate`): a parameter value it changes must be one of that
+parameter's options, and a test moved to another column or definition is checked in full. A row
+stored before these rules, or one whose column ingestion has since retyped, stays editable.
+
+useDynamicAssertion is not checked against supportsDynamicAssertion: a client can still send it on
+a definition without dynamic support (the Collate UI does, from a toggle left on), and rejecting it
+would break that client. Test generation already sends it only where it is supported.
+
+## Consequences
+
+Creates by POST (the UI and API clients), `createMany`, MCP `CreateTestCaseTool` and ODCS
+materialization get these checks. Ingestion (dbt, Great Expectations and the Python client's
+`create_or_update` create by PUT) and the CSV import get them once #35667 lands. ODCS
+materialization catches the `IllegalArgumentException` and records the rule as skipped. dbt and
+Great Expectations create their own definitions with no supportedDataTypes or optionValues, so
+their test cases pass; a definition from the test library is held to the types and options its
+author declared.
+
+`createMany` prepares every test of a batch before it stores any, so one rejected test fails the
+batch with no test case stored. A basic test suite that an earlier test's `prepare()` created is
+not rolled back, as with any other `prepare()` rejection in a batch: no transaction spans
+`createMany`.
+
+The ai-platform test generator
+applies the same reading before it sends a request (ADR:ai-platform/2026-10-12-generated-test-parameters-conform-to-their-definition),
+so a server rejection there means the two have drifted.

docs/decisions/2026-10-16-a-set-parameter-is-read-in-its-column-type.md

@@ -0,0 +1,36 @@
+# An allowed or forbidden set is read as a Python literal or JSON, in its column's type
+
+- **Status:** Accepted
+- **Revisions:** v1 2026-10-16 (initial)
+- **Deciders:** Shreyansh Verma
+- **Guard:** `tests/unit/observability/data_quality/validations/test_utils.py` (`parse_set_values`, `is_boolean`), `test_validations_databases.py` (`..._boolean_strings`, `..._boolean_json`, `..._missing_column`), `test_validations_datalake.py` (`test_string_booleans_on_a_datalake_column_stay_strings`)
+- **Related:** open-metadata/ai-platform#1650, #35274, #35276 (2.0 line); ADR:2026-10-12-test-cases-are-checked-against-their-definition-at-create
+
+## Context
+
+`columnValuesToBeInSet` and `columnValuesToBeNotInSet` parsed `allowedValues` / `forbiddenValues`
+with `literal_eval`. The UI writes a set as a JSON array of strings, and a generated test writes a
+JSON array, whose booleans and null are lowercase (`[false]`). `literal_eval` rejects JSON's
+lowercase `false` / `true` / `null`, which aborted the test, and on a BOOLEAN column it kept
+`"true"` / `"false"` as strings, which an engine such as Databricks refuses to compare with a
+boolean.
+
+## Decision
+
+`parse_set_values` reads a set as a Python literal and, when `literal_eval` rejects it, as JSON.
+`literal_eval` runs first, so a value it read before reads the same. A value neither reads still
+Aborts the test, as before. A value that parses but is not a list (a bare string or number) is now
+rejected with a ValueError, so it Aborts too. On a column typed BOOLEAN, `"true"` / `"false"` in
+any case become booleans; nothing else is mapped.
+
+The column type is the one OpenMetadata records for the column, which the SQL runner maps to
+SQLAlchemy (`SQAValidatorMixin.get_set_values_type`). The pandas runner returns none, so a datalake
+set keeps its values as written: `GenericDataFrameColumnParser` types a column of `"True"` /
+`"False"` strings as BOOLEAN.
+
+## Consequences
+
+A set test no longer aborts on a JSON set, and on a column typed BOOLEAN `"true"` / `"false"`
+compare as booleans. On a datalake the set is not typed, so its values compare as written. That is
+deliberate: typing it by `GenericDataFrameColumnParser` would turn a set of `"True"` / `"False"`
+strings into booleans, which the column's string values do not equal.

ingestion/src/metadata/data_quality/validations/base_test_handler.py

@@ -23,6 +23,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from decimal import Decimal
+from functools import partial
 from typing import (
     TYPE_CHECKING,
     TypedDict,
@@ -706,7 +707,7 @@
             return True
         return self.get_failure_threshold().unit is ThresholdUnit.PERCENTAGE
 
     def _run_results_with_row_count(self, metric: Metrics, column, **kwargs) -> dict:
         """Compute a violation count, and the row count denominator when one is needed
 
         The denominator is only computed when `_needs_row_count()` asks for it, so an
@@ -725,7 +726,7 @@
 
         return self._run_results_and_row_count(metric, column, **kwargs)
 
     def _run_results_and_row_count(self, metric: Metrics, column, **kwargs) -> dict:
         """Compute the violation count and the row count denominator together
 
         Two passes over the dataset by default. Validators whose runner can compute both
@@ -888,7 +889,7 @@
     def format_statistic_message(
         self,
         statistic: str,
         value,
         effective_bounds: tuple[float | None, float | None],
         matched: bool,
         dimension_info: DimensionInfo | None = None,
@@ -918,8 +919,8 @@
     def format_expected_value_message(
         self,
         statistic: str,
         value,
         expected,
         matched: bool,
         dimension_info: DimensionInfo | None = None,
     ) -> str:
@@ -1356,6 +1357,21 @@
             default=float("inf"),
         )
 
+    def get_set_param_value(self, param_name: str) -> list:
+        """An allowed or forbidden set, typed by the column it is checked against"""
+        return self.get_test_case_param_value(
+            self.test_case.parameterValues,  # type: ignore
+            param_name,
+            partial(utils.parse_set_values, column_type=self.get_set_values_type()),
+        )
+
+    def get_set_values_type(self):
+        """The column type a set is parsed against, or None to keep its values as written.
+
+        Implementation provided by SQAValidatorMixin or PandasValidatorMixin via MRO chain.
+        """
+        return super().get_set_values_type()  # type: ignore
+
     def get_bounds(self, min_param_name: str, max_param_name: str) -> tuple[float | None, float | None]:
         """Resolve the test case bounds and widen them by the failure threshold.