diff --git a/airbyte_cdk/test/models/scenario.py b/airbyte_cdk/test/models/scenario.py index a12accd9f..3d7627986 100644 --- a/airbyte_cdk/test/models/scenario.py +++ b/airbyte_cdk/test/models/scenario.py @@ -219,8 +219,8 @@ def __hash__(self) -> int: configured_catalog_path: Path | None = None empty_streams: list[AcceptanceTestEmptyStream] | None = None status: Literal["succeed", "failed", "exception"] | None = None - # Why the entry tests nothing. Read by the zero-scenario rule (a connector without test - # scenarios must declare why, airbyte-internal-issues#17131), which is not implemented yet. + # Why this scenario is not run: set on the placeholder scenarios that report a declared + # `bypass_reason` of `acceptance-test-config.yml` as skipped tests. bypass_reason: str | None = None # CAT's `basic_read` opt-out from validating records against the stream schemas. validate_schema: bool = True diff --git a/airbyte_cdk/test/standard_tests/_acceptance_test_config.py b/airbyte_cdk/test/standard_tests/_acceptance_test_config.py new file mode 100644 index 000000000..25dd9c4f8 --- /dev/null +++ b/airbyte_cdk/test/standard_tests/_acceptance_test_config.py @@ -0,0 +1,364 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""Read `acceptance-test-config.yml` into test scenarios and declared bypasses. + +Every connector that runs the standard tests (Python and manifest-only sources and destinations) +needs an `acceptance-test-config.yml`, and the file must yield at least one runnable scenario (a +config listed under a collected section) or declare why it has none: + +- a top-level `bypass_reason` bypasses the whole connector; a file holding only that key is the + opt-out for a connector without test configs; +- a `bypass_reason` on a collected section (`connection: {bypass_reason: ...}`) bypasses that + section, the layout the Connector Acceptance Tests used; +- a `bypass_reason` on an entry without `config_path` bypasses that entry. + +Only Java connectors, which are tested by their own Gradle suites, may have no file at all. + +Failure messages name sections, entries and keys, never values, as configs may hold secrets. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, TypeGuard + +import yaml +from pydantic import ValidationError + +from airbyte_cdk.test.models.scenario import ( + SCENARIO_SECTIONS, + ConnectorTestScenario, + entry_key_problems, +) +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG, METADATA_YAML + +JAVA_LANGUAGE_TAGS = frozenset({"language:java"}) +"""The `metadata.yaml` tags of connectors that may have no `acceptance-test-config.yml`.""" + +JAVA_BYPASS_REASON = ( + f"Java connector without {ACCEPTANCE_TEST_CONFIG}: Java connectors are tested by their " + "Gradle suites, not by the standard tests." +) + +BYPASS_ONLY_EXAMPLE = 'bypass_reason: "No test config in GSM for this connector"' + + +class AcceptanceTestConfigError(ValueError): + """`acceptance-test-config.yml` cannot be turned into test scenarios.""" + + +class MissingAcceptanceTestConfigError(AcceptanceTestConfigError, FileNotFoundError): + """A connector that runs the standard tests has no `acceptance-test-config.yml`.""" + + +@dataclass(frozen=True) +class ScenarioBypass: + """A declared reason why a part of `acceptance-test-config.yml` has no runnable config.""" + + reason: str + location: str + """Where the reason is declared, e.g. `acceptance_tests.connection.bypass_reason`.""" + sections: tuple[str, ...] = () + """The collected sections it bypasses; empty for a connector-wide bypass.""" + + +@dataclass +class CollectedScenarios: + """What `acceptance-test-config.yml` yields: runnable scenarios, bypasses, and why.""" + + config_path: Path + scenarios: list[ConnectorTestScenario] = field(default_factory=list) + """One scenario per runnable entry, before deduplication by config.""" + bypasses: list[ScenarioBypass] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + """Why each section or entry that is not a runnable config or a bypass produced nothing.""" + + def describe_empty(self, connector_name: str) -> str: + """The collection error for a connector with no runnable scenario and no bypass.""" + found = self.notes or ["(no sections)"] + return "\n".join( + [ + f"No test scenarios for connector '{connector_name}': {self.config_path} lists " + "no config the standard tests can run, and declares no `bypass_reason`. " + "Found:", + *(f" - {note}" for note in found), + "Add a `config_path` entry under one of " + f"{', '.join(f'`{section}`' for section in SCENARIO_SECTIONS)}, or declare why " + "the connector has none: a `bypass_reason` on each section " + "(`connection: {bypass_reason: ...}`), or a top-level `bypass_reason` for the " + "whole connector.", + ] + ) + + +def section_entries(section: Any) -> list[Any] | None: + """Return the entries of an `acceptance_tests` section, in either layout. + + The current layout lists them under `tests` (`spec: {tests: [...]}`); the legacy one makes the + section the list itself (`spec: [...]`). A section of any other shape has no entries. + """ + if isinstance(section, Mapping): + tests = section.get("tests") + return tests if isinstance(tests, list) else None + if isinstance(section, list): + return section + return None + + +def is_java_connector(connector_root: Path) -> bool: + """Whether `metadata.yaml` tags the connector as Java (`language:java`).""" + metadata_path = connector_root / METADATA_YAML + try: + metadata = yaml.safe_load(metadata_path.read_text()) + except (OSError, yaml.YAMLError): + return False + data = metadata.get("data") if isinstance(metadata, Mapping) else None + tags = data.get("tags") if isinstance(data, Mapping) else None + return isinstance(tags, list) and any(tag in JAVA_LANGUAGE_TAGS for tag in tags) + + +def load_acceptance_test_config(config_path: Path) -> dict[str, Any]: + """Load and validate the layout of `acceptance-test-config.yml`. + + Raises: + MissingAcceptanceTestConfigError: The file does not exist. + AcceptanceTestConfigError: The file is not valid YAML, is empty, uses the legacy + top-level `tests:` layout, or has neither `acceptance_tests` nor a top-level + `bypass_reason`. + """ + config_path = config_path.absolute() + if not config_path.exists(): + raise MissingAcceptanceTestConfigError( + f"Acceptance test config file not found at: {config_path}. Every Python and " + "manifest-only connector needs one for the standard tests (only Java connectors may " + "omit it). List the connector's test configs in it, or, if it has none, create the " + f"file with only a top-level `bypass_reason` that says why, e.g. " + f"`{BYPASS_ONLY_EXAMPLE}`." + ) + + try: + config = yaml.safe_load(config_path.read_text()) + except yaml.YAMLError as error: + raise AcceptanceTestConfigError(f"{config_path} is not valid YAML: {error}") from error + + if not isinstance(config, Mapping): + found = "is empty" if config is None else f"holds a {type(config).__name__}" + raise AcceptanceTestConfigError( + f"{config_path} {found}; expected a mapping with an `acceptance_tests` key, or only a " + f"top-level `bypass_reason` (`{BYPASS_ONLY_EXAMPLE}`)." + ) + + top_level_reason = config.get("bypass_reason") + if "bypass_reason" in config and not _is_reason(top_level_reason): + raise AcceptanceTestConfigError( + f"The top-level `bypass_reason` of {config_path} must be a non-empty string." + ) + + if "acceptance_tests" not in config: + if "tests" in config: + raise AcceptanceTestConfigError( + f"{config_path} uses the legacy layout with a top-level `tests:` key, which the " + "standard tests do not read. Migrate it: rename `tests:` to `acceptance_tests:` " + "and nest each section's entries under `tests` " + "(`connection: {tests: [{config_path: ...}]}`), keeping the other top-level keys." + ) + if top_level_reason is None: + raise AcceptanceTestConfigError( + f"{config_path} has no `acceptance_tests` key and no top-level `bypass_reason`. " + f"Found top-level keys: {', '.join(f'`{key}`' for key in config) or 'none'}." + ) + elif not isinstance(config["acceptance_tests"], Mapping): + raise AcceptanceTestConfigError( + f"`acceptance_tests` in {config_path} must be a mapping of sections " + f"(`connection: {{tests: [...]}}`), not a {type(config['acceptance_tests']).__name__}." + ) + + return dict(config) + + +def collect_scenarios(config: Mapping[str, Any], config_path: Path) -> CollectedScenarios: + """Collect the runnable scenarios and the declared bypasses of a loaded config. + + Every section of `SCENARIO_SECTIONS` is read, in that order, in the current layout + (`section: {tests: [...]}`) and the legacy list layout (`section: [...]`). An entry with a + `config_path` is a runnable scenario, except an IAM-role config, which the standard tests do + not support. An entry without `config_path` contributes its `bypass_reason`, if it has one. + A section `bypass_reason` counts only when the section lists no runnable config. + + Raises: + AcceptanceTestConfigError: An entry sets a key the standard tests do not read (see + `entry_key_problems`; every entry is checked, including those that are not run), an + entry is not a valid scenario (e.g. `empty_streams` as plain strings), or a top-level + `bypass_reason` is declared next to runnable configs, which would leave it unclear + whether the connector is tested. The first two list every problem of the file. + """ + collected = CollectedScenarios(config_path=config_path) + top_level_reason = config.get("bypass_reason") + if _is_reason(top_level_reason): + collected.bypasses.append(ScenarioBypass(reason=top_level_reason, location="bypass_reason")) + + acceptance_tests = config.get("acceptance_tests") or {} + _raise_if_invalid( + config_path, + [ + problem + for section_name in SCENARIO_SECTIONS + for entry in section_entries(acceptance_tests.get(section_name)) or [] + if isinstance(entry, Mapping) + for problem in entry_key_problems(entry, (section_name,)) + ], + ) + + strictness_level = config.get("test_strictness_level") + problems: list[str] = [] + for section_name in SCENARIO_SECTIONS: + if section_name in acceptance_tests: + _collect_section( + collected, + problems, + section_name, + acceptance_tests[section_name], + strictness_level=strictness_level, + ) + _raise_if_invalid(config_path, problems) + for section_name in acceptance_tests: + if section_name not in SCENARIO_SECTIONS: + collected.notes.append( + f"`acceptance_tests.{section_name}`: not a section the standard tests collect." + ) + + if collected.scenarios and _is_reason(top_level_reason): + raise AcceptanceTestConfigError( + f"{config_path} declares a top-level `bypass_reason`, which bypasses the whole " + f"connector, but also lists {len(collected.scenarios)} runnable config entries. Remove " + "the top-level `bypass_reason` to run them, or remove the entries." + ) + return collected + + +def _collect_section( + collected: CollectedScenarios, + problems: list[str], + section_name: str, + section: Any, + *, + strictness_level: Any, +) -> None: + where = f"acceptance_tests.{section_name}" + entries = section_entries(section) + section_reason = section.get("bypass_reason") if isinstance(section, Mapping) else None + + runnable = 0 + if entries is None: + if section is None: + collected.notes.append(f"`{where}`: empty.") + elif not isinstance(section, Mapping): + collected.notes.append( + f"`{where}`: a {type(section).__name__}, not a mapping with a `tests` list." + ) + elif section_reason is None: + keys = ", ".join(f"`{key}`" for key in section) or "none" + collected.notes.append( + f"`{where}`: no `tests` list and no `bypass_reason` (keys: {keys})." + ) + else: + prefix = f"{where}.tests" if isinstance(section, Mapping) else where + if not entries: + collected.notes.append(f"`{prefix}`: empty list.") + for index, entry in enumerate(entries): + location = f"{prefix}[{index}]" + if _collect_entry(collected, problems, section_name, location, entry, strictness_level): + runnable += 1 + + if section_reason is None: + return + if not _is_reason(section_reason): + collected.notes.append(f"`{where}.bypass_reason`: not a non-empty string, so not counted.") + elif runnable: + collected.notes.append( + f"`{where}.bypass_reason`: ignored, as the section lists {runnable} runnable config " + "entries." + ) + else: + collected.bypasses.append( + ScenarioBypass( + reason=section_reason, + location=f"{where}.bypass_reason", + sections=(section_name,), + ) + ) + + +def _collect_entry( + collected: CollectedScenarios, + problems: list[str], + section_name: str, + location: str, + entry: Any, + strictness_level: Any, +) -> bool: + """Collect one section entry; return whether it is a runnable scenario. + + An entry that is not a valid scenario adds its problems to `problems` instead. + """ + if not isinstance(entry, Mapping): + collected.notes.append(f"`{location}`: a {type(entry).__name__}, not a mapping.") + return False + + config_path = entry.get("config_path") + if not config_path: + reason = entry.get("bypass_reason") + if _is_reason(reason): + collected.bypasses.append( + ScenarioBypass( + reason=reason, + location=f"{location}.bypass_reason", + sections=(section_name,), + ) + ) + else: + keys = ", ".join(f"`{key}`" for key in entry) or "none" + collected.notes.append( + f"`{location}`: no `config_path` and no `bypass_reason` (keys: {keys})." + ) + return False + + if "iam_role" in str(config_path): + collected.notes.append( + f"`{location}`: an IAM-role config, which the standard tests do not support." + ) + return False + + fields = {key: value for key, value in entry.items() if key != "bypass_reason"} + try: + scenario = ConnectorTestScenario.model_validate( + { + **fields, + "sections": (section_name,), + "test_strictness_level": strictness_level, + } + ) + except ValidationError as error: + # Pydantic's own message repeats the offending input; name the field and rule only. + problems.extend( + f"`{section_name}` entry for config `{config_path}`: " + f"`{'.'.join(str(part) for part in detail['loc']) or '(entry)'}`: {detail['msg']}" + for detail in error.errors(include_input=False, include_url=False) + ) + return False + collected.scenarios.append(scenario) + return True + + +def _raise_if_invalid(config_path: Path, problems: list[str]) -> None: + """Fail with every problem found in `acceptance-test-config.yml`, one per line, if any.""" + if problems: + raise AcceptanceTestConfigError( + f"`{config_path}` is not valid for the standard tests:\n" + + "\n".join(f"- {problem}" for problem in problems) + ) + + +def _is_reason(value: Any) -> TypeGuard[str]: + return isinstance(value, str) and bool(value.strip()) diff --git a/airbyte_cdk/test/standard_tests/_spec_compatibility.py b/airbyte_cdk/test/standard_tests/_spec_compatibility.py index c62f34f5e..10ec13659 100644 --- a/airbyte_cdk/test/standard_tests/_spec_compatibility.py +++ b/airbyte_cdk/test/standard_tests/_spec_compatibility.py @@ -26,6 +26,8 @@ from packaging.version import InvalidVersion, Version from requests.adapters import HTTPAdapter, Retry +from airbyte_cdk.test.standard_tests._acceptance_test_config import section_entries + DeploymentMode = Literal["oss", "cloud"] REGISTRY_ENTRY_URL_TEMPLATE = ( @@ -1119,9 +1121,8 @@ def disabled_for_version(acceptance_test_config: Any) -> list[str]: acceptance_tests = acceptance_test_config.get("acceptance_tests") if not isinstance(acceptance_tests, Mapping): return [] - spec_section = acceptance_tests.get("spec") - tests = spec_section.get("tests") if isinstance(spec_section, Mapping) else spec_section - if not isinstance(tests, list): + tests = section_entries(acceptance_tests.get("spec")) + if tests is None: return [] versions: list[str] = [] diff --git a/airbyte_cdk/test/standard_tests/docker_base.py b/airbyte_cdk/test/standard_tests/docker_base.py index b66b11f27..c70e85181 100644 --- a/airbyte_cdk/test/standard_tests/docker_base.py +++ b/airbyte_cdk/test/standard_tests/docker_base.py @@ -17,7 +17,6 @@ import requests import yaml from boltons.typeutils import classproperty -from pydantic import ValidationError from airbyte_cdk.models import ( AirbyteStream, @@ -30,10 +29,14 @@ from airbyte_cdk.models.connector_metadata import MetadataFile from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.models import ConnectorTestScenario -from airbyte_cdk.test.models.scenario import ( - SCENARIO_SECTIONS, - ScenarioCommand, - entry_key_problems, +from airbyte_cdk.test.models.scenario import ScenarioCommand +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + JAVA_BYPASS_REASON, + CollectedScenarios, + ScenarioBypass, + collect_scenarios, + is_java_connector, + load_acceptance_test_config, ) from airbyte_cdk.test.standard_tests._assertions import ( assert_check_outcome, @@ -297,22 +300,15 @@ def is_destination_connector(cls) -> bool: def acceptance_test_config(cls) -> Any: """Get the contents of acceptance test config file. - Also perform some basic validation that the file has the expected structure. - """ - acceptance_test_config_path = cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG - if not acceptance_test_config_path.exists(): - raise FileNotFoundError( - f"Acceptance test config file not found at: {str(acceptance_test_config_path)}" - ) - - tests_config = yaml.safe_load(acceptance_test_config_path.read_text()) + Also perform some basic validation that the file has the expected structure: an + `acceptance_tests` mapping, or only a top-level `bypass_reason`. - if "acceptance_tests" not in tests_config: - raise ValueError( - f"Acceptance tests config not found in {acceptance_test_config_path}." - f" Found only: {str(tests_config)}." - ) - return tests_config + Raises: + MissingAcceptanceTestConfigError: The file does not exist (a `FileNotFoundError`). + AcceptanceTestConfigError: The file has an unusable layout, such as the legacy + top-level `tests:` key (a `ValueError`). + """ + return load_acceptance_test_config(cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG) @staticmethod def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTestScenario]: @@ -397,14 +393,21 @@ def _dedup_scenarios(scenarios: list[ConnectorTestScenario]) -> list[ConnectorTe return deduped_scenarios @classmethod - def _raise_if_invalid(cls, problems: list[str]) -> None: - """Fail collection with every problem found in `acceptance-test-config.yml`, if any.""" - if problems: - config_file = cls.get_connector_root_dir().absolute() / ACCEPTANCE_TEST_CONFIG - raise ValueError( - f"`{config_file}` is not valid for the standard tests:\n" - + "\n".join(f"- {problem}" for problem in problems) + def _collect_scenarios(cls) -> CollectedScenarios: + """Read the runnable scenarios and the declared bypasses of `acceptance-test-config.yml`. + + A Java connector without the file is reported as bypassed: Java connectors are tested by + their Gradle suites. Every other connector without it fails, with + `MissingAcceptanceTestConfigError`. + """ + connector_root = cls.get_connector_root_dir() + config_path = (connector_root / ACCEPTANCE_TEST_CONFIG).absolute() + if not config_path.exists() and is_java_connector(connector_root): + return CollectedScenarios( + config_path=config_path, + bypasses=[ScenarioBypass(reason=JAVA_BYPASS_REASON, location="metadata.yaml")], ) + return collect_scenarios(cls.acceptance_test_config, config_path) @classmethod def get_scenarios( @@ -417,84 +420,39 @@ def get_scenarios( against the spec, `connection` runs `check` against the declared `status`, `discovery` runs `discover`, and `basic_read` and `full_refresh` run a full-refresh `read` of the streams of the config's `configured_catalog_path`, or of every discovered stream when it - declares none (`SECTION_COMMANDS`). The `incremental` section is not collected. + declares none (`SECTION_COMMANDS`). The `incremental` section is not collected. A section + in the legacy list layout (`spec: [...]`) is read as its `tests` list. + + The file is required (except for Java connectors), and a legacy top-level `tests:` key + fails; see `load_acceptance_test_config`. An empty result is not an error here: + `pytest_generate_tests` fails collection when there is no scenario and no declared + bypass (`get_scenario_bypasses`). An entry of a collected section that sets a key the standard tests do not read, or one they fill in themselves (`sections`, `test_strictness_level`), fails collection, with one line per key naming the key, the section and the config, including entries that are - skipped (no `config_path`, or an `iam_role` config). See `entry_key_problems`. + skipped (no `config_path`, or an `iam_role` config). See `collect_scenarios`. This has to be a separate function because pytest does not allow parametrization of fixtures with arguments from the test class itself. """ - try: - all_tests_config = cls.acceptance_test_config - except FileNotFoundError as e: - # Destinations sometimes do not have an acceptance tests file. - warnings.warn( - f"Acceptance test config file not found: {e!s}. No scenarios will be loaded.", - category=UserWarning, - stacklevel=1, - ) - return [] + return cls._dedup_scenarios(cls._collect_scenarios().scenarios) - test_strictness_level = all_tests_config.get("test_strictness_level") - entries: list[tuple[str, dict[str, Any]]] = [] - for category in SCENARIO_SECTIONS: - if ( - category not in all_tests_config["acceptance_tests"] - or "tests" not in all_tests_config["acceptance_tests"][category] - ): - continue - - for test in all_tests_config["acceptance_tests"][category]["tests"]: - if isinstance(test, dict): - entries.append((category, test)) - - # Every entry is checked, including those skipped below, so that a key that is not read - # fails collection wherever it is set. - cls._raise_if_invalid( - [ - problem - for category, test in entries - for problem in entry_key_problems(test, (category,)) - ] - ) - - problems: list[str] = [] - test_scenarios: list[ConnectorTestScenario] = [] - for category, test in entries: - if "config_path" not in test: - # Skip tests without a config_path - continue - - if "iam_role" in test["config_path"]: - # We skip iam_role tests for now, as they are not supported in the test suite. - continue - - try: - scenario = ConnectorTestScenario.model_validate( - { - **test, - "sections": (category,), - "test_strictness_level": test_strictness_level, - } - ) - except ValidationError as error: - # Reported without the input, which can hold config values. - problems.extend( - f"`{category}` entry for config `{test['config_path']}`: " - f"`{'.'.join(str(part) for part in detail['loc'])}`: {detail['msg']}" - for detail in error.errors() - ) - continue - - test_scenarios.append(scenario) - cls._raise_if_invalid(problems) + @classmethod + def get_scenario_bypasses(cls) -> list[ScenarioBypass]: + """The `bypass_reason`s that `acceptance-test-config.yml` declares for collected parts. - deduped_test_scenarios = cls._dedup_scenarios(test_scenarios) + A top-level `bypass_reason` bypasses the connector; one on a collected section with no + runnable config, or on an entry without `config_path`, bypasses that section or entry. + `pytest_generate_tests` reports them as skipped tests, and accepts a connector with no + scenario only when it declares at least one. + """ + return cls._collect_scenarios().bypasses - return deduped_test_scenarios + @classmethod + def describe_empty_scenarios(cls) -> str: + """Why `acceptance-test-config.yml` yields neither a scenario nor a bypass.""" + return cls._collect_scenarios().describe_empty(cast(str, cls.connector_name)) @classmethod def _connector_image( diff --git a/airbyte_cdk/test/standard_tests/pytest_hooks.py b/airbyte_cdk/test/standard_tests/pytest_hooks.py index fa905e715..51567f1f5 100644 --- a/airbyte_cdk/test/standard_tests/pytest_hooks.py +++ b/airbyte_cdk/test/standard_tests/pytest_hooks.py @@ -13,11 +13,17 @@ ``` """ +from dataclasses import dataclass from typing import Literal, cast import pytest from airbyte_cdk.test.models.scenario import ConnectorTestScenario +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + AcceptanceTestConfigError, + ScenarioBypass, +) +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG @pytest.fixture @@ -195,8 +201,36 @@ class TestMyConnector(ConnectorTestSuiteBase): "Please define the 'scenarios' attribute in the test class." ) - # Get the scenarios defined or discovered in the test class - scenarios = test_class.get_scenarios() + # Get the scenarios defined or discovered in the test class, and the declared bypasses. + config_error: str | None = None + try: + scenarios = test_class.get_scenarios() + get_bypasses = getattr(test_class, "get_scenario_bypasses", None) + bypasses: list[ScenarioBypass] = get_bypasses() if get_bypasses else [] + except AcceptanceTestConfigError as error: + config_error = str(error) + if config_error is not None: + # Failed outside the `except` block, so pytest does not print the error twice. + pytest.fail(config_error, pytrace=False) + + if not scenarios and not bypasses: + # Parametrizing with an empty list would skip every scenario test and exit 0. + describe = getattr(test_class, "describe_empty_scenarios", None) + pytest.fail( + describe() + if describe + else f"Test class {test_class.__name__} has no test scenarios.", + pytrace=False, + ) + + bypass_groups = _group_bypasses(bypasses) + if bypass_groups: + connector_name = getattr(test_class, "connector_name", test_class.__name__) + metafunc.config.stash.setdefault(_BYPASS_REPORT_KEY, {})[str(connector_name)] = ( + f"{connector_name}: {ACCEPTANCE_TEST_CONFIG} declares {len(bypasses)} bypass(es), " + "reported as skipped tests: " + + "; ".join(f"{group.scope}: {group.reason}" for group in bypass_groups) + ) # Create pytest.param objects with special marks as needed parametrized_scenarios = [ @@ -206,13 +240,73 @@ class TestMyConnector(ConnectorTestSuiteBase): ) for scenario in scenarios ] + # One skipped param per distinct bypass reason, so the CI summary shows the reason. + bypass_params = [ + pytest.param( + ConnectorTestScenario(bypass_reason=group.reason, sections=group.sections), + marks=pytest.mark.skip( + reason=f"{ACCEPTANCE_TEST_CONFIG} bypasses {group.scope}: {group.reason}" + ), + ) + for group in bypass_groups + ] - # Parametrize the 'scenario' argument with the scenarios + # Parametrize the 'scenario' argument with the scenarios, then the bypasses metafunc.parametrize( "scenario", - parametrized_scenarios, - ids=_scenario_test_ids(scenarios), + parametrized_scenarios + bypass_params, + ids=_scenario_test_ids(scenarios) + [group.test_id for group in bypass_groups], + ) + + +_BYPASS_REPORT_KEY = pytest.StashKey[dict[str, str]]() +"""The collection report line of each connector whose config declares bypasses.""" + + +def pytest_report_collectionfinish(config: pytest.Config) -> list[str]: + """Print one line per connector that declares bypasses, after the collection summary.""" + return [line for _, line in sorted(config.stash.get(_BYPASS_REPORT_KEY, {}).items())] + + +@dataclass(frozen=True) +class _BypassGroup: + """The bypasses that share one reason, reported as one skipped test per scenario test.""" + + reason: str + scope: str + """What is bypassed: `the connector`, or the sections and entries, e.g. `connection.tests[1]`.""" + sections: tuple[str, ...] + """The bypassed sections; empty when the whole connector is bypassed.""" + test_id: str + + +def _group_bypasses(bypasses: list[ScenarioBypass]) -> list[_BypassGroup]: + """Group bypasses by reason: the usual config repeats one reason on every section.""" + by_reason: dict[str, list[ScenarioBypass]] = {} + for bypass in bypasses: + by_reason.setdefault(bypass.reason.strip(), []).append(bypass) + + groups: list[_BypassGroup] = [] + for reason, group in by_reason.items(): + connector_wide = any(not bypass.sections for bypass in group) + sections = ( + () + if connector_wide + else tuple(dict.fromkeys(section for bypass in group for section in bypass.sections)) + ) + scope = ( + "the connector" + if connector_wide + else ", ".join( + bypass.location.removeprefix("acceptance_tests.").removesuffix(".bypass_reason") + for bypass in group + ) ) + test_id = "bypassed-" + ("connector" if connector_wide else "-".join(sections)) + if any(existing.test_id == test_id for existing in groups): + test_id = f"{test_id}-{len(groups) + 1}" + groups.append(_BypassGroup(reason=reason, scope=scope, sections=sections, test_id=test_id)) + return groups def _scenario_test_ids(scenarios: list[ConnectorTestScenario]) -> list[str]: diff --git a/cdk-migrations.md b/cdk-migrations.md index 31426600c..0d92b67c8 100644 --- a/cdk-migrations.md +++ b/cdk-migrations.md @@ -2,6 +2,40 @@ ## Upgrading to the next minor release (unreleased) +### A connector without test scenarios fails unless it says why + +This release changes how the Standard Tests (`airbyte-cdk connector test` and `airbyte-cdk image test`) read `acceptance-test-config.yml`. There is no change to how connectors run. Previously a connector whose file listed no config the tests could run, or that had no file at all, collected an empty scenario list: every test that takes a scenario (`test_check`, `test_discover`, `test_basic_read` and their Docker counterparts) was skipped as "got empty parameter set" and the run passed. Now: + +| `acceptance-test-config.yml` | Before | Now | +|---|---|---| +| Lists at least one config under `spec`, `connection`, `discovery`, `basic_read` or `full_refresh` | Scenarios run | Scenarios run. Declared bypasses (below) are also reported as skipped tests. | +| Lists no runnable config, but declares a `bypass_reason` | Every scenario test skipped as an empty parameter set | One skipped test per scenario test and distinct reason, whose skip message is the declared reason, plus one line after collection. | +| Lists no runnable config and declares no `bypass_reason` | Every scenario test skipped as an empty parameter set | Collection fails, naming the file and why each section and entry produced nothing. | +| Missing (Python or manifest-only connector) | `UserWarning`, every scenario test skipped | Collection fails. | +| Missing (Java connector, `language:java` in `metadata.yaml`) | `UserWarning`, every scenario test skipped | Reported as a bypass: Java connectors are tested by their Gradle suites. | +| Legacy top-level `tests:` key instead of `acceptance_tests:` | Collection failed with "Acceptance tests config not found" and the whole parsed file | Collection fails with a message naming the file and the migration. | +| A section written as a plain list (`connection: [...]`, the legacy layout) | Silently ignored | Read as the section's `tests` list. Its entries are checked like the entries under `tests`: a key the Standard Tests do not read, or an invalid value, fails collection. | + +A `bypass_reason` counts in three places: + +- **Top level** (`bypass_reason: "..."` next to `acceptance_tests`): bypasses the whole connector. A file holding only this key is the opt-out for a connector without test configs. It cannot be combined with runnable configs, which fails collection. +- **On a section** (`connection: {bypass_reason: "..."}`), the layout the Connector Acceptance Tests used: bypasses that section, when it lists no runnable config. A section `bypass_reason` next to runnable configs is ignored, as before. +- **On an entry without `config_path`** (`- bypass_reason: "..."` in a section's `tests` list): bypasses that entry. An entry with a `config_path` always runs; its `bypass_reason` is ignored. + +Only the sections the Standard Tests collect count. A `bypass_reason` under `incremental`, which is not collected, does not, and neither does an IAM-role config (`config_path` containing `iam_role`), which the Standard Tests do not support. `DockerConnectorTestSuite.get_scenario_bypasses()` returns the bypasses, and `ConnectorTestScenario.bypass_reason` holds the reason on the placeholder scenario of each skipped test. `get_scenarios()` still returns only the runnable scenarios, and now raises `MissingAcceptanceTestConfigError` (a `FileNotFoundError`) or `AcceptanceTestConfigError` (a `ValueError`) instead of warning or failing with a generic `ValueError`. + +Migration steps: + +- A connector without `acceptance-test-config.yml`: list its test configs in a new file, or create the file with only a top-level `bypass_reason` that says why it has none, e.g. `bypass_reason: "No test config in GSM for this connector"`. Earlier CDK versions reject a file without `acceptance_tests`, so a file that must also load under them adds `acceptance_tests: {}`. +- A file whose sections list no `config_path`, such as one holding only a `spec_path` entry or only a `backward_compatibility_tests_config` waiver: add the configs, or declare a `bypass_reason` at the top level or on each section. +- A file with a top-level `tests:` key: rename it to `acceptance_tests:` and nest each section's entries under `tests` (`connection: {tests: [{config_path: ...}]}`). +- A section written as a plain list now runs its configs. Make sure they pass, or move the ones that cannot run out and declare a `bypass_reason` for them. +- `empty_streams` entries must be mappings (`- name: `), not plain strings. + +Rationale: an empty scenario list looked like a green test run, so a connector whose configs were never collected, or whose configuration silently degraded to nothing, reported coverage it did not have. The opt-out is explicit and machine-readable so that the connectors that legitimately have no test configs say so in the test report instead of being indistinguishable from a broken configuration. + +## Upgrading to 7.39.0 + ### `acceptance-test-config.yml` keys the Standard Tests do not read fail collection `ConnectorTestScenario` declared `expect_records` (with `exact_order`), `timeout_seconds` and `file_types`, and no test read them: they parsed, and the check they configured in the retired Connector Acceptance Tests (CAT) never ran. They are removed. Any other key that the model did not declare was dropped without a word, so a misspelled key (`empty_stream` for `empty_streams`) was ignored as well. diff --git a/unit_tests/test/test_empty_scenarios.py b/unit_tests/test/test_empty_scenarios.py new file mode 100644 index 000000000..cf3ab51f7 --- /dev/null +++ b/unit_tests/test/test_empty_scenarios.py @@ -0,0 +1,510 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +"""An `acceptance-test-config.yml` with no runnable scenario must fail, unless it says why. + +Collection fails when the config yields no runnable scenario and declares no `bypass_reason`; +declared bypasses are reported as skipped tests with their reason. A missing file fails except +for Java connectors, a legacy top-level `tests:` key fails with the migration, and a section in +the legacy list layout is read as its `tests` list. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from airbyte_cdk.test.standard_tests._acceptance_test_config import ( + JAVA_BYPASS_REASON, + AcceptanceTestConfigError, + MissingAcceptanceTestConfigError, + ScenarioBypass, + collect_scenarios, + load_acceptance_test_config, + section_entries, +) +from airbyte_cdk.test.standard_tests._spec_compatibility import disabled_for_version +from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite +from airbyte_cdk.utils.connector_paths import ACCEPTANCE_TEST_CONFIG + +pytest_plugins = ["pytester"] + +BUILDER_REASON = "This is a builder contribution, and we do not have secrets at this time" + +# The shape of the ~290 builder contributions without secrets: a `spec_path` entry, and a +# `bypass_reason` on every other section. +BYPASSED_SECTIONS_CONFIG: dict[str, Any] = { + "connector_image": "airbyte/source-test:dev", + "acceptance_tests": { + "spec": {"tests": [{"spec_path": "manifest.yaml"}]}, + "connection": {"bypass_reason": BUILDER_REASON}, + "discovery": {"bypass_reason": BUILDER_REASON}, + "basic_read": {"bypass_reason": BUILDER_REASON}, + "incremental": {"bypass_reason": BUILDER_REASON}, + "full_refresh": {"bypass_reason": BUILDER_REASON}, + }, +} + +# No runnable config and no bypass of a collected section: only `incremental` declares one. +ZERO_SCENARIOS_CONFIG: dict[str, Any] = { + "acceptance_tests": { + "spec": {"tests": [{"spec_path": "manifest.yaml"}]}, + "connection": {"tests": [{"config_path": "secrets/config_iam_role.json"}]}, + "basic_read": {"tests": []}, + "incremental": {"bypass_reason": "This connector does not implement incremental sync"}, + } +} + +RUNNABLE_CONFIG: dict[str, Any] = { + "acceptance_tests": { + "connection": {"tests": [{"config_path": "secrets/config.json", "status": "succeed"}]}, + "basic_read": {"bypass_reason": "The sandbox has no data"}, + } +} + + +def _write_connector( + root: Path, + config: dict[str, Any] | str | None, + *, + language: str = "manifest-only", +) -> Path: + root.mkdir(parents=True, exist_ok=True) + (root / "metadata.yaml").write_text( + yaml.safe_dump({"data": {"tags": [f"language:{language}"]}}) + ) + if config is not None: + text = config if isinstance(config, str) else yaml.safe_dump(config) + (root / ACCEPTANCE_TEST_CONFIG).write_text(text) + return root + + +def _suite(root: Path) -> type[DockerConnectorTestSuite]: + class _Suite(DockerConnectorTestSuite): + @classmethod + def get_connector_root_dir(cls) -> Path: + return root + + return _Suite + + +# --- Reading the config ----------------------------------------------------------------------- + + +def test_bypassed_sections_yield_no_scenario_and_one_bypass_per_collected_section( + tmp_path: Path, +) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", BYPASSED_SECTIONS_CONFIG)) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason=BUILDER_REASON, + location=f"acceptance_tests.{section}.bypass_reason", + sections=(section,), + ) + for section in ("connection", "discovery", "basic_read", "full_refresh") + ], "the `incremental` bypass is not a bypass of anything the standard tests run" + + +def test_zero_scenarios_without_bypass_lists_why_each_part_produced_nothing( + tmp_path: Path, +) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", ZERO_SCENARIOS_CONFIG)) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [] + message = suite.describe_empty_scenarios() + assert message.startswith("No test scenarios for connector 'source-test': ") + assert str(tmp_path / "source-test" / ACCEPTANCE_TEST_CONFIG) in message + for expected in [ + "`acceptance_tests.spec.tests[0]`: no `config_path` and no `bypass_reason` " + "(keys: `spec_path`).", + "`acceptance_tests.connection.tests[0]`: an IAM-role config", + "`acceptance_tests.basic_read.tests`: empty list.", + "`acceptance_tests.incremental`: not a section the standard tests collect.", + "a top-level `bypass_reason` for the whole connector", + ]: + assert expected in message + + +def test_entry_without_config_path_declares_a_bypass(tmp_path: Path) -> None: + config = { + "acceptance_tests": {"basic_read": {"tests": [{"bypass_reason": "Read needs a paid plan"}]}} + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason="Read needs a paid plan", + location="acceptance_tests.basic_read.tests[0].bypass_reason", + sections=("basic_read",), + ) + ] + + +def test_section_bypass_is_ignored_next_to_runnable_configs(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "connection": { + "bypass_reason": "stale", + "tests": [{"config_path": "secrets/config.json"}], + } + } + } + collected = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + assert [str(s.config_path) for s in collected.scenarios] == ["secrets/config.json"] + assert collected.bypasses == [] + assert collected.notes == [ + "`acceptance_tests.connection.bypass_reason`: ignored, as the section lists 1 runnable " + "config entries." + ] + + +def test_runnable_entry_does_not_keep_its_bypass_reason(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "connection": {"tests": [{"config_path": "secrets/config.json", "bypass_reason": "x"}]} + } + } + [scenario] = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG).scenarios + assert scenario.bypass_reason is None, "a config with a `config_path` is run, not bypassed" + + +@pytest.mark.parametrize("reason", ["", " ", 1, True]) +def test_blank_or_non_string_section_bypass_does_not_count(tmp_path: Path, reason: Any) -> None: + config = {"acceptance_tests": {"connection": {"bypass_reason": reason}}} + collected = collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + assert collected.bypasses == [] + assert collected.notes == [ + "`acceptance_tests.connection.bypass_reason`: not a non-empty string, so not counted." + ] + + +def test_top_level_bypass_only_file_bypasses_the_connector(tmp_path: Path) -> None: + root = _write_connector( + tmp_path / "destination-test", 'bypass_reason: "No test config in GSM for this connector"\n' + ) + suite = _suite(root) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass(reason="No test config in GSM for this connector", location="bypass_reason") + ] + assert disabled_for_version(suite.acceptance_test_config) == [] + + +def test_top_level_bypass_next_to_runnable_configs_fails(tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", {**RUNNABLE_CONFIG, "bypass_reason": "x"}) + + with pytest.raises(AcceptanceTestConfigError, match="top-level `bypass_reason`.*1 runnable"): + _suite(root).get_scenarios() + + +def test_top_level_bypass_with_bypassed_sections_is_accepted(tmp_path: Path) -> None: + """A spec waiver file can carry the connector-wide bypass next to its `spec` entry.""" + config = { + "bypass_reason": "No test config in GSM for this connector", + "acceptance_tests": { + "spec": { + "tests": [{"backward_compatibility_tests_config": {"disable_for_version": "1.0.0"}}] + } + }, + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + assert [bypass.location for bypass in suite.get_scenario_bypasses()] == ["bypass_reason"] + assert disabled_for_version(suite.acceptance_test_config) == ["1.0.0"] + + +def test_invalid_entry_fails_with_its_location_and_without_values(tmp_path: Path) -> None: + """A legacy list section listing `empty_streams` as plain strings, as source-netsuite did.""" + config = { + "acceptance_tests": { + "basic_read": [ + {"config_path": "secrets/config.json", "empty_streams": ["secret_stream_name"]} + ] + } + } + with pytest.raises(AcceptanceTestConfigError) as error: + collect_scenarios(config, tmp_path / ACCEPTANCE_TEST_CONFIG) + + message = str(error.value) + assert message.startswith(f"`{tmp_path / ACCEPTANCE_TEST_CONFIG}` is not valid for the") + assert ( + "- `basic_read` entry for config `secrets/config.json`: `empty_streams.0`: Input should " + "be a valid dictionary" in message + ) + assert "secret_stream_name" not in message, "input values are not printed" + + +# --- Legacy layouts --------------------------------------------------------------------------- + + +def test_legacy_list_sections_are_read_as_tests_lists(tmp_path: Path) -> None: + config = { + "acceptance_tests": { + "spec": [{"spec_path": "manifest.yaml"}], + "connection": [ + {"config_path": "secrets/config.json", "status": "succeed"}, + {"config_path": "integration_tests/invalid_config.json", "status": "failed"}, + ], + "basic_read": [ + {"config_path": "secrets/config.json", "empty_streams": [{"name": "a"}]} + ], + "full_refresh": [{"bypass_reason": "Covered by basic_read"}], + } + } + suite = _suite(_write_connector(tmp_path / "source-test", config)) + + scenarios = {str(s.config_path): s for s in suite.get_scenarios()} + assert sorted(scenarios) == ["integration_tests/invalid_config.json", "secrets/config.json"] + assert scenarios["secrets/config.json"].sections == ("connection", "basic_read") + assert scenarios["integration_tests/invalid_config.json"].status == "failed" + assert suite.get_scenario_bypasses() == [ + ScenarioBypass( + reason="Covered by basic_read", + location="acceptance_tests.full_refresh[0].bypass_reason", + sections=("full_refresh",), + ) + ] + + +@pytest.mark.parametrize( + "section, expected", + [ + ({"tests": [{"a": 1}]}, [{"a": 1}]), + ([{"a": 1}], [{"a": 1}]), + ({"bypass_reason": "x"}, None), + ({"tests": None}, None), + (None, None), + ("tests", None), + ], +) +def test_section_entries_reads_both_layouts(section: Any, expected: Any) -> None: + assert section_entries(section) == expected + + +def test_legacy_top_level_tests_key_fails_with_the_migration(tmp_path: Path) -> None: + config = { + "connector_image": "airbyte/source-test:dev", + "tests": {"spec": [{"spec_path": "manifest.yaml"}]}, + } + root = _write_connector(tmp_path / "source-test", config) + + with pytest.raises(AcceptanceTestConfigError) as error: + _suite(root).get_scenarios() + message = str(error.value) + assert str(root / ACCEPTANCE_TEST_CONFIG) in message + assert "legacy layout with a top-level `tests:` key" in message + assert "rename `tests:` to `acceptance_tests:`" in message + assert "airbyte/source-test:dev" not in message, "values are not printed" + + +@pytest.mark.parametrize( + "text, match", + [ + ("", "is empty"), + ("# only comments\n", "is empty"), + ("- a\n", "holds a list"), + ("acceptance_tests: [a]\n", "must be a mapping of sections"), + ("connector_image: x\n", "no `acceptance_tests` key and no top-level `bypass_reason`"), + ("bypass_reason: ''\n", "must be a non-empty string"), + ("acceptance_tests: [\n", "is not valid YAML"), + ], +) +def test_unusable_config_layouts_fail(tmp_path: Path, text: str, match: str) -> None: + path = tmp_path / ACCEPTANCE_TEST_CONFIG + path.write_text(text) + + with pytest.raises(AcceptanceTestConfigError, match=match): + load_acceptance_test_config(path) + + +# --- Missing file ----------------------------------------------------------------------------- + + +@pytest.mark.parametrize("language", ["python", "manifest-only"]) +def test_missing_config_fails_for_connectors_that_run_the_standard_tests( + tmp_path: Path, language: str +) -> None: + root = _write_connector(tmp_path / "destination-test", None, language=language) + + with pytest.raises(MissingAcceptanceTestConfigError, match="only a top-level `bypass_reason`"): + _suite(root).get_scenarios() + + +def test_missing_config_error_is_a_file_not_found_error(tmp_path: Path) -> None: + """Callers that tolerated a missing file by catching `FileNotFoundError` keep working.""" + root = _write_connector(tmp_path / "source-test", None) + with pytest.raises(FileNotFoundError): + _suite(root).acceptance_test_config + + +def test_missing_config_is_a_declared_bypass_for_java_connectors(tmp_path: Path) -> None: + suite = _suite(_write_connector(tmp_path / "source-test", None, language="java")) + + assert suite.get_scenarios() == [] + assert suite.get_scenario_bypasses() == [ + ScenarioBypass(reason=JAVA_BYPASS_REASON, location="metadata.yaml") + ] + + +# --- Collection (pytest_generate_tests) ------------------------------------------------------- + + +def _run_collection(pytester: pytest.Pytester, root: Path, *args: str) -> pytest.RunResult: + pytester.makepyfile( + test_standard=textwrap.dedent( + f""" + from pathlib import Path + + from airbyte_cdk.test.standard_tests.docker_base import DockerConnectorTestSuite + + + class TestSuite(DockerConnectorTestSuite): + @classmethod + def get_connector_root_dir(cls) -> Path: + return Path({str(root)!r}) + + def test_with_scenario(self, scenario) -> None: + pass + """ + ) + ) + return pytester.runpytest_inprocess( + "-p", + "airbyte_cdk.test.standard_tests.pytest_hooks", + "-p", + "no:cacheprovider", + "-k", + "test_with_scenario", + "-rs", + *args, + ) + + +def test_collection_fails_when_no_scenario_and_no_bypass( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", ZERO_SCENARIOS_CONFIG) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines( + [ + "*No test scenarios for connector 'source-test': *acceptance-test-config.yml lists no " + "config the standard tests can run, and declares no `bypass_reason`. Found:", + "*`acceptance_tests.spec.tests[[]0[]]`: no `config_path` and no `bypass_reason`*", + "*1 error*", + ] + ) + assert "Traceback" not in result.stdout.str(), "the message is reported without a traceback" + + +def test_collection_fails_on_a_missing_config(pytester: pytest.Pytester, tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", None) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(["*Acceptance test config file not found at: *", "*1 error*"]) + output = result.stdout.str() + assert output.count("Acceptance test config file not found at:") == 1, "reported once" + assert "During handling of the above exception" not in output + + +def test_collection_fails_on_the_legacy_top_level_tests_key( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", {"tests": {"spec": []}}) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(["*legacy layout with a top-level `tests:` key*", "*1 error*"]) + + +def test_bypasses_are_reported_as_skipped_tests_and_at_collection( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", BYPASSED_SECTIONS_CONFIG) + + result = _run_collection(pytester, root) + + assert result.ret == pytest.ExitCode.OK + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines( + [ + "source-test: acceptance-test-config.yml declares 4 bypass(es), reported as skipped " + "tests: connection, discovery, basic_read, full_refresh: " + BUILDER_REASON, + "SKIPPED [[]1[]] *: acceptance-test-config.yml bypasses connection, discovery, " + "basic_read, full_refresh: " + BUILDER_REASON, + ] + ) + + +def test_bypass_params_have_readable_ids(pytester: pytest.Pytester, tmp_path: Path) -> None: + root = _write_connector(tmp_path / "source-test", RUNNABLE_CONFIG) + + result = _run_collection(pytester, root, "--collect-only", "-q") + + result.stdout.fnmatch_lines( + [ + "*test_with_scenario[[]'config' Test Scenario[]]", + "*test_with_scenario[[]bypassed-basic_read[]]", + ] + ) + + +def test_connector_wide_bypass_is_one_skipped_test( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + root = _write_connector(tmp_path / "source-test", "bypass_reason: No sandbox account\n") + + result = _run_collection(pytester, root, "-v") + + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines( + [ + "*test_with_scenario[[]bypassed-connector[]] SKIPPED*", + "SKIPPED [[]1[]] *: acceptance-test-config.yml bypasses the connector: " + "No sandbox account", + ] + ) + + +def test_bypasses_are_grouped_by_reason_and_name_their_entries( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + config = { + "acceptance_tests": { + "connection": { + "tests": [ + {"config_path": "integration_tests/invalid_config.json", "status": "failed"}, + {"bypass_reason": "The GSM config fails check"}, + ] + }, + "basic_read": {"bypass_reason": "The GSM config fails check"}, + "full_refresh": {"bypass_reason": "Covered by basic_read"}, + } + } + root = _write_connector(tmp_path / "source-test", config) + + result = _run_collection(pytester, root, "-v") + + result.assert_outcomes(passed=1, skipped=2) + result.stdout.fnmatch_lines( + [ + "source-test: acceptance-test-config.yml declares 3 bypass(es), reported as skipped " + "tests: connection.tests[[]1[]], basic_read: The GSM config fails check; " + "full_refresh: Covered by basic_read", + "*test_with_scenario[[]bypassed-connection-basic_read[]] SKIPPED*", + "*test_with_scenario[[]bypassed-full_refresh[]] SKIPPED*", + ] + ) diff --git a/unit_tests/test/test_scenario_keys.py b/unit_tests/test/test_scenario_keys.py index 1d58f9999..a59eaa33f 100644 --- a/unit_tests/test/test_scenario_keys.py +++ b/unit_tests/test/test_scenario_keys.py @@ -34,9 +34,7 @@ # Fields with no reader yet, each with the change that adds the reader. The guard below fails once # such a field is read, so the entry is removed by the change that starts reading it. -AWAITING_READER: dict[str, str] = { - "bypass_reason": "the zero-scenario rule, airbyte-internal-issues#17131", -} +AWAITING_READER: dict[str, str] = {} # --- Guard: every field is read outside the model --------------------------------------------