diff --git a/Dockerfile b/Dockerfile index d9eb986295..b0fe584921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,9 @@ RUN poetry config virtualenvs.create false \ # Build and install the package RUN pip install dist/*.whl +# pandas is imported lazily by the CDK, so verify it is importable at build time +RUN python -c "import pandas" + # Recreate the original structure RUN mkdir -p source_declarative_manifest \ && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \ diff --git a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py index f9a655db0c..145885e14c 100644 --- a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py @@ -10,7 +10,6 @@ from math import nan from typing import Any, Dict, Iterable, Mapping, Optional, Tuple -import pandas as pd import requests from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -139,6 +138,13 @@ def _read_with_chunks( ValueError: If an IO/Error occurs while reading the temporary data. """ + # Imported lazily: pandas (and numpy) are slow to import and only needed for CSV parsing. + try: + import pandas as pd + except ImportError: + os.remove(path) + raise + try: with open(path, "r", encoding=file_encoding) as data: # When preserving NA values, disable pandas' default NA parsing so strings like "NA"/"N/A" are kept as-is, diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py new file mode 100644 index 0000000000..d916e7a5a2 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. + +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = """ +import json +import sys +from unittest.mock import patch + +import requests_mock +import yaml + +from airbyte_cdk.cli.source_declarative_manifest import _run + +manifest_path, config_path = sys.argv[1:] +with open(manifest_path) as manifest_file: + manifest = yaml.safe_load(manifest_file) + +with patch( + "airbyte_cdk.cli.source_declarative_manifest._run._is_local_manifest_command", + return_value=True, +): + with patch( + "airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file", + return_value=manifest, + ): + with requests_mock.Mocker() as mocker: + mocker.get( + "https://pokeapi.co/api/v2/pokemon/blastoise", + json={"name": "blastoise"}, + ) + _run.handle_command(["spec"]) + _run.handle_command(["check", "--config", config_path]) + _run.handle_command(["discover", "--config", config_path]) + +print("LOADED_HEAVY_MODULES=" + json.dumps(sorted(m for m in ("pandas", "numpy") if m in sys.modules))) +""" + + +def test_spec_check_discover_do_not_import_pandas_or_numpy( + valid_local_manifest, valid_local_config_file +) -> None: + """Declarative/SDM start-up must not import pandas or numpy. Import heavy libraries inside the function that needs them; `python -X importtime -c "import airbyte_cdk.cli.source_declarative_manifest._run"` shows which module pulls one in.""" + result = subprocess.run( + [ + sys.executable, + "-c", + SCRIPT, + str(valid_local_manifest), + str(valid_local_config_file), + ], + capture_output=True, + text=True, + timeout=120, + cwd=Path(__file__).resolve().parents[2], + ) + + output = f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + assert result.returncode == 0, output + assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout, output + assert '"catalog"' in result.stdout, output + marker = next( + (line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=")), + None, + ) + assert marker is not None, output + assert json.loads(marker.split("=", 1)[1]) == [], output diff --git a/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py b/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py index 6294db49d8..314788b4de 100644 --- a/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py +++ b/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py @@ -5,6 +5,7 @@ from pathlib import Path from unittest import TestCase +import pandas # noqa: F401 # loaded here so limit_memory measures CSV parsing, not the extractor's lazy import import pytest import requests import requests_mock