From dfc8407e98781bc51c06c47922b7926bccc93194 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:19:53 +0000 Subject: [PATCH 1/2] perf: defer pandas import in ResponseToFileExtractor Co-Authored-By: bot_apk --- .../extractors/response_to_file_extractor.py | 4 +- ...est_source_declarative_manifest_imports.py | 60 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py diff --git a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py index f9a655db0c..10896c57e5 100644 --- a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py @@ -10,7 +10,6 @@ from math import nan from typing import Any, Dict, Iterable, Mapping, Optional, Tuple -import pandas as pd import requests from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -139,6 +138,9 @@ def _read_with_chunks( ValueError: If an IO/Error occurs while reading the temporary data. """ + # Imported lazily: pandas (and numpy) are slow to import and only needed for CSV parsing. + import pandas as pd + try: with open(path, "r", encoding=file_encoding) as data: # When preserving NA values, disable pandas' default NA parsing so strings like "NA"/"N/A" are kept as-is, diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py new file mode 100644 index 0000000000..f3bbab3cc3 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. + +import json +import subprocess +import sys +from pathlib import Path + +SCRIPT = """ +import json +import sys +from unittest.mock import patch + +import requests_mock +import yaml + +from airbyte_cdk.cli.source_declarative_manifest import _run + +manifest_path, config_path = sys.argv[1:] +with open(manifest_path) as manifest_file: + manifest = yaml.safe_load(manifest_file) + +with patch( + "airbyte_cdk.cli.source_declarative_manifest._run._is_local_manifest_command", + return_value=True, +): + with patch( + "airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file", + return_value=manifest, + ): + with requests_mock.Mocker() as mocker: + mocker.get( + "https://pokeapi.co/api/v2/pokemon/blastoise", + json={"name": "blastoise"}, + ) + _run.handle_command(["spec"]) + _run.handle_command(["check", "--config", config_path]) + _run.handle_command(["discover", "--config", config_path]) + +print("LOADED_HEAVY_MODULES=" + json.dumps(sorted(m for m in ("pandas", "numpy") if m in sys.modules))) +""" + + +def test_spec_check_discover_do_not_import_pandas_or_numpy() -> None: + manifest_path = Path(__file__).parent.parent / "resources" / "valid_local_manifest.yaml" + config_path = Path(__file__).parent.parent / "resources" / "valid_local_pokeapi_config.json" + + result = subprocess.run( + [sys.executable, "-c", SCRIPT, str(manifest_path), str(config_path)], + capture_output=True, + text=True, + timeout=120, + ) + + assert result.returncode == 0, result.stderr + assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout + assert '"catalog"' in result.stdout + marker = next( + line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=") + ) + assert json.loads(marker.split("=", 1)[1]) == [] From 5d732dee5e3685345d27f6382c0ea07c97156a34 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 19:29:47 +0000 Subject: [PATCH 2/2] fix: address review feedback on lazy pandas import Co-Authored-By: bot_apk --- Dockerfile | 3 ++ .../extractors/response_to_file_extractor.py | 6 +++- ...est_source_declarative_manifest_imports.py | 30 ++++++++++++------- .../test_response_to_file_extractor.py | 1 + 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9eb986295..b0fe584921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,9 @@ RUN poetry config virtualenvs.create false \ # Build and install the package RUN pip install dist/*.whl +# pandas is imported lazily by the CDK, so verify it is importable at build time +RUN python -c "import pandas" + # Recreate the original structure RUN mkdir -p source_declarative_manifest \ && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \ diff --git a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py index 10896c57e5..145885e14c 100644 --- a/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +++ b/airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py @@ -139,7 +139,11 @@ def _read_with_chunks( """ # Imported lazily: pandas (and numpy) are slow to import and only needed for CSV parsing. - import pandas as pd + try: + import pandas as pd + except ImportError: + os.remove(path) + raise try: with open(path, "r", encoding=file_encoding) as data: diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py index f3bbab3cc3..d916e7a5a2 100644 --- a/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py +++ b/unit_tests/source_declarative_manifest/test_source_declarative_manifest_imports.py @@ -40,21 +40,31 @@ """ -def test_spec_check_discover_do_not_import_pandas_or_numpy() -> None: - manifest_path = Path(__file__).parent.parent / "resources" / "valid_local_manifest.yaml" - config_path = Path(__file__).parent.parent / "resources" / "valid_local_pokeapi_config.json" - +def test_spec_check_discover_do_not_import_pandas_or_numpy( + valid_local_manifest, valid_local_config_file +) -> None: + """Declarative/SDM start-up must not import pandas or numpy. Import heavy libraries inside the function that needs them; `python -X importtime -c "import airbyte_cdk.cli.source_declarative_manifest._run"` shows which module pulls one in.""" result = subprocess.run( - [sys.executable, "-c", SCRIPT, str(manifest_path), str(config_path)], + [ + sys.executable, + "-c", + SCRIPT, + str(valid_local_manifest), + str(valid_local_config_file), + ], capture_output=True, text=True, timeout=120, + cwd=Path(__file__).resolve().parents[2], ) - assert result.returncode == 0, result.stderr - assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout - assert '"catalog"' in result.stdout + output = f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + assert result.returncode == 0, output + assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout, output + assert '"catalog"' in result.stdout, output marker = next( - line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=") + (line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=")), + None, ) - assert json.loads(marker.split("=", 1)[1]) == [] + assert marker is not None, output + assert json.loads(marker.split("=", 1)[1]) == [], output diff --git a/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py b/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py index 6294db49d8..314788b4de 100644 --- a/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py +++ b/unit_tests/sources/declarative/extractors/test_response_to_file_extractor.py @@ -5,6 +5,7 @@ from pathlib import Path from unittest import TestCase +import pandas # noqa: F401 # loaded here so limit_memory measures CSV parsing, not the extractor's lazy import import pytest import requests import requests_mock