From b32ec48dbace55537ffd30411e33dae87b83e23a Mon Sep 17 00:00:00 2001 From: Seth Grover <13872653+mmguero@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:06:32 -0600 Subject: [PATCH 01/19] fix(repo): add REPO_URL=local sentinel to bypass git entirely (#124) DTLRepo always ran a clone or a fetch, even when REPO_PATH already held the library contents locally. validate_git_url() rejected the old 'local' value outright, and even a valid URL still triggered a real git fetch against the remote on every run. REPO_URL=local now skips Repo(), clone_from(), and fetch() entirely. REPO_PATH is used as-is and must already contain device-types/, module-types/, and rack-types/ (no .git required, and none is used). --- core/repo.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/repo.py b/core/repo.py index 869bedd6..74688641 100644 --- a/core/repo.py +++ b/core/repo.py @@ -419,11 +419,13 @@ class DTLRepo: def __init__(self, config, handle): """Initialize repository management, updating an existing clone or creating a new one. - If the target path already holds a Git clone, the repository will be updated from - its configured remote; otherwise the provided URL is validated and a new clone is - created. The initializer sets instance attributes used by other methods (handler, - supported YAML extensions, URL, repo path, branch, repo reference, and current - working directory). + If REPO_URL is the sentinel "local", no git operation of any kind is performed — + REPO_PATH is used as-is and must already contain the device-type file tree. + Otherwise, if the target path already holds a Git clone, the repository will be + updated from its configured remote; if not, the provided URL is validated and a new + clone is created. The initializer sets instance attributes used by other methods + (handler, supported YAML extensions, URL, repo path, branch, repo reference, and + current working directory). Args: config (RunConfig): Supplies `repo_url`, `repo_branch`, and `repo_path`. @@ -441,6 +443,14 @@ def __init__(self, config, handle): self.repo = None self.cwd = os.getcwd() + if str(self.url).strip().casefold() == "local": + if not os.path.isdir(self.get_absolute_path()): + raise InvalidRepoPathError( + self.repo_path, reason="REPO_URL=local requires REPO_PATH to already contain the library files" + ) + self.handle.log(f"REPO_URL=local: using {self.get_absolute_path()} as-is, no git operations") + return + is_path_valid, path_error = validate_repo_path(self.repo_path) if not is_path_valid: raise InvalidRepoPathError(self.repo_path, reason=path_error) From 5694aaa8965afd3d88e5146649b2d78550b902b6 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:35:07 +0200 Subject: [PATCH 02/19] fix(repo): validate the local library layout and share the check with export (#126) * fix(repo): validate the local library layout and share the check with export REPO_URL=local only checked that REPO_PATH was a directory. An unrelated or empty path passed that check, and discover_vendors skips type directories that are absent, so the run imported nothing and still exited 0. Export mode already had the check it needed, in Exporter._verify_repo_available. Both sides now read one definition of what makes a checkout a library: LIBRARY_TYPE_DIRS and library_dirs_present() in core/repo.py. The sentinel value moves to LOCAL_REPO_URL in core/config.py, beside the other REPO_* defaults, so config and repo cannot drift on it. Local mode still skips validate_repo_path on purpose: that check demands write access, which a read-only or air-gapped mount cannot give, and no import step writes to REPO_PATH. A test pins the read-only case. REPO_BRANCH is ignored under the sentinel, so a run that sets both now says so through the existing config notice mechanism instead of looking like it checked the branch out. Documents the mode in the README and .env.example, including the read-only Docker mount, which is the case that motivated the sentinel. * fix(import): treat an absent type root as empty instead of crashing The layout check accepts any one of device-types/, module-types/, and rack-types/, matching what export mode already accepted. plan_vendor then called get_devices() on all three regardless, and get_devices() lists the directory, so a local checkout holding only some of them raised FileNotFoundError before importing the types it did hold. A device-types-only library is the likeliest local layout, and it crashed on module-types. _parse_vendor_racks already had the guard this needed. It is now _parse_vendor_files and all three roots go through it, so the guard cannot apply to one root and not the others again. The repo mock in test_nb_dt_import pointed at /tmp/devices, /tmp/modules and /tmp/rack-types, paths that never existed. Nothing stat'd them, so the mock passed for a filesystem that was not there, which is why this went unseen. It now points at a real empty library tree, and the tests that matched on those literal paths match on the directory names instead. Found by CodeRabbit on #126. --- .env.example | 6 +++ README.md | 25 ++++++++++- core/config.py | 14 +++++- core/export.py | 8 ++-- core/import_run.py | 25 ++++++----- core/repo.py | 49 +++++++++++++++----- tests/test_config.py | 19 ++++++++ tests/test_exporter.py | 30 +++++++++++++ tests/test_import_run.py | 31 +++++++++++++ tests/test_nb_dt_import.py | 36 ++++++++++----- tests/test_repo.py | 91 ++++++++++++++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index be9a7476..0c6b5dc4 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,12 @@ NETBOX_URL=https://netbox.example.org REPO_BRANCH=master REPO_URL=https://github.com/netbox-community/devicetype-library.git +# Set REPO_URL=local to read REPO_PATH as it stands, with no git operation at all: +# no clone, no fetch, and no .git needed. REPO_PATH must already hold at least one of +# device-types/, module-types/, or rack-types/. REPO_BRANCH is ignored in this mode. +# Use it for air-gapped runs or a library you version yourself. +#REPO_URL=local + # Local path where the device-type library repository is cloned. # Defaults to a "repo" directory in the project root when not set. # Use an absolute path or a path relative to where you run the script. diff --git a/README.md b/README.md index 447a98e9..5b004a04 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ and device, creating anything that is missing from NetBox while skipping entries | --- | --- | --- | --- | | `NETBOX_URL` | ✅ | — | URL of your NetBox instance | | `NETBOX_TOKEN` | ✅ | — | API token with write access | -| `REPO_URL` | | community library | Git URL of the device-type library to clone | +| `REPO_URL` | | community library | Git URL of the device-type library to clone. Set to `local` to read `REPO_PATH` as it stands, with no git operation (see [Offline / local library](#offline--local-library)) | | `REPO_BRANCH` | | `master` | Branch to check out | | `REPO_PATH` | | `./repo` | Local path where the library is cloned. Accepts absolute or relative paths. | | `VENDORS` | | all | Comma-separated vendors to import (same effect as `--vendors`) | @@ -435,6 +435,29 @@ uv run nb-dt-import.py --vendors nokia --verify-images - After replacing a local image file with a higher-quality version and wanting NetBox to pick it up +#### Offline / local library + +Set `REPO_URL=local` to point the importer at a directory that already holds the library and +skip git completely: no clone, no fetch, and no `.git` needed. `REPO_PATH` must contain at +least one of `device-types/`, `module-types/`, or `rack-types/`, and an import stops with an +error if it does not, rather than reporting an empty library as nothing to do. + +Nothing in an import writes to `REPO_PATH`, so a read-only mount works. `REPO_BRANCH` is +ignored in this mode, and a run that sets both says so. + +```shell +REPO_URL=local REPO_PATH=/srv/devicetype-library uv run nb-dt-import.py +``` + +```shell +docker run --rm -e REPO_URL=local -e REPO_PATH=/library \ + -v /srv/devicetype-library:/library:ro \ + -e NETBOX_URL -e NETBOX_TOKEN ghcr.io/marcinpsk/device-type-library-import +``` + +**When to use**: air-gapped networks, a library you version yourself, or any run that must not +reach the network for the library. + #### Export Mode `--export-diff` runs in the opposite direction to every other mode: instead of importing the diff --git a/core/config.py b/core/config.py index f71d9074..7a67f694 100644 --- a/core/config.py +++ b/core/config.py @@ -11,6 +11,8 @@ DEFAULT_REPO_URL = "https://github.com/netbox-community/devicetype-library.git" DEFAULT_REPO_BRANCH = "master" +# REPO_URL value that reads REPO_PATH as it stands, with no git operation at all. +LOCAL_REPO_URL = "local" DEFAULT_GRAPHQL_PAGE_SIZE = 5000 DEFAULT_PRELOAD_THREADS = 8 @@ -20,6 +22,11 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def is_local_repo_url(url): + """Return True when *url* is the sentinel that turns off every git operation.""" + return str(url or "").strip().casefold() == LOCAL_REPO_URL + + class ConfigError(FatalError): """A configuration value the run cannot start with, phrased for the person who set it.""" @@ -102,7 +109,7 @@ def build_argument_parser(env): "--url", "--git", default=_text(env, "REPO_URL", DEFAULT_REPO_URL), - help="Git URL with valid Device Type YAML files", + help=f'Git URL with valid Device Type YAML files, or "{LOCAL_REPO_URL}" to read REPO_PATH with no git', ) parser.add_argument( "--slugs", @@ -264,6 +271,11 @@ def resolve_run_config(argv=None, env=None): # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: + notices.append( + f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " + "and checks out no branch." + ) return RunConfig( netbox_url=_text(env, "NETBOX_URL"), diff --git a/core/export.py b/core/export.py index 02490761..c3e43aa1 100644 --- a/core/export.py +++ b/core/export.py @@ -28,12 +28,10 @@ serialize_rack_type, ) from core.netbox_api import IMAGE_EXTENSIONS, _build_auth_header +from core.repo import LIBRARY_TYPE_DIRS, library_dirs_present _SKIP = object() # sentinel: image already exists, no download needed -# Top-level directories that make a checkout a device-type library. -_LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") - # Maps Content-Type to a canonical extension for extension-less attachments. _CONTENT_TYPE_EXT = { "image/png": ".png", @@ -474,11 +472,11 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: def _verify_repo_available(self) -> None: """Raise FileNotFoundError when the library is absent, which would otherwise read as an empty one.""" - if any((self.repo_path / name).is_dir() for name in _LIBRARY_TYPE_DIRS): + if library_dirs_present(self.repo_path): return raise FileNotFoundError( f"No device-type library found at {self.repo_path}: expected at least one of " - f"{', '.join(_LIBRARY_TYPE_DIRS)}. Export mode does not clone the library. " + f"{', '.join(LIBRARY_TYPE_DIRS)}. Export mode does not clone the library. " "Clone it to that path, or run an import first, which clones it for you." ) diff --git a/core/import_run.py b/core/import_run.py index 37a3caa9..00da6870 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -529,12 +529,15 @@ def _log_run_summary(handle, summary): handle.log("These duplicates would otherwise oscillate on every run. Please report/fix them upstream.") -def _parse_vendor_racks(repo, racks_path, vendor_name, slugs): - """Parse rack types for one vendor when the rack path exists.""" - if not os.path.isdir(racks_path): +def _parse_vendor_files(repo, base_path, vendor_name, slugs): + """Parse one vendor's types under *base_path*, treating an absent type root as empty. + + A local library may ship only some of the three type roots, which is a layout, not a fault. + """ + if not os.path.isdir(base_path): return [] - rack_files, _ = repo.get_devices(racks_path, [vendor_name.casefold()]) - return repo.parse_files(rack_files, slugs=slugs) + files, _ = repo.get_devices(base_path, [vendor_name.casefold()]) + return repo.parse_files(files, slugs=slugs) def _finalize_task_registry(progress, task_registry): @@ -641,16 +644,18 @@ def plan_vendor(self, selection, vendor): device_files = slug_resolved["device_files"].get(vendor["slug"], []) device_types = self.repo.parse_files(device_files) if device_files else [] else: - device_files, _ = self.repo.get_devices(selection.devices_path, [vendor["name"].casefold()]) - device_types = self.repo.parse_files(device_files, slugs=self.config.slugs or []) + device_types = _parse_vendor_files( + self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] + ) if self.netbox.modules: module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] else: - module_files, _ = self.repo.get_devices(selection.modules_path, [vendor["name"].casefold()]) - module_types = self.repo.parse_files(module_files, slugs=self.config.slugs or []) + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) else: module_types = [] @@ -659,7 +664,7 @@ def plan_vendor(self, selection, vendor): if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] else: - rack_types = _parse_vendor_racks( + rack_types = _parse_vendor_files( self.repo, selection.racks_path, vendor["name"], diff --git a/core/repo.py b/core/repo.py index 74688641..57656278 100644 --- a/core/repo.py +++ b/core/repo.py @@ -11,8 +11,12 @@ from git import Repo, exc import yaml +from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +# Top-level directories that make a checkout a device-type library. +LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") + class GitCommandError(FatalError): """A Git command that failed for a repository.""" @@ -184,6 +188,11 @@ def _safe_index_load(path: str): return _safe_pickle_load(path) +def library_dirs_present(path): + """Return True when *path* holds at least one device-type library directory.""" + return any(os.path.isdir(os.path.join(str(path), name)) for name in LIBRARY_TYPE_DIRS) + + def validate_git_url(url): """Determine whether a Git remote URL is allowed (HTTPS, SSH, or file://). @@ -419,11 +428,11 @@ class DTLRepo: def __init__(self, config, handle): """Initialize repository management, updating an existing clone or creating a new one. - If REPO_URL is the sentinel "local", no git operation of any kind is performed — - REPO_PATH is used as-is and must already contain the device-type file tree. - Otherwise, if the target path already holds a Git clone, the repository will be - updated from its configured remote; if not, the provided URL is validated and a new - clone is created. The initializer sets instance attributes used by other methods + If REPO_URL is the sentinel "local", no git operation runs at all: REPO_PATH is read + as it stands and must already hold the device-type file tree. Otherwise, if the target + path already holds a Git clone, the repository will be updated from its configured + remote; if not, the provided URL is validated and a new clone is created. The + initializer sets instance attributes used by other methods (handler, supported YAML extensions, URL, repo path, branch, repo reference, and current working directory). @@ -443,12 +452,8 @@ def __init__(self, config, handle): self.repo = None self.cwd = os.getcwd() - if str(self.url).strip().casefold() == "local": - if not os.path.isdir(self.get_absolute_path()): - raise InvalidRepoPathError( - self.repo_path, reason="REPO_URL=local requires REPO_PATH to already contain the library files" - ) - self.handle.log(f"REPO_URL=local: using {self.get_absolute_path()} as-is, no git operations") + if is_local_repo_url(self.url): + self._use_local_checkout() return is_path_valid, path_error = validate_repo_path(self.repo_path) @@ -467,6 +472,28 @@ def __init__(self, config, handle): raise InvalidGitURLError(self.url, reason=error_msg) self.clone_repo() + def _use_local_checkout(self): + """Accept REPO_PATH as it stands: no clone, no fetch, and no .git needed. + + Skips validate_repo_path on purpose: it demands write access, which a read-only + or air-gapped mount cannot give, and no import step writes to REPO_PATH. + """ + path = self.get_absolute_path() + if not os.path.isdir(path): + raise InvalidRepoPathError( + self.repo_path, + reason=f"REPO_URL={LOCAL_REPO_URL} needs REPO_PATH to be an existing directory", + ) + if not library_dirs_present(path): + raise InvalidRepoPathError( + self.repo_path, + reason=( + f"No device-type library found: expected at least one of {', '.join(LIBRARY_TYPE_DIRS)} " + f"inside it. REPO_URL={LOCAL_REPO_URL} does not clone the library." + ), + ) + self.handle.log(f"REPO_URL={LOCAL_REPO_URL}: reading {path} as it stands, with no git operation") + def get_relative_path(self): """Get the repository path configured for this instance relative to the current working directory. diff --git a/tests/test_config.py b/tests/test_config.py index 0bb65ee4..1aae1838 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -120,3 +120,22 @@ def test_an_explicit_repo_path_is_taken_as_given(self, tmp_path): config = _resolve(REPO_PATH=str(tmp_path)) assert config.repo_path == str(tmp_path) + + +class TestLocalRepoUrlIgnoresTheBranch: + """REPO_URL=local checks out nothing, so a REPO_BRANCH set beside it silently does nothing.""" + + def test_a_branch_set_beside_the_sentinel_is_reported_as_ignored(self): + config = _resolve(REPO_URL="local", REPO_BRANCH="feature") + + assert any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + def test_the_sentinel_alone_needs_no_notice(self): + config = _resolve(REPO_URL="local") + + assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + def test_a_branch_set_beside_a_real_url_needs_no_notice(self): + config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") + + assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices diff --git a/tests/test_exporter.py b/tests/test_exporter.py index ae5d1797..a356e528 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1485,3 +1485,33 @@ def test_all_known_content_types_are_recognised(self): for ct, ext in _CONTENT_TYPE_EXT.items(): result = _sanitize_attachment_filename("img", "/media/img", ct) assert result.endswith(ext), f"Expected {ext} for {ct}, got {result}" + + +class TestRepoAvailability: + """Export mode and REPO_URL=local must agree on what makes a directory a library checkout.""" + + def _exporter(self, tmp_path, repo_path): + settings = replace(_make_settings(tmp_path), repo_path=str(repo_path)) + return Exporter(settings, _make_handle(), str(tmp_path / "extra"), False, None) + + @pytest.mark.parametrize("present", ["device-types", "module-types", "rack-types"]) + def test_one_library_directory_is_enough(self, present, tmp_path): + repo = tmp_path / "repo" + (repo / present).mkdir(parents=True) + + self._exporter(tmp_path, repo)._verify_repo_available() + + def test_a_directory_without_library_directories_stops_the_export(self, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + + with pytest.raises(FileNotFoundError, match="No device-type library found"): + self._exporter(tmp_path, repo)._verify_repo_available() + + def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "device-types").write_text("not a directory") + + with pytest.raises(FileNotFoundError, match="No device-type library found"): + self._exporter(tmp_path, repo)._verify_repo_available() diff --git a/tests/test_import_run.py b/tests/test_import_run.py index caeb96fa..59f14405 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -8,6 +8,7 @@ from core.errors import VendorSelectionError from core.import_run import ImportRun, RunSummary, VendorPlan, _process_device_types from core.log_handler import LogHandler +from core.repo import DTLRepo class _ComponentCache: @@ -277,3 +278,33 @@ def test_execute_releases_console_when_progress_setup_fails(make_config, tmp_pat assert progress_factory.exited is True assert handle.console is None assert netbox.device_types.components.close_count == 1 + + +class TestPartialLibraryLayouts: + """A local checkout may hold one type root only, so planning must not read the absent ones.""" + + TYPE_DIRS = ("device-types", "module-types", "rack-types") + + def _repo(self, tmp_path, present, make_config): + """Build a real DTLRepo over a checkout holding *present* alone.""" + vendor = tmp_path / present / "TestVendor" + vendor.mkdir(parents=True) + (vendor / "thing.yaml").write_text("manufacturer: TestVendor\nmodel: Test\nslug: test\n") + config = make_config(repo_url="local", repo_path=str(tmp_path)) + return DTLRepo(config, LogHandler(False)), config + + @pytest.mark.parametrize("present", TYPE_DIRS) + def test_one_type_root_plans_without_reading_the_absent_ones(self, present, make_config, tmp_path): + repo, config = self._repo(tmp_path, present, make_config) + run = ImportRun(config, repo, _NetBoxBoundary(), LogHandler(False), _ProgressFactory()) + + selection = run.discover() + plan = run.plan_vendor(selection, {"name": "TestVendor", "slug": "testvendor"}) + + parsed = { + "device-types": len(plan.device_types), + "module-types": len(plan.module_types), + "rack-types": len(plan.rack_types), + } + assert parsed[present] == 1, parsed + assert sum(parsed.values()) == 1, parsed diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 71da96d2..929231ca 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -308,13 +308,27 @@ def test_items_per_second_column_uses_elapsed_fallback_when_finished_speed_missi _PROJECT_ROOT = Path(__file__).resolve().parents[1] +_LIBRARY_ROOT = None + + +@pytest.fixture(scope="session", autouse=True) +def _real_library_root(tmp_path_factory): + """Point the repo mocks at a real library tree: the pipeline stats these paths before reading them.""" + global _LIBRARY_ROOT + root = tmp_path_factory.mktemp("library") + for name in ("device-types", "module-types", "rack-types"): + (root / name).mkdir() + _LIBRARY_ROOT = root + yield root + + def _make_mock_repo(device_types=None): """Return a pre-configured DTLRepo mock with no files by default.""" mock_repo = MagicMock() mock_repo.get_devices.return_value = ([], []) - mock_repo.get_devices_path.return_value = "/tmp/devices" - mock_repo.get_modules_path.return_value = "/tmp/modules" - mock_repo.get_racks_path.return_value = "/tmp/rack-types" + mock_repo.get_devices_path.return_value = str(_LIBRARY_ROOT / "device-types") + mock_repo.get_modules_path.return_value = str(_LIBRARY_ROOT / "module-types") + mock_repo.get_racks_path.return_value = str(_LIBRARY_ROOT / "rack-types") mock_repo.discover_vendors.return_value = [] mock_repo.parse_files.return_value = device_types if device_types is not None else [] mock_repo.resolve_slug_files.return_value = None # no pickle available by default @@ -905,7 +919,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): repo = _make_mock_repo() repo.discover_vendors.return_value = [{"name": "Vendor One", "slug": "vendor-one"}] repo.get_devices.side_effect = lambda path, vendors=None: ( - (["module.yaml"], []) if "modules" in path else ([], []) + (["module.yaml"], []) if "module-types" in path else ([], []) ) repo.parse_files.side_effect = lambda files, slugs=None: [module_type] if files == ["module.yaml"] else [] MockRepo.return_value = repo @@ -1600,13 +1614,13 @@ def test_no_pulse_bar_column_uses_static_empty_bar_for_unknown_total(self, nb_dt assert bar.total == 1.0 assert bar.completed == 0.0 - def test_parse_vendor_racks_calls_repo_when_directory_exists(self, nb_dt_import): + def test_parse_vendor_files_calls_repo_when_directory_exists(self, nb_dt_import): repo = MagicMock() repo.get_devices.return_value = (["rack.yaml"], []) repo.parse_files.return_value = [{"model": "Rack"}] with patch("core.import_run.os.path.isdir", return_value=True): - result = import_run_module._parse_vendor_racks(repo, "/racks", "nokia", ["rack"]) + result = import_run_module._parse_vendor_files(repo, "/racks", "nokia", ["rack"]) assert result == [{"model": "Rack"}] repo.get_devices.assert_called_once_with("/racks", ["nokia"]) @@ -1844,7 +1858,7 @@ def _parse_files(files, slugs=None, progress=None): return [] mock_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) mock_repo.parse_files.side_effect = _parse_files mock_nb = _make_mock_netbox() @@ -1878,7 +1892,7 @@ def test_main_stops_preload_job_in_finally_on_error(self, nb_dt_import): mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Cisco", "slug": "cisco"}] mock_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) mock_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["device.yaml"] else [] @@ -1964,14 +1978,14 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c dtl_repo = _make_mock_repo() dtl_repo.get_devices.side_effect = lambda path, vendors=None: ( ([f"{vendors[0]}-{path.split('/')[-1]}.yaml"], []) - if vendors and vendors[0] == "cisco" and path in {"/tmp/modules", "/tmp/rack-types"} + if vendors and vendors[0] == "cisco" and path.endswith(("module-types", "rack-types")) else ([], []) ) dtl_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["resolved.yaml"] else [{"manufacturer": {"slug": "cisco"}, "model": "M", "slug": "m"}] - if files == ["cisco-modules.yaml"] + if files == ["cisco-module-types.yaml"] else [] ) netbox = _make_mock_netbox(modules=True) @@ -2030,7 +2044,7 @@ def test_import_run_stops_preload_in_finally_on_error(self, make_config): dtl_repo = _make_mock_repo() dtl_repo.discover_vendors.return_value = [{"slug": "cisco", "name": "Cisco"}] dtl_repo.get_devices.side_effect = lambda path, vendors=None: ( - (["device.yaml"], []) if path == "/tmp/devices" else ([], []) + (["device.yaml"], []) if path.endswith("device-types") else ([], []) ) dtl_repo.parse_files.side_effect = lambda files, slugs=None, progress=None: ( [{"manufacturer": {"slug": "cisco"}, "model": "X", "slug": "x"}] if files == ["device.yaml"] else [] diff --git a/tests/test_repo.py b/tests/test_repo.py index a260c8bb..ab4e695a 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -154,6 +154,97 @@ def test_invalid_path_raises_before_repository_access(self, tmp_path): _dtl_repo(config, str(invalid_path), LogHandler(False)) +class TestDTLRepoLocalMode: + """REPO_URL=local reads REPO_PATH as it stands: no clone, no fetch, no .git.""" + + @staticmethod + def _make_library(path, *dir_names): + """Create a library checkout at *path* holding one YAML file under each named directory.""" + path.mkdir(parents=True, exist_ok=True) + for name in dir_names: + vendor = path / name / "TestVendor" + vendor.mkdir(parents=True) + (vendor / "device.yaml").write_text("manufacturer: TestVendor\nmodel: Test\n") + return path + + def _init_local(self, repo_path, url="local", handle=None): + config = MagicMock(repo_url=url, repo_branch="master") + return _dtl_repo(config, str(repo_path), handle or LogHandler(False)) + + def test_reads_the_checkout_without_running_git(self, tmp_path, mock_git_repo): + """The whole point of the sentinel: the files are read and git is never reached.""" + library = self._make_library(tmp_path / "library", "device-types") + + repo = self._init_local(library) + files, vendors = repo.get_devices(repo.get_devices_path()) + + assert [os.path.basename(path) for path in files] == ["device.yaml"] + assert vendors == [{"name": "TestVendor", "slug": "testvendor"}] + assert [parsed["model"] for parsed in repo.parse_files(files)] == ["Test"] + assert not (library / ".git").exists() + mock_git_repo.assert_not_called() + mock_git_repo.clone_from.assert_not_called() + + @pytest.mark.parametrize("value", ["local", "LOCAL", "Local", " local "]) + def test_the_sentinel_ignores_case_and_padding(self, value, tmp_path, mock_git_repo): + library = self._make_library(tmp_path / "library", "device-types") + + self._init_local(library, url=value) + + mock_git_repo.assert_not_called() + mock_git_repo.clone_from.assert_not_called() + + @pytest.mark.parametrize("present", ["device-types", "module-types", "rack-types"]) + def test_one_library_directory_is_enough(self, present, tmp_path, mock_git_repo): + """Export mode accepts any one of the three, so local import mode must agree.""" + library = self._make_library(tmp_path / "library", present) + + self._init_local(library) + + mock_git_repo.assert_not_called() + + def test_a_missing_directory_is_reported_as_a_path_error(self, tmp_path): + with pytest.raises(InvalidRepoPathError, match="existing directory"): + self._init_local(tmp_path / "absent") + + def test_a_directory_without_library_directories_is_rejected(self, tmp_path): + """An unrelated path would otherwise import nothing and still exit successfully.""" + empty = tmp_path / "library" + empty.mkdir() + + with pytest.raises(InvalidRepoPathError, match="No device-type library found"): + self._init_local(empty) + + def test_a_stray_file_named_like_a_library_directory_is_rejected(self, tmp_path): + library = tmp_path / "library" + library.mkdir() + (library / "device-types").write_text("not a directory") + + with pytest.raises(InvalidRepoPathError, match="No device-type library found"): + self._init_local(library) + + @pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores the write bit") + def test_a_read_only_checkout_is_accepted(self, tmp_path): + """The air-gapped case: nothing writes to REPO_PATH, so a read-only mount must work.""" + library = self._make_library(tmp_path / "library", "device-types") + library.chmod(0o555) + + try: + repo = self._init_local(library) + assert repo.get_devices(repo.get_devices_path())[0] + finally: + library.chmod(0o755) + + def test_the_branch_is_left_unused(self, tmp_path, mock_git_repo): + """No checkout happens, so REPO_BRANCH must not reach git (config warns about it instead).""" + library = self._make_library(tmp_path / "library", "device-types") + config = MagicMock(repo_url="local", repo_branch="a-branch-that-does-not-exist") + + _dtl_repo(config, str(library), LogHandler(False)) + + mock_git_repo.assert_not_called() + + class TestDTLRepoRealGit: """Clone/pull branching driven against a real local Git repository (no git mocks).""" From d5e848aa2c898cf3ef0b029c6dd5316443696705 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:02:18 +0200 Subject: [PATCH 03/19] feat(module-bay-types): carry NetBox 4.7 module bay types through the importer (#132) --- core/change_detector.py | 139 ++++++- core/compat.py | 20 + core/component_registry.py | 24 +- core/export.py | 11 +- core/graphql_client.py | 63 +++- core/module_bay_types.py | 241 ++++++++++++ core/nb_serializer.py | 30 +- core/netbox_api.py | 238 +++++++++++- tests/conftest.py | 8 +- tests/helpers.py | 199 ++++++++++ tests/test_component_registry.py | 17 +- tests/test_exporter.py | 32 +- tests/test_graphql_client.py | 143 ++++++- tests/test_module_bay_type_sync.py | 577 +++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 250 +++++++++++++ tests/test_nb_dt_import.py | 46 ++- tests/test_nb_serializer.py | 66 ++++ tests/test_netbox_api.py | 243 ++++++------ tests/test_relation_scope.py | 188 ++++++++++ tests/test_suite_hygiene.py | 19 + 20 files changed, 2383 insertions(+), 171 deletions(-) create mode 100644 core/module_bay_types.py create mode 100644 tests/test_module_bay_type_sync.py create mode 100644 tests/test_module_bay_types.py create mode 100644 tests/test_relation_scope.py diff --git a/core/change_detector.py b/core/change_detector.py index 24fbd214..12ea6f0d 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -10,7 +10,7 @@ from typing import Any, List, Optional from enum import Enum -from core.component_registry import COMPONENT_TYPES +from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES from core.normalization import normalize_values from core.formatting import log_property_diffs from core.schema_reader import load_properties_for_type @@ -26,6 +26,110 @@ class ChangeType(Enum): COMPONENT_REMOVED = "component_removed" +def _manufacturer_slug_of(yaml_data): + """Return the owning manufacturer's slug from a parsed definition. + + ``core.repo`` rewrites every YAML manufacturer to ``{"slug": ...}`` before the importer + sees it, so that is the usual shape here. + """ + manufacturer = (yaml_data or {}).get("manufacturer") + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or "" + return manufacturer or "" + + +def _relation_properties(comp_type): + """Return the relation field names the registry declares for *comp_type*.""" + component = BY_YAML_KEY.get(comp_type) + return component.relations if component else () + + +def _is_relation_list(value): + """Return True when *value* is a list of non-empty strings, the only shape a reference takes.""" + return isinstance(value, list) and all(isinstance(item, str) and item for item in value) + + +def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): + """Return a PropertyChange when a relation differs, or None when there is nothing to do. + + A relation is an unordered set of related objects. Where the catalog is available the + comparison is between *identities*, ``(manufacturer slug, slug)``, because two + manufacturers may define the same class name and a bay holding the wrong one reads as + equal on names alone. Without a catalog it falls back to comparing names, which is + what a caller that has not wired one can still do. + + An omitted key leaves the relation unmanaged and an empty list clears it, but a bare + ``module_bay_types:`` parses as None and a malformed entry is not a name: those are left + unmanaged, because reading them as empty would clear a restriction nobody removed. A + field the query did not return is skipped for the same reason. + """ + if prop not in yaml_comp: + return None + declared = yaml_comp.get(prop) + if not _is_relation_list(declared): + if handle is not None: + handle.log( + f"Ignored {prop} on {yaml_comp.get('name', 'Unknown')!r}: expected a list of names, got {declared!r}" + ) + return None + netbox_value = getattr(netbox_comp, prop, _MISSING) + if netbox_value is _MISSING: + return None + + if catalog is not None and manufacturer: + from core.module_bay_types import ModuleBayTypeError + + try: + wanted = catalog.identities_for(manufacturer, declared) + except ModuleBayTypeError: + # Reported as a change so the write path sees it; "no change" is silent. + return PropertyChange( + property_name=prop, + old_value=sorted(_relation_names(netbox_value)), + new_value=sorted(set(declared)), + ) + current = _relation_identities(netbox_value) + if current is not None and wanted == current: + return None + if current is not None: + return PropertyChange( + property_name=prop, old_value=sorted(_relation_names(netbox_value)), new_value=sorted(set(declared)) + ) + + yaml_names = frozenset(declared) + netbox_names = frozenset(_relation_names(netbox_value)) + if yaml_names == netbox_names: + return None + return PropertyChange(property_name=prop, old_value=sorted(netbox_names), new_value=sorted(yaml_names)) + + +def _relation_identities(value): + """Return {(manufacturer slug, slug)} for a relation, or None if the shape lacks them. + + A read path that returns only ``id`` and ``name`` cannot answer the scope question, so + the caller falls back to comparing names rather than inventing an identity. + """ + identities = set() + for item in value or []: + slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) + manufacturer = item.get("manufacturer") if isinstance(item, dict) else getattr(item, "manufacturer", None) + owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) + if not slug or not owner: + return None + identities.add((owner, slug)) + return frozenset(identities) + + +def _relation_names(value): + """Return the ``name`` of each related object NetBox returned for a relation.""" + names = [] + for item in value or []: + name = item.get("name") if isinstance(item, dict) else getattr(item, "name", None) + if name is not None: + names.append(name) + return names + + @dataclass class PropertyChange: """Represents a single property change.""" @@ -339,7 +443,11 @@ def _compare_components( # Check for property changes on existing component existing = existing_components[comp_name] prop_changes = self._compare_component_properties( - yaml_comp, existing, component.compare_properties, comp_type=yaml_key + yaml_comp, + existing, + component.compare_properties, + comp_type=yaml_key, + manufacturer=_manufacturer_slug_of(yaml_data), ) if prop_changes: changes.append( @@ -353,14 +461,31 @@ def _compare_components( return changes + def _relation_catalog(self): + """Return the module bay type catalog, or None when this detector has no real one. + + Tests build the detector around a stand-in for DeviceTypes, and a stand-in cannot + answer what a reference means. Returning None there keeps the name comparison, + which is what those tests are asserting on. + """ + from core.module_bay_types import ModuleBayTypeCatalog + + catalog = getattr(self.device_types, "module_bay_types", None) + return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None + def _compare_component_properties( self, yaml_comp: dict, netbox_comp, properties: List[str], comp_type: str = "", + manufacturer: str = "", ) -> List[PropertyChange]: - """Compare properties between YAML and NetBox component.""" + """Compare properties between YAML and NetBox component. + + *manufacturer* is the owning manufacturer's slug, used to resolve a relation + reference to the object it means rather than the name it is written as. + """ changes = [] for prop in properties: @@ -426,6 +551,14 @@ def _compare_component_properties( ) continue + if prop in _relation_properties(comp_type): + relation_change = _relation_change( + prop, yaml_comp, netbox_comp, self._relation_catalog(), manufacturer, self.handle + ) + if relation_change is not None: + changes.append(relation_change) + continue + # Only compare properties explicitly present in the YAML component; # an omitted property means the YAML doesn't manage it (absent key != removal). if prop not in yaml_comp: diff --git a/core/compat.py b/core/compat.py index 62c33bf1..97c59bf5 100644 --- a/core/compat.py +++ b/core/compat.py @@ -14,6 +14,26 @@ from __future__ import annotations +import re + +# Selecting or sending the relation below this release fails the whole query. +MODULE_BAY_TYPE_MINIMUM_VERSION = (4, 7) + + +def parse_netbox_version(version) -> tuple[int, int]: + """Return ``(major, minor)`` from a NetBox version string. + + Padded to two parts so a single-component string cannot raise, and tolerant of the + suffixes NetBox ships ("4.7.0-beta2"). + """ + raw = [int(re.sub(r"\D.*", "", part.strip()) or "0") for part in str(version).split(".")] + return tuple((raw + [0, 0])[:2]) # type: ignore[return-value] + + +def supports_module_bay_types(version) -> bool: + """Return True when this NetBox release exposes the module bay type relation.""" + return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION + def device_type_filter_key(new_filters: bool) -> str: """Return the correct filter parameter name for device-type component queries. diff --git a/core/component_registry.py b/core/component_registry.py index ed991052..a1fc18e7 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -17,6 +17,12 @@ LINK_POWER_PORT = "power_port" LINK_REAR_PORTS = "rear_ports" +# Fields holding names that must become NetBox ids before a POST. +RELATION_MODULE_BAY_TYPES = "module_bay_types" + +# Relation fields a module type carries itself, rather than through one of its components. +MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) + @dataclass(frozen=True) class ComponentType: @@ -27,19 +33,30 @@ class ComponentType: label: str fields: tuple[str, ...] module_types: bool = True + relations: tuple[str, ...] = field(default_factory=tuple) graphql_extra: tuple[str, ...] = field(default_factory=tuple) compare_extra: tuple[str, ...] = field(default_factory=tuple) link: Optional[str] = None @property def graphql_fields(self): - """Fields to select in a GraphQL query, including the id every consumer needs.""" - return ["id", *self.fields, *self.graphql_extra] + """Fields to select in a GraphQL query, including the id every consumer needs. + + A relation is a list of related objects, so it is selected by name rather than + read as a scalar. + """ + return ["id", *self.fields, *self.graphql_extra, *self.graphql_relation_fields] + + @property + def graphql_relation_fields(self): + """GraphQL selections for this row's relations, one nested block per relation.""" + # slug plus owning manufacturer is the identity; the name alone is ambiguous. + return [f"{name} {{ id name slug manufacturer {{ slug }} }}" for name in self.relations] @property def compare_properties(self): """Properties change detection compares between YAML and NetBox.""" - return [*self.fields, *self.compare_extra] + return [*self.fields, *self.compare_extra, *self.relations] @property def list_key(self): @@ -121,6 +138,7 @@ def create_label(self, parent_type): endpoint="module_bay_templates", label="Module Bay", fields=("name", "position", "label", "description"), + relations=(RELATION_MODULE_BAY_TYPES,), ), ) diff --git a/core/export.py b/core/export.py index c0dd57ac..f0911ee6 100644 --- a/core/export.py +++ b/core/export.py @@ -241,6 +241,9 @@ def run(self, progress=None) -> None: ) self.handle.log(f"Export-diff: fetching NetBox device/module/rack types{scope}") + # Decided before the first query: the selection depends on the answer. + self.graphql.detect_module_bay_type_support() + # ── Fetch all types from NetBox ────────────────────────────────────── by_model, by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) all_mt = self.graphql.get_module_types(manufacturer_slugs=self.vendor_slugs) @@ -624,13 +627,7 @@ def _fetch_vendor_components(self, mfr_slug: str) -> tuple: def _fetch_one(endpoint_name): if not getattr(_thread_local, "graphql", None): - client = NetBoxGraphQLClient( - self.graphql.url, - self.graphql.token, - self.graphql.ignore_ssl, - self.graphql.handle, - self.graphql.DEFAULT_PAGE_SIZE, - ) + client = self.graphql.clone() _thread_local.graphql = client with _clients_lock: _clients.append(client) diff --git a/core/graphql_client.py b/core/graphql_client.py index 70baabfd..4a8fe786 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -5,12 +5,14 @@ compatible with the existing REST-based code in ``netbox_api.py``. """ +import copy import threading import time from collections.abc import Sequence import requests +from core.compat import supports_module_bay_types from core.component_registry import BY_ENDPOINT # Module-level dedup: tracks (url, requested_page_size) pairs that have already @@ -102,6 +104,9 @@ def _to_dotdict(obj): # connections, which is transient during a long paginated run. _RETRYABLE_STATUSES = {429, 500, 502, 503, 504} +# One small metadata read; a run that cannot get it fails at the next query anyway. +_STATUS_TIMEOUT_SECONDS = 30 + def _response_body_detail(response): """Return the response body as a suffix for an error message, truncated and stripped.""" @@ -136,7 +141,7 @@ class NetBoxGraphQLClient: or raise the server's ``MAX_PAGE_SIZE`` setting to match. """ - def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): + def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000, supports_module_bay_types=False): """Store connection parameters for later use in :meth:`query`. Args: @@ -148,6 +153,8 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): ``print`` when not provided. page_size: Default number of items per GraphQL page (default: 5 000). + supports_module_bay_types: True when NetBox is >= 4.7 and its schema + exposes the module bay type relation. """ self.DEFAULT_PAGE_SIZE = page_size self.url = url.rstrip("/") @@ -155,22 +162,28 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): self.token = token self.ignore_ssl = ignore_ssl self._handle = handle + self.supports_module_bay_types = supports_module_bay_types + + self._session = self._new_session() - self._session = requests.Session() + def _new_session(self): + """Return an HTTP session carrying this client's auth and TLS settings.""" + session = requests.Session() # v2 tokens start with "nbt_" prefix (format: nbt_.); # v1 tokens are plain 40-char hex strings using legacy Token auth. auth_scheme = "Bearer" if self.token.startswith("nbt_") else "Token" - self._session.headers.update( + session.headers.update( { "Authorization": f"{auth_scheme} {self.token}", "Content-Type": "application/json", } ) - self._session.verify = not self.ignore_ssl + session.verify = not self.ignore_ssl if self.ignore_ssl: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + return session @property def handle(self): @@ -178,8 +191,35 @@ def handle(self): return self._handle def clone(self): - """Return an independent client with the same connection settings.""" - return type(self)(self.url, self.token, self.ignore_ssl, self.handle, self.DEFAULT_PAGE_SIZE) + """Return an independent client with the same settings and its own HTTP session. + + Copied rather than reconstructed, so a setting added to ``__init__`` travels with + the clone instead of quietly reverting to its default. The session is the one + thing a worker must not share, so it is the one thing rebuilt here. + """ + clone = copy.copy(self) + clone._session = self._new_session() + return clone + + def detect_module_bay_type_support(self): + """Ask NetBox for its version and record whether the relation can be selected. + + The importer learns this from its pynetbox client; the export entry point has no + such client, so it asks here. Both sides decide from core.compat, so the line + cannot drift between them. + """ + status_url = f"{self.url}/api/status/" + try: + response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) + response.raise_for_status() + version = response.json().get("netbox-version", "") + except requests.RequestException as exc: + raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc + except ValueError as exc: + # A proxy error page answers 200 with HTML, so the body is not JSON. + raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + self.supports_module_bay_types = supports_module_bay_types(version) + return self.supports_module_bay_types def close(self): """Close the underlying HTTP session.""" @@ -477,6 +517,12 @@ def get_module_types(self, manufacturer_slugs=None): sequence of non-blank strings. """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) + module_bay_type_selection = ( + "module_bay_types {\n id\n name\n slug\n" + " manufacturer {\n slug\n }\n }\n " + if self.supports_module_bay_types + else "" + ) query = f""" query($pagination: OffsetPaginationInput{var_decl}) {{ @@ -490,7 +536,7 @@ def get_module_types(self, manufacturer_slugs=None): weight weight_unit last_updated - manufacturer {{ + {module_bay_type_selection}manufacturer {{ id name slug @@ -801,6 +847,9 @@ def get_component_templates(self, endpoint_name, manufacturer_slug=None, on_page raise ValueError("manufacturer_slug must be None or a non-empty string") fields = component.graphql_fields + if not self.supports_module_bay_types: + # Selecting a field the server's schema lacks fails the whole query. + fields = [f for f in fields if f not in component.graphql_relation_fields] list_key = component.list_key parent_fields = "device_type { id }" diff --git a/core/module_bay_types.py b/core/module_bay_types.py new file mode 100644 index 00000000..68a23fbc --- /dev/null +++ b/core/module_bay_types.py @@ -0,0 +1,241 @@ +"""Resolve module-bay-type reference names to NetBox ids. + +A device type's bay says which classes of module it accepts, and a module type says +which classes it belongs to. Both sides name those classes as plain strings, so +something has to turn a name into the id of one specific NetBox object, creating it +when the target instance has never seen it. + +That is this module. Callers pass the owning manufacturer and the names; they get +ids back. The scope rule, the catalog files, the lookup, the creation, and the cache +stay in here, so the four call sites (device-type create, module-type create, change +detection, export) do not each restate them. +""" + +import os +import re + +import pynetbox +import requests +import yaml + +from core.errors import FatalError + +# Directory in the devicetype-library holding the catalog, one directory per manufacturer. +CATALOG_DIRNAME = "module-bay-types" + +# Second resolution scope, for a class one vendor defines and another fills. +FALLBACK_MANUFACTURER = "Generic" + + +def manufacturer_slug(name): + """Slugify a manufacturer name the way the library loader does. + + ``core.repo`` reduces every YAML ``manufacturer`` to this slug, because upstream data + disagrees on case (RuggedCOM vs RuggedCom). The catalog keys on the same slug so both + sides meet on one value. + """ + return re.sub(r"\W+", "-", (name or "").lower()) + + +class ModuleBayTypeError(FatalError): + """A reference could not be resolved to exactly one NetBox object. + + Raised per definition so the caller can log it and continue with the next one. + Never raised for a difference this module could paper over: an unresolved name and + a conflicting identity are both reported rather than guessed at. + """ + + +class ModuleBayTypeCatalog: + """Turn module-bay-type names into NetBox ids, creating what is missing. + + ``ids_for(manufacturer, names)`` resolves each name in the owning manufacturer's + scope, then in ``Generic``, and raises :class:`ModuleBayTypeError` if neither has + it. Resolution creates the NetBox object, and the manufacturer that owns it, when + they do not exist yet. Returned order is not meaningful; callers compare as sets. + + Export does not use this class: it reads the names off the records NetBox returned, + because an export run resolves nothing and so has no id-to-name mapping of its own. + """ + + def __init__(self, netbox, repo_path, handle): + """Store the NetBox client, the library checkout to read the catalog from, and the log handle.""" + self._netbox = netbox + self._catalog_dir = os.path.join(repo_path, CATALOG_DIRNAME) + self._handle = handle + self._entries = None + self._load_error = None + self._ids = {} + self._manufacturer_ids = {} + + def ids_for(self, manufacturer, names): + """Return the NetBox id for each name, resolved against *manufacturer* then Generic. + + *manufacturer* is a manufacturer slug, as produced by :func:`manufacturer_slug`. + An empty list is a valid instruction to hold no classes; anything that is not a + list of non-empty strings is refused rather than read as empty, because reading a + malformed value as empty would drop a restriction the author asked for. + """ + self._validate(names) + # Duplicates collapse: the relationship is a set. + return sorted({self._id_for(manufacturer, name) for name in names}) + + def identities_for(self, manufacturer, names): + """Return the identity each name resolves to, without touching NetBox. + + The identity is ``(manufacturer slug, slug)``: the object a reference means, not + the name it is written as. Two manufacturers may both define a class called + ``X``, so a name alone cannot say which object is intended. + + This is the query half of the module. :meth:`ids_for` is the command half and + creates what is missing; this one reads only the catalog files, so change + detection can ask what a reference means without causing anything to exist. + """ + self._validate(names) + return frozenset( + (manufacturer_slug(entry["manufacturer"]), entry["slug"]) + for entry in (self._lookup(manufacturer, name) for name in names) + ) + + @staticmethod + def _validate(names): + """Reject anything that is not a list of non-empty names.""" + if names is None or not isinstance(names, list): + raise ModuleBayTypeError(f"module_bay_types must be a list of names, got {names!r}") + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ModuleBayTypeError(f"module_bay_types entries must be non-empty names, got {name!r}") + + def _id_for(self, manufacturer, name): + entry = self._lookup(manufacturer, name) + cache_key = (manufacturer_slug(entry["manufacturer"]), entry["name"]) + if cache_key not in self._ids: + self._ids[cache_key] = self._netbox_id(entry) + return self._ids[cache_key] + + def _lookup(self, manufacturer, name): + """Find the catalog entry for *name*, owner scope first, then Generic.""" + entries = self._load_catalog() + for scope in (manufacturer, manufacturer_slug(FALLBACK_MANUFACTURER)): + entry = entries.get((scope, name)) + if entry is not None: + return entry + raise ModuleBayTypeError( + f"Module bay type {name!r} is not in the catalog for manufacturer " + f"{manufacturer!r} or {FALLBACK_MANUFACTURER!r}" + ) + + def _load_catalog(self): + """Read every catalog file once, indexed by (manufacturer, name). + + A failure is cached too: it is terminal for the run, and re-walking the tree for + every later lookup only repeats the same error more slowly. + """ + if self._entries is not None: + return self._entries + if self._load_error is not None: + raise self._load_error + try: + self._entries = self._read_catalog() + except ModuleBayTypeError as exc: + self._load_error = exc + raise + return self._entries + + def _read_catalog(self): + """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + entries = {} + for root, _dirs, files in os.walk(self._catalog_dir): + for filename in sorted(files): + if not filename.endswith((".yaml", ".yml")): + continue + path = os.path.join(root, filename) + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + except (OSError, yaml.YAMLError) as exc: + raise ModuleBayTypeError(f"Module bay type catalog file {path!r} could not be read: {exc}") from exc + if not isinstance(data, dict): + continue + # Reject a half-written entry here; the readers index on name and dereference slug. + invalid = [ + field + for field in ("name", "slug", "manufacturer") + if not isinstance(data.get(field), str) or not data[field].strip() + ] + if invalid: + raise ModuleBayTypeError( + f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" + ) + key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) + if key in entries: + raise ModuleBayTypeError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + entries[key] = data + return entries + + @staticmethod + def _request(action, description): + """Run one NetBox request, reporting a rejection as a catalog error. + + Callers resolve one definition at a time and recover from ModuleBayTypeError. A + raw RequestError or a dropped connection escapes that recovery and ends the run, + which can leave a parent half created and every later definition unprocessed. + """ + try: + return action() + except pynetbox.RequestError as exc: + raise ModuleBayTypeError(f"NetBox rejected {description}: {exc}") from exc + except requests.exceptions.RequestException as exc: + raise ModuleBayTypeError(f"NetBox could not be reached for {description}: {exc}") from exc + + def _netbox_id(self, entry): + """Return the id of the NetBox object for *entry*, creating it if absent.""" + manufacturer_id = self._manufacturer_id(entry["manufacturer"]) + existing = self._request( + lambda: list( + self._netbox.dcim.module_bay_types.filter(manufacturer_id=manufacturer_id, name=entry["name"]) + ), + f"the lookup of module bay type {entry['name']!r}", + ) + for record in existing: + if record.slug != entry["slug"]: + raise ModuleBayTypeError( + f"NetBox already has module bay type {entry['name']!r} for " + f"{entry['manufacturer']!r} with slug {record.slug!r}, but the catalog " + f"says {entry['slug']!r}. Resolve the conflict in NetBox; this import " + f"will not rename it." + ) + return record.id + + payload = {"name": entry["name"], "slug": entry["slug"], "manufacturer": manufacturer_id} + if entry.get("description"): + payload["description"] = entry["description"] + created = self._request( + lambda: self._netbox.dcim.module_bay_types.create(payload), + f"creating module bay type {entry['name']!r}", + ) + self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") + return created.id + + def _manufacturer_id(self, name): + """Return the id of *name*, creating the manufacturer when the catalog needs it. + + A Generic-scoped class is reachable from any vendor, so an import filtered to one + manufacturer still has to create the manufacturer that owns the class. + """ + if name in self._manufacturer_ids: + return self._manufacturer_ids[name] + found = self._request( + lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), + f"the lookup of manufacturer {name!r}", + ) + if found: + self._manufacturer_ids[name] = found[0].id + else: + created = self._request( + lambda: self._netbox.dcim.manufacturers.create({"name": name, "slug": manufacturer_slug(name)}), + f"creating manufacturer {name!r}", + ) + self._handle.verbose_log(f"Manufacturer Created: {name} - {created.id}") + self._manufacturer_ids[name] = created.id + return self._manufacturer_ids[name] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index c6535506..5ba5b2ca 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -7,7 +7,7 @@ import warnings from typing import Any, Sequence -from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES +from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS # Row order sets the component key order of the serialized YAML. COMPONENT_ENDPOINT_NAMES = [component.endpoint for component in COMPONENT_TYPES] @@ -125,6 +125,26 @@ def _serialize_component(record: Any, fields: Sequence[str]) -> dict: return result +def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: + """Return the catalog name of each related object, which is how YAML names them. + + The names come off the record the query returned, not from the import-side catalog: + an export run resolves nothing, so it has no id-to-name mapping of its own. + + An empty relation writes no key. Emitting an empty list instead would add the key to + every bay in the library and make _repo_supersedes report every existing definition as + differing, and it tells a fresh import nothing that omitting it does not. + """ + result = {} + for relation in relations: + names = sorted( + name for name in (getattr(item, "name", None) for item in getattr(record, relation, None) or []) if name + ) + if names: + result[relation] = names + return result + + def _serialize_front_port(record: Any) -> dict: """Serialize a front port template, including rear_port mapping.""" result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) @@ -161,12 +181,15 @@ def _serialize_front_port(record: Any) -> dict: def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" + component = BY_ENDPOINT[endpoint_name] out = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): if endpoint_name == "front_port_templates": - out.append(_serialize_front_port(record)) + serialized = _serialize_front_port(record) else: - out.append(_serialize_component(record, BY_ENDPOINT[endpoint_name].fields)) + serialized = _serialize_component(record, component.fields) + serialized.update(_serialize_relations(record, component.relations)) + out.append(serialized) return out @@ -236,6 +259,7 @@ def serialize_module_type(nb_record: Any, components_by_mt_id: dict) -> dict: if _should_include(field, val): result[field] = val + result.update(_serialize_relations(nb_record, MODULE_TYPE_RELATIONS)) _add_components(result, nb_record.id, components_by_mt_id) return result diff --git a/core/netbox_api.py b/core/netbox_api.py index 220efae1..979e704d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -5,7 +5,6 @@ from functools import lru_cache import hashlib import json -import re import tempfile import time import pynetbox @@ -24,7 +23,9 @@ LINK_POWER_PORT, LINK_REAR_PORTS, MODULE_TYPE_COMPONENTS, + MODULE_TYPE_RELATIONS, ) +from core.compat import parse_netbox_version, supports_module_bay_types from core.formatting import log_property_diffs from core.errors import FatalError, UnknownError from core.graphql_client import GraphQLError, NetBoxGraphQLClient @@ -50,6 +51,35 @@ def __init__(self, ignore_ssl_errors: bool, cause=None): ) +# Oldest NetBox this importer is tested against; see the CI matrix and the README. +MINIMUM_NETBOX_VERSION = (4, 3) + + +def _relation_identities_differ(catalog, manufacturer, declared, related, *, names_differ): + """Return True when the assigned objects are not the ones the references resolve to. + + Comparing names alone reports equality when NetBox holds a same-named class from + another manufacturer's scope, so the identity is compared where both sides can supply + one. Where they cannot, the name comparison the caller already made stands. + """ + from core.module_bay_types import ModuleBayTypeError, manufacturer_slug + + if not catalog or not manufacturer: + return names_differ + current = set() + for item in related: + slug = getattr(item, "slug", None) + owner = getattr(getattr(item, "manufacturer", None), "slug", None) + if not slug or not owner: + return names_differ + current.add((manufacturer_slug(owner), slug)) + try: + return catalog.identities_for(manufacturer, declared) != frozenset(current) + except ModuleBayTypeError: + # A matching name must not make an unresolvable reference look applied. + return True + + class NetBoxError(FatalError): """A fatal error reported by the NetBox integration.""" @@ -464,6 +494,7 @@ def __init__(self, config, handle): self.ignore_ssl = config.ignore_ssl_errors self.modules = False self.new_filters = False + self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts @@ -495,6 +526,7 @@ def __init__(self, config, handle): self.ignore_ssl, handle=self.handle, page_size=config.graphql_page_size, + supports_module_bay_types=self.module_bay_types, ) try: self.existing_manufacturers = self.get_manufacturers() @@ -509,6 +541,7 @@ def __init__(self, config, handle): self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, + module_bay_types_supported=self.module_bay_types, repo_path=config.repo_path, max_threads=config.preload_threads, ) @@ -601,8 +634,15 @@ def verify_compatibility(self): msg += f"\nResponse body (may be from an intermediate proxy):\n{body}" msg += f"\nHint: Verify that {self.url} is reachable and not blocked by a proxy." raise NetBoxError(msg) from e - _raw = [int(re.sub(r"\D.*", "", x.strip()) or "0") for x in nb_version.split(".")] - version_split = (_raw + [0, 0])[:2] # pad to (major, minor) to guard against single-component strings + version_split = parse_netbox_version(nb_version) + + # Below the floor the run dies later naming a schema field, not the real cause. + if tuple(version_split) < MINIMUM_NETBOX_VERSION: + minimum = ".".join(str(part) for part in MINIMUM_NETBOX_VERSION) + raise NetBoxError( + f"NetBox {nb_version} is not supported: this importer requires NetBox {minimum} or later. " + f"Older releases fail part way through with a GraphQL schema error rather than here." + ) # Later than 3.2 # Might want to check for the module-types entry as well? @@ -622,6 +662,10 @@ def verify_compatibility(self): self.m2m_front_ports = True self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + if supports_module_bay_types(nb_version): + self.module_bay_types = True + self.handle.log(f"Netbox version {self.netbox.version} found. Module bay types are supported.") + def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" return self.graphql.get_manufacturers() @@ -1162,7 +1206,13 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima continue if yaml_key == "module-bays" and not self.modules: continue - self.device_types.create_components(yaml_key, device_type[yaml_key], dt_id, context=src_file) + self.device_types.create_components( + yaml_key, + device_type[yaml_key], + dt_id, + context=src_file, + manufacturer=device_type.get("manufacturer"), + ) if component_errors: # The type exists but not all of its components do. self.outcomes.record( @@ -1522,6 +1572,8 @@ def filter_actionable_module_types(self, module_types, all_module_types, only_ne if not values_equal(module_type[f], nb_val): changed_fields_info.append((f, nb_val, module_type[f])) + changed_fields_info += self._type_relation_changes(module_type, existing_module) + component_changes = detector._compare_components(module_type, existing_module.id, parent_type="module") if changed_fields_info or component_changes: @@ -1574,6 +1626,57 @@ def _fetch_module_type_existing_images(self): ) return module_type_existing_images + def _type_relation_changes(self, module_type, existing_module): + """Return (field, current, wanted) for each of the type's own relations that differs. + + A relation is a list, so the scalar comparison loop never sees it. A field the + query did not return is skipped rather than read as empty, which would otherwise + report a change on every run. + """ + changes: list[tuple[str, list[str], list[str]]] = [] + if not self.module_bay_types: + return changes + for field in MODULE_TYPE_RELATIONS: + if field not in module_type: + continue + netbox_value = getattr(existing_module, field, _MISSING) + if netbox_value is _MISSING: + continue + related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] + declared = module_type[field] + if not isinstance(declared, list) or any(not isinstance(x, str) or not x for x in declared): + # Malformed or bare key: leave the relation unmanaged rather than clear it. + continue + wanted = sorted(set(declared)) + current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) + if _relation_identities_differ( + self.device_types.module_bay_types, + self.device_types._manufacturer_slug(module_type.get("manufacturer")), + declared, + related, + names_differ=wanted != current, + ): + changes.append((field, current, wanted)) + return changes + + def _resolve_type_relations(self, payload): + """Return *payload* with its own relation fields resolved from names to ids. + + Raises ModuleBayTypeError if a name does not resolve, so the caller can report the + module type rather than write it with the restriction silently dropped. + """ + names = {field: payload[field] for field in MODULE_TYPE_RELATIONS if field in payload} + if not names: + return payload + if not self.module_bay_types: + # The server predates ModuleBayType; sending the field would be rejected. + return {k: v for k, v in payload.items() if k not in names} + manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) + resolved = { + field: self.device_types.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + } + return {**payload, **resolved} + def _try_update_module_type(self, curr_mt, module_type_res, src_file): """Apply pending field updates to an existing module type in NetBox. @@ -1590,6 +1693,18 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + if self.module_bay_types: + from core.module_bay_types import ModuleBayTypeError + + for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): + try: + updates[field] = self.device_types.module_bay_types.ids_for( + self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted + ) + except ModuleBayTypeError as exc: + # Not fatal to the run, but not a success either; the caller records it. + self.handle.log(f"Skipped {field} on {curr_mt.get('model')}: {exc} (Context: {src_file})") + return False, False if not updates: return True, False try: @@ -1622,7 +1737,12 @@ def _create_module_type_components(self, curr_mt, module_type_id, src_file): yaml_key = component.yaml_key if yaml_key in curr_mt: self.device_types.create_components( - yaml_key, curr_mt[yaml_key], module_type_id, parent_type="module", context=src_file + yaml_key, + curr_mt[yaml_key], + module_type_id, + parent_type="module", + context=src_file, + manufacturer=curr_mt.get("manufacturer"), ) if component_errors: # The module type exists but not all of its components do. @@ -1776,7 +1896,20 @@ def _process_single_module_type( ) else: try: - module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, curr_mt) + from core.module_bay_types import ModuleBayTypeError + + try: + payload = self._resolve_type_relations(curr_mt) + except ModuleBayTypeError as exc: + self.handle.log(f"Error creating Module Type: {exc} (Context: {src_file})") + self._record_failure( + EntityKind.MODULE_TYPE, + self._yaml_identity(curr_mt), + str(exc), + src_file, + ) + return False + module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, payload) self.counter["module_added"] += 1 is_new = True manufacturer_slug = curr_mt["manufacturer"]["slug"] @@ -2192,6 +2325,7 @@ def __init__( graphql, repo_path, m2m_front_ports=False, + module_bay_types_supported=False, max_threads=8, ): """Initialize empty DeviceTypes cache structures; no data is fetched at construction time. @@ -2206,6 +2340,7 @@ def __init__( ignore_ssl (bool): Whether SSL certificate verification is disabled. new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. + module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. m2m_front_ports (bool): Whether NetBox uses the 4.5+ M2M port mapping model. max_threads (int): Maximum number of concurrent threads for component preloading. @@ -2218,6 +2353,7 @@ def __init__( self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports + self.module_bay_types_supported = module_bay_types_supported self.max_threads = max_threads self.components = ComponentCache( netbox, @@ -2228,6 +2364,7 @@ def __init__( wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None + self._module_bay_types = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2486,6 +2623,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, if change.component_name in existing: comp = existing[change.component_name] update_data = {"id": comp.id} + unresolved = False for pc in change.property_changes: if comp_type == "front-ports" and pc.property_name == "_mappings": yaml_front_port = next( @@ -2501,8 +2639,21 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, parent_type, ) continue + if pc.property_name in component.relations: + # The comparison works in names; NetBox wants ids. + from core.module_bay_types import ModuleBayTypeError + + try: + update_data[pc.property_name] = self.module_bay_types.ids_for( + self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value + ) + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{change.component_name}': {exc}") + unresolved = True + break + continue update_data[pc.property_name] = pc.new_value - if len(update_data) > 1: # has fields beyond just "id" + if not unresolved and len(update_data) > 1: # has fields beyond just "id" updates.append(update_data) success_count = 0 @@ -2547,7 +2698,13 @@ def _apply_additions_for_type(self, comp_type, changes, yaml_data, device_type_i if not components_to_add: return - self.create_components(comp_type, components_to_add, device_type_id, parent_type=parent_type) + self.create_components( + comp_type, + components_to_add, + device_type_id, + parent_type=parent_type, + manufacturer=yaml_data.get("manufacturer"), + ) def update_components(self, yaml_data, device_type_id, component_changes, parent_type="device"): """Update existing components and add new components based on detected changes. @@ -2837,7 +2994,66 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): f"Connection error bridging interfaces after {_MAX_RETRIES} retries: {e} (Context: {context})" ) - def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None): + @property + def module_bay_types(self): + """The module-bay-type catalog, built once per run from the library checkout.""" + if self._module_bay_types is None: + from core.module_bay_types import ModuleBayTypeCatalog + + self._module_bay_types = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_types + + @staticmethod + def _manufacturer_slug(manufacturer): + """Return the manufacturer slug, whatever shape the caller happens to hold. + + ``core.repo`` rewrites every YAML ``manufacturer`` to ``{"slug": ...}`` before the + importer sees it, so that is the usual shape; a plain name is slugified here. + """ + from core.module_bay_types import manufacturer_slug + + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or manufacturer_slug(manufacturer.get("name")) + return manufacturer_slug(getattr(manufacturer, "name", manufacturer)) + + def _resolve_relations(self, component, items, manufacturer): + """Turn each relation field's names into NetBox ids, dropping items that cannot resolve. + + A component whose restriction cannot be resolved is skipped and logged rather than + created without it: creating the bay anyway would silently discard the restriction. + """ + if not component.relations: + return items + if not self.module_bay_types_supported: + # The server predates ModuleBayType; drop the field rather than have it rejected. + return [{k: v for k, v in item.items() if k not in component.relations} for item in items] + from core.module_bay_types import ModuleBayTypeError + + manufacturer = self._manufacturer_slug(manufacturer) + resolved = [] + for item in items: + names = {field: item[field] for field in component.relations if field in item} + if not names: + resolved.append(item) + continue + if not manufacturer: + # No scope to resolve in, and NetBox wants ids: sending the names would fail. + self._log_component_error( + f"Skipped {component.label} '{item.get('name', 'Unknown')}': no manufacturer to " + f"resolve {', '.join(sorted(names))} in" + ) + continue + try: + replacements = { + field: self.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + } + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") + continue + resolved.append({**item, **replacements}) + return resolved + + def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None, manufacturer=None): """Create component templates of one kind for one parent, skipping those that exist. The registry row for *yaml_key* supplies the endpoint, the cache name, the log label @@ -2850,6 +3066,8 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co parent_id (int): NetBox ID of the parent device or module type. parent_type (str): ``"device"`` or ``"module"``. context (str | None): Optional context string appended to log messages. + manufacturer (dict | str | None): Owning manufacturer, used to resolve any + relation fields the registry row declares. """ component = BY_YAML_KEY[yaml_key] label = component.create_label(parent_type) @@ -2869,7 +3087,7 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co self._create_generic( component, - items, + self._resolve_relations(component, items, manufacturer), parent_id, parent_type=parent_type, post_process=post_process, diff --git a/tests/conftest.py b/tests/conftest.py index 364bd128..a1df5ff4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,14 @@ def mock_git_repo(request): @pytest.fixture def mock_pynetbox(): - """Mock pynetbox to prevent API calls.""" + """Mock pynetbox to prevent API calls. + + Defaults the reported version to the oldest supported release, so a test that does not + care about version gating still constructs a NetBox; the importer refuses anything + older. Individual tests override it to exercise a specific release. + """ with patch("core.netbox_api.pynetbox") as mock_nb: + mock_nb.api.return_value.version = "4.3" yield mock_nb diff --git a/tests/helpers.py b/tests/helpers.py index 9bbad389..c6f0f381 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,13 @@ """Shared test utilities for the NetBox device-type importer test suite.""" +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlparse + +import pynetbox def paginate_dispatch(data_dict): @@ -49,3 +56,195 @@ def recording_handle(): console = RecordingConsole() handle.set_console(console) return handle, console + + +class FakeNetBox: + """A local HTTP server answering the slice of the NetBox REST API the importer uses. + + Collections are addressed by their URL segment with dashes turned into underscores, + so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates and PATCH + bulk-updates behave the way pynetbox expects, so a real client drives it and the + serialising, filtering and paginating under test are the production ones. + + ``/graphql/`` answers every query with an empty list, which is enough to bring a real + :class:`~core.netbox_api.NetBox` up; the REST half is where the assertions live. + + A test using this must carry the ``real_http`` marker. Without it the suite patches + ``requests.Session`` and no request ever leaves the client. + """ + + _IGNORED_FILTERS = ("limit", "offset", "brief", "exclude") + + # GraphQL collections are named "_list"; the client asks for one per query. + _LIST_KEY = re.compile(r"\b(\w+_list)\b") + + def __init__(self, errors=None, netbox_version="4.7.0", **collections): + """Start the server with *collections* seeded, keyed by collection name. + + *errors* maps a collection name to an HTTP status the server answers it with, so a + test can drive a rejection the importer has to survive. *netbox_version* is what + ``/api/status/`` reports, which is how the export side decides what it may select. + """ + self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} + self.errors = dict(errors or {}) + self.netbox_version = netbox_version + self.requests = [] + self._server = HTTPServer(("127.0.0.1", 0), self._handler()) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def url(self): + """Return the base URL a client should talk to.""" + return f"http://127.0.0.1:{self._server.server_port}" + + def api(self, token="test-token"): + """Return a real pynetbox client pointed at this server.""" + return pynetbox.api(self.url, token=token) + + def close(self): + """Stop serving and release the listening socket.""" + self._server.shutdown() + self._server.server_close() + + def collection(self, name): + """Return the stored records for a collection, creating it empty when unseen.""" + return self.collections.setdefault(name, []) + + def sent(self, verb, name): + """Return the payload of each *verb* request made to collection *name*. + + A bulk create or update sends a list; its entries are returned individually, so a + caller asserting on what was written does not have to care which shape was used. + """ + out = [] + for method, collection, payload in self.requests: + if method != verb or collection != name: + continue + out.extend(payload) if isinstance(payload, list) else out.append(payload) + return out + + def matches(self, name, query): + """Filter a collection the way the NetBox REST API filters it.""" + out = [] + for record in self.collection(name): + ok = True + for key, values in query.items(): + if key in self._IGNORED_FILTERS: + continue + field = key[:-3] if key.endswith("_id") else key + actual = record.get(field) + if isinstance(actual, dict): + actual = actual.get("id") + if str(actual) not in values: + ok = False + break + if ok: + out.append(record) + return out + + def _create(self, name, payload): + """Store the new record(s) and echo them back the way NetBox does. + + A list payload is a bulk create, which is how the importer adds components, and it + answers with a list. + """ + if isinstance(payload, list): + return [self._create_one(name, item) for item in payload] + return self._create_one(name, payload) + + def _create_one(self, name, payload): + """Store one new record and return it as NetBox would echo it back.""" + collection = self.collection(name) + record = {"id": 1000 + len(collection), **payload} + # NetBox echoes a foreign key back as a nested object, not the id it was given. + if isinstance(record.get("manufacturer"), int): + owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) + record["manufacturer"] = {"id": record["manufacturer"], "name": owner.get("name")} + collection.append(record) + return record + + def _patch(self, name, payload): + """Merge a bulk update into the stored records and return the updated ones.""" + updated = [] + for entry in payload if isinstance(payload, list) else [payload]: + record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) + if record is None: + continue + record.update(entry) + updated.append(record) + return updated + + def _handler(self): + """Build the request handler class bound to this server's state.""" + state = self + + class Handler(BaseHTTPRequestHandler): + def _collection_name(self): + return urlparse(self.path).path.rstrip("/").rsplit("/", 1)[-1].replace("-", "_") + + def _body(self): + return json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + + def _reply(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _refused(self, name): + """Answer with the configured status for *name*, if the test set one.""" + status = state.errors.get(name) + if status is None: + return False + self._reply(status, {"detail": f"{name} refused with {status}"}) + return True + + def do_GET(self): + parsed = urlparse(self.path) + name = self._collection_name() + state.requests.append(("GET", name, parsed.query)) + if name == "status": + self._reply(200, {"netbox-version": state.netbox_version}) + return + if self._refused(name): + return + results = state.matches(name, parse_qs(parsed.query)) + self._reply(200, {"count": len(results), "next": None, "previous": None, "results": results}) + + def do_POST(self): + name = self._collection_name() + payload = self._body() + if name == "graphql": + state.requests.append(("POST", name, payload)) + key = state._LIST_KEY.search(payload.get("query", "")) + self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) + return + state.requests.append(("POST", name, payload)) + if self._refused(name): + return + self._reply(201, state._create(name, payload)) + + def do_PATCH(self): + name = self._collection_name() + payload = self._body() + state.requests.append(("PATCH", name, payload)) + if self._refused(name): + return + self._reply(200, state._patch(name, payload)) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + return Handler + + +def write_module_bay_type(root, manufacturer, slug, name, description=None): + """Write one module-bay-type catalog file where the devicetype-library puts it.""" + directory = root / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + body = f"name: {name}\nslug: {slug}\nmanufacturer: {manufacturer}\n" + if description: + body += f"description: {description}\n" + (directory / f"{slug}.yaml").write_text(body, encoding="utf-8") diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 2b19c784..7e536f53 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -131,7 +131,14 @@ def test_the_query_selects_exactly_these_fields(self): _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], - "module_bay_templates": ["id", "name", "position", "label", "description"], + "module_bay_templates": [ + "id", + "name", + "position", + "label", + "description", + "module_bay_types { id name slug manufacturer { slug } }", + ], } def test_only_the_front_port_query_selects_a_nested_block(self): @@ -146,13 +153,17 @@ class TestDerivedComparisonAndExport: @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_every_compared_property_is_fetched(self, component): """A property compared but never queried reads as missing and is skipped in silence.""" - queried = set(component.graphql_fields) | {"_mappings"} + # A relation is selected as a nested block, so take the field name each selection + # opens with. Adding component.relations here instead would make the assertion + # hold even if the query stopped selecting them. + queried = {selection.split(None, 1)[0] for selection in component.graphql_fields} | {"_mappings"} assert set(component.compare_properties) <= queried @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_the_export_writes_every_scalar_the_query_reads(self, component): """Export fields are the query's scalars: an unexported scalar drops out of a round trip.""" - scalars = [name for name in component.graphql_fields if name != "id" and name not in component.graphql_extra] + non_scalar = set(component.graphql_extra) | set(component.graphql_relation_fields) + scalars = [name for name in component.graphql_fields if name != "id" and name not in non_scalar] assert list(component.fields) == scalars def test_front_ports_compare_the_mapping_the_query_selects(self): diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 18c06bf2..d9eebe29 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -239,6 +239,26 @@ def test_yaml_equal_normalizes_component_order_and_numbers(self): class TestRepoSupersedes: """Tests for _repo_supersedes / _is_subset (asymmetric containment).""" + def test_an_empty_relation_does_not_make_every_definition_differ(self): + """The serializer omits an empty relation, and this is why it has to. + + _is_subset requires every NetBox leaf to be present in the repo YAML, and + _normalize_for_compare does not drop empty lists. A serialized + "module_bay_types: []" would therefore be absent from every library definition + and re-export the whole library on a NetBox 4.7 server. + """ + from core.nb_serializer import _serialize_relations + + bay = type("Bay", (), {"name": "FPC 0", "module_bay_types": []})() + assert _serialize_relations(bay, ("module_bay_types",)) == {} + + repo = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} + as_serialized = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} + with_empty_key = {"model": "MX304", "module-bays": [{"name": "FPC 0", "module_bay_types": []}]} + + assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" + assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -853,11 +873,17 @@ def test_fetch_vendor_components_groups_device_and_module_records(self, tmp_path def _side_effect(endpoint_name, manufacturer_slug=None): return [dt_rec, mt_rec] if endpoint_name == "interface_templates" else [] + # A worker must fetch through its own clone, so only the clone answers. + worker = MagicMock() + worker.get_component_templates.side_effect = _side_effect mock_client = MagicMock() - mock_client.get_component_templates.side_effect = _side_effect - with patch("core.export.NetBoxGraphQLClient", return_value=mock_client): - dt_result, mt_result = exporter._fetch_vendor_components("nokia") + mock_client.get_component_templates.side_effect = AssertionError("worker must use clone()") + mock_client.clone.return_value = worker + exporter.graphql = mock_client + + dt_result, mt_result = exporter._fetch_vendor_components("nokia") + assert mock_client.clone.called assert dt_result[11]["interface_templates"] == [dt_rec] assert mt_result[22]["interface_templates"] == [mt_rec] diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 6c5b22b5..b72ddae6 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -118,22 +118,115 @@ def test_logging_dependency_has_a_public_read_only_handle(self): client.handle = LogHandler(True) def test_clone_uses_an_independent_session_with_the_same_settings(self): + """Every stored setting is compared, so a new one cannot be dropped unnoticed. + + clone() re-lists constructor arguments by hand. Listing the settings here by hand + too let supports_module_bay_types default back to False in every cloned worker. + """ handle = MagicMock() sessions = [MagicMock(), MagicMock()] with patch("core.graphql_client.requests.Session", side_effect=sessions): - client = NetBoxGraphQLClient("http://netbox.local", "token", True, handle, 250) + client = NetBoxGraphQLClient( + "http://netbox.local", "token", True, handle, 250, supports_module_bay_types=True + ) clone = client.clone() assert clone is not client assert clone._session is sessions[1] - assert (clone.url, clone.token, clone.ignore_ssl, clone.handle, clone.DEFAULT_PAGE_SIZE) == ( - client.url, - client.token, - client.ignore_ssl, - client.handle, - client.DEFAULT_PAGE_SIZE, - ) + settings = lambda c: {k: v for k, v in vars(c).items() if k != "_session"} # noqa: E731 + assert settings(clone) == settings(client) + + @pytest.mark.real_http + def test_the_cloned_session_is_configured_not_merely_new(self): + """A bare requests.Session() would be independent and completely unauthenticated. + + Marked real_http only to get real Session objects; nothing here sends a request. + """ + client = NetBoxGraphQLClient("http://netbox.local", "nbt_key.secret", ignore_ssl=True) + + clone = client.clone() + + assert clone._session is not client._session + assert clone._session.headers["Authorization"] == "Bearer nbt_key.secret" + assert clone._session.headers["Content-Type"] == "application/json" + assert clone._session.verify is False + + @pytest.mark.real_http + def test_a_v1_token_clone_keeps_the_legacy_auth_scheme(self): + client = NetBoxGraphQLClient("http://netbox.local", "0123456789abcdef") + + assert client.clone()._session.headers["Authorization"] == "Token 0123456789abcdef" + + @pytest.mark.real_http + def test_an_unreachable_server_fails_the_probe_as_a_graphql_error(self): + """It is the export's first request, so a raw transport error escapes as a traceback.""" + from core.graphql_client import GraphQLError + from helpers import FakeNetBox + + server = FakeNetBox() + url = server.url + server.close() # nothing is listening on that port any more + + with pytest.raises(GraphQLError): + NetBoxGraphQLClient(url, "tok").detect_module_bay_type_support() + + @pytest.mark.real_http + def test_a_non_json_status_body_fails_the_probe_as_a_graphql_error(self): + """A proxy error page answers 200 with HTML; json() then raises ValueError.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"gateway" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): + """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" + from helpers import FakeNetBox + + for version, expected in (("4.7.0", True), ("4.7.0-beta2", True), ("4.6.9", False), ("4.3.7", False)): + server = FakeNetBox(netbox_version=version) + try: + client = NetBoxGraphQLClient(server.url, "tok") + assert client.detect_module_bay_type_support() is expected, version + assert client.supports_module_bay_types is expected + finally: + server.close() + + def test_clone_carries_a_setting_it_does_not_name(self): + """clone() must not enumerate settings: the one it forgets is the one that breaks. + + supports_module_bay_types was lost exactly that way, and adding it to the argument + list only fixes the setting that was already missed. + """ + with patch("core.graphql_client.requests.Session"): + client = NetBoxGraphQLClient("http://netbox.local", "token") + client.added_after_this_test_was_written = "carried" + clone = client.clone() + + assert clone.added_after_this_test_was_written == "carried" def test_init_stores_config(self): from core.graphql_client import NetBoxGraphQLClient @@ -1025,6 +1118,40 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): + """The prefetch runs on clones, not on the client it was cloned from. + + A clone that drops the relation reads every bay without it, and the comparison + then skips a field it never received. + """ + from core.graphql_client import NetBoxGraphQLClient + + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=True) + mock_post.side_effect = _make_paged_responses({"module_bay_template_list": []}, "module_bay_template_list") + + client.clone().get_component_templates("module_bay_templates") + + queries = [call.kwargs["json"]["query"] for call in mock_post.call_args_list] + assert queries and all("module_bay_types" in q for q in queries) + + def test_the_module_type_query_selects_the_relation_only_when_supported(self, mock_post): + """The selection has to track the server, in both directions. + + An unselected field reads as missing and is skipped by the comparison, and + selecting it on a server below 4.7 fails the whole query. + """ + from core.graphql_client import NetBoxGraphQLClient + + def _query_for(supported): + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=supported) + mock_post.reset_mock() + mock_post.side_effect = _make_paged_responses({"module_type_list": []}, "module_type_list") + client.get_module_types() + return mock_post.call_args_list[0].kwargs["json"]["query"] + + assert "module_bay_types" in _query_for(True) + assert "module_bay_types" not in _query_for(False) + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py new file mode 100644 index 00000000..ed2a27e1 --- /dev/null +++ b/tests/test_module_bay_type_sync.py @@ -0,0 +1,577 @@ +"""Tests for keeping module bay type assignments in sync with NetBox. + +Resolving a reference name to an id is :mod:`core.module_bay_types` and is tested there. +This file covers the wiring around it: the relation a module type carries itself, the +component update path, and the gate that keeps the field off a server that predates it. + +Both sides are real. The catalog is written to disk and read by a real +``ModuleBayTypeCatalog``; a local HTTP server answers a real ``pynetbox`` client and the +real GraphQL client, so the payloads asserted on here are the ones that would go over the +wire. Only the version handshake is stood in for, to fix the server release under test. +""" + +import pynetbox +import pytest + +from core.change_detector import ChangeType, ComponentChange, PropertyChange +from core.component_registry import BY_YAML_KEY +from core.graphql_client import NetBoxGraphQLClient +from core.module_bay_types import ModuleBayTypeCatalog +from core.netbox_api import DeviceTypes, NetBox +from core.outcomes import EntityKind, Outcome +from helpers import FakeNetBox, recording_handle, write_module_bay_type + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +JUNIPER = {"id": 1, "name": "Juniper", "slug": "juniper"} + + +def stub(**attrs): + """Return an object carrying exactly *attrs*, the way NetBox nests a related object.""" + return type("Stub", (), attrs)() + + +def created_id(server, slug): + """Return the id the server gave the module bay type it created for *slug*.""" + return next(record["id"] for record in server.collection("module_bay_types") if record["slug"] == slug) + + +def related(name, slug=None, manufacturer_slug=None): + """Return a module bay type as NetBox returns it nested inside a relation.""" + return stub(name=name, slug=slug, manufacturer=stub(slug=manufacturer_slug)) + + +class NetBoxRecord: + """An existing NetBox object as the importer holds it: an id, plus what was fetched. + + A field the query did not ask for is genuinely absent, which is the difference the + comparison turns on, so this carries only what a test hands it. + """ + + def __init__(self, record_id, **fields): + """Store the id and the fields this record is meant to have.""" + self.id = record_id + for name, value in fields.items(): + setattr(self, name, value) + + +@pytest.fixture +def catalog_root(tmp_path): + """Write the catalog these tests resolve against and return its root.""" + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot") + return root + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server seeded with the manufacturers the catalog needs.""" + fake = FakeNetBox(manufacturers=[JUNIPER]) + yield fake + fake.close() + + +@pytest.fixture +def make_device_types(server, catalog_root): + """Build a real DeviceTypes talking to the local server, with its cache already primed.""" + + def _make(module_bay_types_supported=True): + handle, console = recording_handle() + device_types = DeviceTypes( + server.api(), + handle, + {}, + False, + True, + graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), + repo_path=str(catalog_root), + module_bay_types_supported=module_bay_types_supported, + ) + device_types.components.ensure_ready() + return device_types, console + + return _make + + +@pytest.fixture +def netbox(make_config, mock_pynetbox, server, catalog_root): + """Build a real NetBox against the local server, reporting the release that has the feature.""" + mock_pynetbox.api.return_value.version = "4.7" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, console = recording_handle() + config = make_config(netbox_url=server.url, repo_path=str(catalog_root)) + nb = NetBox(config, handle) + assert nb.module_bay_types, "a 4.7 server supports module bay types; the rest of this file assumes it" + + # connect_api ran against the patched pynetbox; every call under test uses a real client. + api = server.api() + nb.netbox = api + nb.device_types.netbox = api + nb.device_types.components.netbox = api + nb.device_types.components.ensure_ready() + return nb, console + + +class TestModuleTypeOwnRelation: + """A module type says which classes it belongs to, and that has to stay in sync.""" + + def test_a_missing_class_is_reported_as_a_change(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", [], ["MX304-RE"])] + + def test_the_right_class_is_left_alone(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): + """Juniper owns MX304-RE. A Generic object of that name is not the one referenced.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "generic")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + + def test_names_are_compared_when_netbox_returned_no_identity(self, netbox): + """A record carrying only a name cannot answer the scope question; names still can.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_unresolvable_reference_is_reported_even_when_the_name_matches(self, netbox): + """A name that happens to match must not make an unresolvable reference look applied. + + Comparison cannot say which object the name means, so the write path has to try, + fail, and refuse. Reporting no change here skips that entirely. + """ + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["NO-SUCH-CLASS"]} + + assert nb._type_relation_changes(module_type, existing) == [ + ("module_bay_types", ["NO-SUCH-CLASS"], ["NO-SUCH-CLASS"]) + ] + + def test_a_matching_but_unresolvable_name_does_not_read_as_updated(self, netbox, server): + """The end of that path: the module type is refused, not patched with its scalars.""" + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + assert nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") == (False, False) + assert not server.sent("PATCH", "module_types") + + def test_a_field_the_query_did_not_return_is_skipped(self, netbox): + """Reading an absent field as empty would report a change on every run.""" + nb, _ = netbox + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, NetBoxRecord(7)) == [] + + def test_an_omitted_key_leaves_the_relation_unmanaged(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + + assert nb._type_relation_changes({"model": "JNP304-RE"}, existing) == [] + + def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox): + """A bare key parses as None; clearing on it would drop a restriction nobody removed.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": None} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_names_are_compared_when_the_definition_names_no_manufacturer(self, netbox): + """Without an owning manufacturer there is no scope, so the name comparison stands.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC", "mx304-lmic", "juniper")]) + module_type = {"model": "JNP304-RE", "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_older_server_reports_no_relation_changes(self, netbox): + """Below 4.7 the relation does not exist, so there is nothing to compare.""" + nb, _ = netbox + nb.module_bay_types = False + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + +class TestModuleTypeCreatePayload: + """What the create path sends for a module type's own relation.""" + + def test_names_are_replaced_by_ids(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + resolved = nb._resolve_type_relations(payload) + + created = server.sent("POST", "module_bay_types") + assert [p["slug"] for p in created] == ["mx304-re"] + assert resolved["module_bay_types"] == [created_id(server, "mx304-re")] + assert resolved["model"] == "JNP304-RE" + + def test_a_payload_without_the_relation_is_untouched(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + + assert nb._resolve_type_relations(payload) == payload + assert not server.sent("POST", "module_bay_types") + + def test_an_older_server_never_sees_the_field(self, netbox, server): + """Sending a field the server does not have would fail the whole create.""" + nb, _ = netbox + nb.module_bay_types = False + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._resolve_type_relations(payload) == {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + assert not server.sent("POST", "module_bay_types") + + +class TestModuleTypeUpdate: + """The update path patches the relation, and survives one it cannot resolve.""" + + def test_a_changed_relation_is_patched_as_ids(self, netbox, server): + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (True, True) + assert server.sent("PATCH", "module_types") == [{"id": 7, "module_bay_types": [created_id(server, "mx304-re")]}] + + def test_an_unresolvable_reference_is_logged_and_the_run_goes_on(self, netbox, server): + """One bad module type must not end the run, and must not be written without it.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False), "an unapplied relation must not read as success" + assert not server.sent("PATCH", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + def test_a_scalar_change_is_held_back_when_the_relation_cannot_resolve(self, netbox, server): + """Patching the description while dropping the restriction is a half-applied write.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False) + assert not server.sent("PATCH", "module_types"), "the scalar PATCH must not go out alone" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestModuleTypeCreateRefusal: + """A module type whose relation cannot be resolved is reported, not written without it.""" + + def test_an_unresolvable_reference_skips_the_module_type(self, netbox, server): + nb, console = netbox + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + created = nb._process_single_module_type(curr_mt, "juniper/jnp304-re.yaml", {}, {}, only_new=False) + + assert created is False + assert not server.sent("POST", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + # The registry is the only tally of failures; a path that merely logs is absent + # from the run summary and from the itemised report. + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [(r.kind, r.identity) for r in failed] == [(EntityKind.MODULE_TYPE, "juniper/JNP304-RE")] + assert "NO-SUCH-CLASS" in failed[0].reason + + +class TestModuleTypeUpdateOutcome: + """The run summary must show the module type whose relation could not be applied.""" + + def test_an_unresolvable_relation_is_recorded_as_a_failure(self, netbox, server): + """Driven through the parent operation, because that is where the outcome is recorded.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[], description="old" + ) + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + nb._process_single_module_type( + curr_mt, "juniper/jnp304-re.yaml", {"juniper": {"JNP304-RE": existing}}, {}, only_new=False + ) + + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [r.kind for r in failed] == [EntityKind.MODULE_TYPE], "the failure must reach the run summary" + assert not server.sent("PATCH", "module_types"), "nothing may be written for a type it could not apply" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestComponentCreatePayload: + """A module bay is created with the restriction its definition asked for, or not at all.""" + + def test_names_are_replaced_by_ids(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert resolved == [{"name": "FPC 0", "module_bay_types": [created_id(server, "mx304-lmic")]}] + + def test_an_item_without_the_field_passes_through(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == items + assert not server.sent("POST", "module_bay_types") + + def test_an_unresolvable_restriction_drops_the_bay_rather_than_relaxing_it(self, make_device_types, server): + """Creating the bay without its restriction would silently accept any module.""" + device_types, console = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [ + {"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}, + {"name": "FPC 1", "module_bay_types": ["MX304-LMIC"]}, + ] + + with device_types.collect_component_errors() as collected: + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert [item["name"] for item in resolved] == ["FPC 1"] + # Collected, not just printed: the parent's outcome reason is built from these. + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + assert any("FPC 0" in line for line in console.lines) + + def test_an_older_server_never_sees_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == [{"name": "FPC 0"}] + assert not server.sent("POST", "module_bay_types") + + def test_a_netbox_rejection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A 403 while resolving must not escape the per-component recovery. + + The callers recover from ModuleBayTypeError only, so a raw RequestError ends the + run with a traceback and leaves a partly created parent behind. + """ + device_types, _ = make_device_types() + server.errors["module_bay_types"] = 403 + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e and "403" in e] + + def test_a_lost_connection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A dropped connection is not a RequestError, so it escaped the same recovery.""" + device_types, _ = make_device_types() + server.close() # the port stops answering; resolution now hits a refused connection + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e] + + def test_a_component_kind_with_no_relations_is_untouched(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "xe-0/0/0", "type": "100gbase-x-qsfp28"}] + + assert device_types._resolve_relations(BY_YAML_KEY["interfaces"], items, {"slug": "juniper"}) == items + + def test_an_unknown_manufacturer_drops_the_bay_rather_than_sending_names(self, make_device_types): + """Without a scope the names cannot become ids, and NetBox rejects raw names.""" + device_types, console = make_device_types() + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [] + assert any("FPC 0" in line for line in console.lines) + + def test_an_unknown_manufacturer_still_passes_through_a_bay_with_no_restriction(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == items + + def test_an_older_server_strips_the_field_even_without_a_manufacturer(self, make_device_types): + """The manufacturer question must not decide whether an unsupported field is sent.""" + device_types, _ = make_device_types(module_bay_types_supported=False) + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [{"name": "FPC 0"}] + + +class TestComponentCreateWiring: + """Through create_components(), so the resolution is proved to be wired in, not just present.""" + + def test_a_created_bay_carries_resolved_ids(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert [p["name"] for p in posted] == ["FPC 0"] + assert posted[0]["module_bay_types"] == [created_id(server, "mx304-lmic")] + + def test_an_older_server_creates_the_bay_without_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert posted and "module_bay_types" not in posted[0] + + def test_an_unresolvable_bay_is_never_posted(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + with device_types.collect_component_errors() as collected: + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + assert not server.sent("POST", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e] + + +class TestComponentUpdatePayload: + """An existing bay whose restriction changed is patched with ids, not names.""" + + @staticmethod + def _change(name, new_value): + return ComponentChange( + component_type="module-bays", + component_name=name, + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange(property_name="module_bay_types", old_value=[], new_value=new_value)], + ) + + def test_a_changed_restriction_is_patched_as_ids(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + device_types._apply_updates_for_type( + "module-bays", [self._change("FPC 0", ["MX304-LMIC"])], {"manufacturer": {"slug": "juniper"}}, 3, "device" + ) + + assert server.sent("PATCH", "module_bay_templates") == [ + {"id": 55, "module_bay_types": [created_id(server, "mx304-lmic")]} + ] + + def test_a_rejected_patch_is_collected_rather_than_raised(self, make_device_types, server): + """The bay resolves; NetBox refuses the write. That must land in the entity's report.""" + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + server.errors["module_bay_templates"] = 400 + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["MX304-LMIC"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert [e for e in collected if "55" in e] + + def test_an_unresolvable_restriction_is_logged_and_nothing_is_patched(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["NO-SUCH-CLASS"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert not server.sent("PATCH", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + + +class TestCatalogWiring: + """The catalog is built once per run, from the library checkout the run is using.""" + + def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): + device_types, _ = make_device_types() + + catalog = device_types.module_bay_types + + assert isinstance(catalog, ModuleBayTypeCatalog) + assert device_types.module_bay_types is catalog + assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py new file mode 100644 index 00000000..bdf1581d --- /dev/null +++ b/tests/test_module_bay_types.py @@ -0,0 +1,250 @@ +"""Tests for module bay type reference resolution. + +Driven through the public interface (``ids_for`` / ``identities_for``) against catalog files +the tests write themselves and a real ``pynetbox`` client talking HTTP to a local server. +Nothing here is mocked: the client serialises, filters and paginates for real, which is +where the semantics that matter actually live. +""" + +import pytest + +from core.module_bay_types import ModuleBayTypeCatalog, ModuleBayTypeError +from helpers import FakeNetBox, write_module_bay_type + + +class Handle: + """Capture what the catalog logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +DEFAULT_MANUFACTURERS = ( + {"id": 1, "name": "Juniper", "slug": "juniper"}, + {"id": 2, "name": "Cisco", "slug": "cisco"}, + {"id": 3, "name": "Nokia", "slug": "nokia"}, +) + + +@pytest.fixture +def library(tmp_path): + """Write the catalog the resolution tests resolve against, and return its root. + + QSFP-DD is defined twice on purpose, by Juniper and by Generic, so the owner-scope + rule has two candidates to choose between. + """ + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot compatibility") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot compatibility") + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "Juniper MX304 QSFP-DD cage") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "QSFP-DD pluggable transceiver form factor") + return root + + +@pytest.fixture +def catalog(library): + """Build a catalog wired to a real pynetbox client and a local NetBox-shaped server.""" + servers = [] + + def _make(module_bay_types=(), manufacturers=None, root=None): + server = FakeNetBox( + manufacturers=DEFAULT_MANUFACTURERS if manufacturers is None else manufacturers, + module_bay_types=module_bay_types, + ) + servers.append(server) + return ModuleBayTypeCatalog(server.api(), str(root or library), Handle()), server + + yield _make + for server in servers: + server.close() + + +@pytest.mark.real_http +class TestResolution: + """Names resolve to ids, in the owning manufacturer's scope and then in Generic.""" + + def test_resolves_in_owner_manufacturer_scope(self, catalog): + cat, server = catalog() + ids = cat.ids_for("juniper", ["MX304-RE"]) + assert len(ids) == 1 + created = server.sent("POST", "module_bay_types") + assert created == [ + { + "name": "MX304-RE", + "slug": "mx304-re", + "manufacturer": 1, + "description": "Juniper MX304 routing-engine slot compatibility", + } + ] + + def test_falls_back_to_generic_for_another_manufacturer(self, catalog): + """A Cisco optic reaches the Generic form factor, and Generic is created on demand.""" + cat, server = catalog() + assert len(cat.ids_for("cisco", ["QSFP-DD"])) == 1 + made = server.sent("POST", "manufacturers") + assert made == [{"name": "Generic", "slug": "generic"}] + + def test_owner_scope_wins_over_generic(self, catalog): + """Juniper and Generic both define QSFP-DD, and a Juniper reference means Juniper's.""" + cat, server = catalog() + cat.ids_for("juniper", ["QSFP-DD"]) + created = server.sent("POST", "module_bay_types") + assert [p["manufacturer"] for p in created] == [1] + assert not server.sent("POST", "manufacturers") + + def test_existing_object_is_reused_not_recreated(self, catalog): + cat, server = catalog( + module_bay_types=[{"id": 77, "name": "MX304-RE", "slug": "mx304-re", "manufacturer": {"id": 1}}] + ) + assert cat.ids_for("juniper", ["MX304-RE"]) == [77] + assert not server.sent("POST", "module_bay_types") + + def test_repeated_resolution_is_cached(self, catalog): + cat, server = catalog() + first = cat.ids_for("juniper", ["MX304-RE"]) + before = len(server.requests) + + assert cat.ids_for("juniper", ["MX304-RE"]) == first + assert len(first) == 1, "an empty result would make the request count meaningless" + assert len(server.requests) == before + + def test_order_is_not_meaningful(self, catalog): + cat, _ = catalog() + a = cat.ids_for("juniper", ["MX304-RE", "MX304-LMIC"]) + b = cat.ids_for("juniper", ["MX304-LMIC", "MX304-RE"]) + assert len(a) == 2 + assert sorted(a) == sorted(b) + + +@pytest.mark.real_http +class TestRefusals: + """An unresolved name and a conflicting identity are reported, never papered over.""" + + def test_unresolved_name_raises_rather_than_dropping(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["NO-SUCH-CLASS"]) + assert "NO-SUCH-CLASS" in str(exc.value) + + def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): + """The catalog says mx304-re; NetBox holds mx304_re. Never silently rename.""" + cat, server = catalog( + module_bay_types=[{"id": 88, "name": "MX304-RE", "slug": "mx304_re", "manufacturer": {"id": 1}}] + ) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["MX304-RE"]) + assert "mx304_re" in str(exc.value) and "mx304-re" in str(exc.value) + assert not server.sent("POST", "module_bay_types") + + +@pytest.mark.real_http +class TestCatalogReading: + """The catalog is read from real files on disk, including the shapes that are rejected.""" + + def test_entry_without_a_description_is_created_without_one(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + cat, server = catalog(root=tmp_path) + cat.ids_for("generic", ["SFP"]) + created = server.sent("POST", "module_bay_types") + generic = next(m for m in server.collection("manufacturers") if m["slug"] == "generic") + assert created == [{"name": "SFP", "slug": "sfp", "manufacturer": generic["id"]}] + + def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + (tmp_path / "module-bay-types" / "Generic" / "README.md").write_text("not a catalog entry\n") + (tmp_path / "module-bay-types" / "Generic" / "blank.yaml").write_text("# only a comment\n") + cat, _ = catalog(root=tmp_path) + assert len(cat.ids_for("generic", ["SFP"])) == 1 + + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): + """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayTypeError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) + + def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog): + """A parser error escaping the boundary ends the run before it reports anything.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayTypeError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "sfp.yaml" in str(exc.value) + + def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catalog, monkeypatch): + """The load caches only on success, so a bad catalog re-walked the tree every time.""" + import core.module_bay_types as module + + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + + walks = [] + real_walk = module.os.walk + monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) + + for _ in range(3): + with pytest.raises(ModuleBayTypeError): + cat.identities_for("generic", ["SFP"]) + assert len(walks) == 1 + + def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): + """Two files claiming the same (manufacturer, name) would make a reference ambiguous.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("generic", ["SFP"]) + assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) + + +@pytest.mark.real_http +class TestMalformedReferences: + """A reference list that is not a list of names is refused, never read as empty.""" + + def test_a_bare_key_is_refused_rather_than_clearing(self, catalog): + """`module_bay_types:` with no value parses as None; treating it as [] would clear.""" + cat, server = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", None) + assert "list of names" in str(exc.value) + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_a_non_string_entry_is_refused(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [{"name": "MX304-RE"}]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", ["MX304-RE", 7]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [" "]) + + def test_an_empty_list_is_allowed_and_holds_no_classes(self, catalog): + """`module_bay_types: []` is an explicit instruction, not a malformed value.""" + cat, server = catalog() + assert cat.ids_for("juniper", []) == [] + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_duplicate_names_collapse(self, catalog): + """The relationship is a set, so a repeated name must not produce a repeated id.""" + cat, _ = catalog() + once = cat.ids_for("juniper", ["MX304-RE"]) + + assert len(once) == 1 + assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 31245d0b..a358c6ca 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -2190,7 +2190,7 @@ class TestExportDiffVendorFilterOverRealHTTP: """Same run against a local HTTP server, so the filter is asserted as it is serialized on the wire.""" @staticmethod - def _serve(): + def _serve(netbox_version="4.7.0"): """Serve empty GraphQL pages and record every decoded request body.""" import json import threading @@ -2199,16 +2199,22 @@ def _serve(): bodies = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers["Content-Length"]) - bodies.append(json.loads(self.rfile.read(length))) - payload = b'{"data": {}}' + def _reply(self, payload): self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) + def do_GET(self): + """Answer the version probe the export runs before its first query.""" + self._reply(json.dumps({"netbox-version": netbox_version}).encode()) + + def do_POST(self): + length = int(self.headers["Content-Length"]) + bodies.append(json.loads(self.rfile.read(length))) + self._reply(b'{"data": {}}') + def log_message(self, *args): """Silence the default stderr access log.""" @@ -2229,3 +2235,33 @@ def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypa assert set(filters) == set(_LIST_FIELDS) for field, variables in filters.items(): assert variables["manufacturer_slugs"] == ["cisco", "juniper"], field + + def _queries_for_version(self, nb_dt_import, monkeypatch, tmp_path, library_root, version): + """Run one whole export against a server reporting *version* and return its queries.""" + url, server, bodies = self._serve(version) + monkeypatch.setenv("NETBOX_URL", url) + try: + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, library_root, "Juniper") + finally: + server.shutdown() + server.server_close() + return [payload["query"] for payload in bodies] + + def test_a_47_server_is_asked_for_the_module_bay_type_relation( + self, nb_dt_import, monkeypatch, tmp_path, _real_library_root + ): + """Not selecting it exports every bay without its restriction, silently. + + Asserted on the module-type query, which this run always issues; the component + queries only run once there is something to export. + """ + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.7.0") + + module_types = [q for q in queries if "module_type_list(" in q] + assert module_types and all("module_bay_types" in q for q in module_types) + + def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + """Selecting a field the schema lacks fails the whole query.""" + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.6.9") + + assert not [q for q in queries if "module_bay_types" in q] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 9250234d..3d5edf70 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -677,3 +677,69 @@ def test_components_sorted_by_name(self): result = serialize_device_type(record, components) names = [i["name"] for i in result["interfaces"]] assert names == sorted(names) + + +class TestRelationSerialization: + """A module bay's restriction has to survive the trip back out to YAML.""" + + @staticmethod + def _bay(name, module_bay_types=None, **extra): + """Build a module bay template as the GraphQL query returns it.""" + return _dotdict( + name=name, + position=None, + label="", + description="", + module_bay_types=module_bay_types, + **extra, + ) + + def test_a_bay_exports_the_names_of_its_classes(self): + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = self._bay("FPC 0", [_dotdict(id=9, name="MX304-LMIC"), _dotdict(id=8, name="QSFP-DD")]) + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC", "QSFP-DD"]}] + + def test_a_bay_with_no_classes_writes_no_key(self): + """An empty list here would add the key to every bay in the library. + + See test_an_empty_relation_does_not_make_every_definition_differ for the effect + that has on the export diff. + """ + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + + result = serialize_module_type(record, {1: {"module_bay_templates": [self._bay("FPC 0", [])]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_server_that_never_returned_the_field_omits_the_key(self): + """Below 4.7 the relation is not selected, and absent must not become empty.""" + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = _dotdict(name="FPC 0", position=None, label="", description="") + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_module_type_exports_the_classes_it_belongs_to(self): + record = _dotdict( + id=5, + model="JNP304-RE", + manufacturer=_make_mfr(name="Juniper", slug="juniper"), + part_number=None, + module_bay_types=[_dotdict(id=9, name="MX304-RE")], + ) + + assert serialize_module_type(record, {})["module_bay_types"] == ["MX304-RE"] + + def test_a_device_type_bay_exports_its_classes_too(self): + record = _dotdict( + id=2, model="MX304", slug="mx304", manufacturer=_make_mfr(), u_height=None, is_full_depth=None + ) + bay = self._bay("RE0", [_dotdict(id=9, name="MX304-RE")]) + + result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 3f7b3315..eb535191 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -112,7 +112,7 @@ def _mark_cache_ready(device_types): def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): # Mock api call - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.url == "http://mock-netbox" @@ -128,7 +128,7 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc remove_unmanaged_types=True, verify_images=True, ) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" netbox = NetBox(config, mock_handle) @@ -138,24 +138,16 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): - # Test 5.0 - mock_pynetbox.api.return_value.version = "5.0" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters - - # Test 4.0 - mock_pynetbox.api.return_value.version = "4.0" - nb = NetBox(mock_settings, mock_handle) - assert not nb.new_filters - - # Test 4.1 - mock_pynetbox.api.return_value.version = "4.1" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters + """Every supported release uses the new filter names; 4.7 adds module bay types.""" + for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): + mock_pynetbox.api.return_value.version = version + nb = NetBox(mock_settings, mock_handle) + assert nb.new_filters, version + assert nb.module_bay_types is module_bay_types, version def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] nb = NetBox(mock_settings, mock_handle) @@ -168,7 +160,7 @@ def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): def test_create_manufacturers_no_new_is_verbose_only(mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -233,7 +225,7 @@ def test_create_generic_counts_the_created_list_at_the_caller(mock_pynetbox, mak def test_redundant_image_upload(mock_settings, mock_pynetbox, mock_handle): # Setup # Ensure modules check doesn't fail - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = MagicMock() @@ -611,7 +603,7 @@ class TestNetBoxConnectApi: def test_ssl_ignore_sets_verify_false(self, mock_pynetbox, mock_handle, make_config): mock_settings = make_config(ignore_ssl_errors=True) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.netbox.http_session.verify is False @@ -622,7 +614,7 @@ class TestCreateManufacturersError: def test_request_error_logged(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" # Make pynetbox.RequestError in the module under test be the real exception class mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -930,7 +922,7 @@ def test_creates_new_device_type_with_components( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) @@ -1118,14 +1110,14 @@ class TestCreateModuleTypes: """Tests for TestCreateModuleTypes.""" def test_empty_module_types_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) # Should not raise and should not call create nb.create_module_types([]) nb.netbox.dcim.module_types.create.assert_not_called() def test_creates_new_module_type(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1250,7 +1242,7 @@ class TestCreateModuleTypesBody: """Tests for TestCreateModuleTypesBody.""" def test_cached_module_type_skips_creation(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1285,7 +1277,7 @@ def test_create_module_type_request_error_logged( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1310,7 +1302,7 @@ def test_create_module_type_request_error_logged( def test_creates_module_type_with_components( self, mock_settings, mock_pynetbox, mock_graphql_requests, graphql_client, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1606,7 +1598,7 @@ class TestCreateManufacturersSuccessLog: def test_verbose_log_per_created_manufacturer(self, mock_settings, mock_pynetbox, mock_handle): """verbose_log should be called for each created manufacturer.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_m = MagicMock() @@ -2333,7 +2325,7 @@ def test_create_failure_reaches_the_end_of_run_failure_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b"{\"manufacturer\":[\"Related object not found using the provided attributes: {'slug': 'ribbon'}\"]}" ) @@ -2574,7 +2566,7 @@ class TestFilterActionableModuleTypesEdge: def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """Empty module_types list returns [], {} immediately.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) result, images, _ = nb.filter_actionable_module_types([], {}, only_new=False) assert result == [] @@ -2582,7 +2574,7 @@ def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mo def test_only_new_delegates_to_filter_new(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """only_new=True returns only genuinely new module types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() all_mts = {"cisco": {"LC": existing_mt}} @@ -2599,7 +2591,7 @@ def test_new_module_type_added_to_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """Module type not in all_module_types is added to actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2622,7 +2614,7 @@ def test_existing_module_with_new_image_is_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """Existing module type with an image not yet in NetBox is actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2673,7 +2665,7 @@ def test_existing_module_with_changed_property_is_actionable( """Existing module type with a changed scalar property (e.g. part_number) is actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2726,7 +2718,7 @@ def test_existing_module_with_missing_image_and_property_change_logs_both( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2782,7 +2774,7 @@ def test_existing_module_with_only_missing_image_is_actionable_but_not_logged( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2827,7 +2819,7 @@ def test_existing_module_with_unchanged_property_is_not_actionable( """Existing module type whose properties all match NetBox is not actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2868,7 +2860,7 @@ class TestCreateModuleTypesEdge: def test_existing_module_type_verbose_logged(self, mock_settings, mock_pynetbox, mock_handle): """When a module type already exists, verbose_log is called with 'Cached'.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() @@ -2894,7 +2886,7 @@ def test_only_new_skips_existing_module_component_creation( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """only_new=True + existing module → skip component creation.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("interface_templates", "module", 5, {}) @@ -2923,7 +2915,7 @@ def test_creates_module_type_with_power_outlets_console_server_ports_front_ports self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """power-outlets, console-server-ports, front-ports branches in create_module_types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) for endpoint_name in ("power_outlet_templates", "console_server_port_templates", "front_port_templates"): @@ -2955,7 +2947,7 @@ def test_a_new_module_type_gets_its_module_bays( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The DTL module-type schema allows module-bays, so creation must not skip them.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("module_bay_templates", "module", 5, {}) @@ -2981,7 +2973,7 @@ def test_existing_module_type_property_update_calls_api(self, mock_settings, moc """Existing module type with changed part_number calls module_types.update and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3017,7 +3009,7 @@ def test_existing_module_type_property_unchanged_no_api_call(self, mock_settings """Existing module type with matching part_number does not call module_types.update.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3048,7 +3040,7 @@ def test_existing_module_type_only_new_skips_property_update(self, mock_settings """only_new=True skips property update even when part_number differs.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3082,7 +3074,7 @@ def test_existing_module_type_component_update_calls_update_components( """Existing module type with changed component property calls update_components and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3146,7 +3138,7 @@ def test_existing_module_type_property_and_component_update_increments_once( """Both property and component change → module_updated incremented only once.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3214,7 +3206,7 @@ def test_existing_module_type_removal_only_no_counter_increment( """COMPONENT_REMOVED-only changes call update_components but do NOT increment module_updated.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3275,7 +3267,7 @@ def test_property_update_plus_removal_only_remove_false_counts_as_updated( """Properties changed + removal-only diff with remove_components=False → module_updated incremented.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.update_components = MagicMock() @@ -3358,7 +3350,7 @@ def test_existing_image_is_skipped( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """If the image name is already in module_type_existing_images, upload is skipped.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -3388,7 +3380,7 @@ def test_new_image_is_uploaded_and_tracked( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """When image is not yet in existing_images, upload_image_attachment is called.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -4218,7 +4210,7 @@ class TestCreateModuleTypesCornerCases: def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_mt = MagicMock() @@ -4252,7 +4244,7 @@ def tracking_iter(): def test_all_module_types_fetched_when_none(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """all_module_types is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4277,7 +4269,7 @@ def test_module_type_existing_images_fetched_when_none( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """module_type_existing_images is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4421,7 +4413,7 @@ class TestGetExistingRackTypes: def test_delegates_to_graphql(self, mock_settings, mock_pynetbox, graphql_client, mock_handle): """get_existing_rack_types() returns whatever graphql.get_rack_types() returns.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.graphql = graphql_client expected = {"apc": {"AR1300": MagicMock()}} @@ -4442,7 +4434,7 @@ class TestCreateRackTypes: """Tests for NetBox.create_rack_types().""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_empty_list_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): @@ -4510,7 +4502,7 @@ def test_existing_rack_type_fields_differ_calls_update(self, mock_settings, mock def test_new_rack_type_calls_create(self, mock_settings, mock_pynetbox, mock_handle): """Non-existing rack type: create called, counter incremented, added to cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 99 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4532,7 +4524,7 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet """RequestError during create is logged; processing continues.""" import pynetbox - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4553,7 +4545,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet import pynetbox from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4573,7 +4565,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, mock_handle): """When all_rack_types=None, get_existing_rack_types() is called to populate the cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) nb.get_existing_rack_types = MagicMock(return_value={}) rack_type = { @@ -4587,7 +4579,7 @@ def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When a progress wrapper is provided, it is used as the iterator.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 1 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4615,16 +4607,15 @@ class TestVerifyCompatibility: @pytest.mark.parametrize( "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", [ - ("3.1", False, False, False, False), - ("3.2", True, False, False, False), - ("4.0", True, False, False, False), - ("4.1", True, True, True, False), + ("4.3", True, True, True, False), ("4.4", True, True, True, False), ("4.5", True, True, True, True), ("4.6", True, True, True, True), + ("4.7", True, True, True, True), + ("5.0", True, True, True, True), # Version strings with non-numeric suffixes ("4.5-beta", True, True, True, True), - ("4.1.0", True, True, True, False), + ("4.3.0", True, True, True, False), ], ) def test_version_thresholds( @@ -4646,16 +4637,19 @@ def test_version_thresholds( assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): - """Version string with only major component (e.g. '4') does not crash.""" - mock_pynetbox.api.return_value.version = "4" + """A version string with only a major component (e.g. '5') does not crash.""" + mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is False # 4.0 → no new filters + assert nb.new_filters is True - def test_version_42_enables_new_filters_not_m2m(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "4.2" + def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( + self, mock_settings, mock_pynetbox, mock_handle + ): + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.new_filters is True assert nb.m2m_front_ports is False + assert nb.module_bay_types is False # ============================================================ @@ -4855,7 +4849,7 @@ class TestLogModuleTypeChanges: def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, mock_handle): """A non-empty changed_property_log triggers verbose logging.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4868,7 +4862,7 @@ def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, def test_empty_log_emits_nothing(self, mock_settings, mock_pynetbox, mock_handle): """An empty changed_property_log does not trigger any logging calls.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4887,7 +4881,7 @@ class TestTryUpdateModuleTypeErrors: """Tests for RequestError and retryable-exception handlers in _try_update_module_type.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def _make_module_type_res(self): @@ -4901,7 +4895,7 @@ def test_request_error_returns_false_and_logs(self, mock_settings, mock_pynetbox """pynetbox.RequestError during update causes (False, False) return and log.""" import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) @@ -4926,7 +4920,7 @@ def test_retryable_exception_returns_false_and_logs(self, mock_settings, mock_py import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) mock_handle.log.reset_mock() @@ -4969,7 +4963,7 @@ def test_retryable_exception_on_create_returns_false( import requests mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -5777,7 +5771,7 @@ class TestUploadModuleTypeImagesVerify: """Tests for _upload_module_type_images with verify_images=True.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types.upload_image_attachment = MagicMock(return_value=True) nb.verify_images = True @@ -5936,7 +5930,7 @@ def test_load_image_hash_cache_returns_empty_dict_on_bad_json(self, tmp_path): def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_settings, mock_pynetbox, mock_handle): from core.graphql_client import GraphQLError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with patch.object(NetBox, "get_manufacturers", side_effect=GraphQLError("bad query")): with pytest.raises(NetBoxError, match="GraphQL error: bad query"): @@ -5945,7 +5939,7 @@ def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_setti def test_init_raises_typed_error_when_device_types_initialization_fails( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with ( patch.object(NetBox, "get_manufacturers", return_value=[]), @@ -6022,7 +6016,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.manufacturers.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6032,7 +6026,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock assert any("Connection error creating manufacturers" in str(c) for c in mock_handle.log.call_args_list) def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") @@ -6049,7 +6043,7 @@ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pyn from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") resolution = SimpleNamespace( @@ -6072,7 +6066,7 @@ def test_try_resolve_update_logs_auto_resolve_failure(self, mock_settings, mock_ from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6105,7 +6099,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( from core.update_failure_resolver import FailureKind mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6132,7 +6126,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( def test_log_device_type_change_outcome_partial_success_mentions_property_failure( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6151,7 +6145,7 @@ def test_log_device_type_change_outcome_partial_success_mentions_property_failur def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6168,7 +6162,7 @@ def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( assert any("Device Type Cached" in str(c) for c in mock_handle.verbose_log.call_args_list) def test_filter_images_for_upload_keeps_changed_image(self, mock_settings, mock_pynetbox, tmp_path, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True @@ -6194,7 +6188,7 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( from core.change_detector import PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.update.side_effect = requests.exceptions.ConnectionError("offline") nb._log_device_type_change_outcome = MagicMock() @@ -6217,7 +6211,7 @@ def test_create_new_device_type_logs_retryable_error(self, mock_settings, mock_p import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6238,7 +6232,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( ): from core.change_detector import ChangeType, ComponentChange, PropertyChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) changes = [ ComponentChange("interfaces", "xe-0", ChangeType.COMPONENT_ADDED), @@ -6261,7 +6255,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True details = {7: {"linecard.front": {"att_id": 5, "url": "/media/linecard.front.jpg"}}} @@ -6273,7 +6267,7 @@ def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( assert nb._module_image_details == details def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(spec=["id", "manufacturer", "model"]) module_type_res.id = 1 @@ -6290,7 +6284,7 @@ def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, def test_filter_actionable_module_types_marks_verify_images_module_actionable( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True _mark_cache_ready(nb.device_types) @@ -6314,7 +6308,7 @@ class TestAdditionalModuleTypeCoverage: def test_apply_module_type_component_updates_records_failed_no_actionable_changes( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6334,7 +6328,7 @@ def test_apply_module_type_component_updates_marks_partial_on_partial_component_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6362,7 +6356,7 @@ def test_apply_module_type_component_updates_marks_partial_when_properties_only_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6383,7 +6377,7 @@ def test_apply_module_type_component_updates_records_failed_when_no_changes_appl ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6510,7 +6504,7 @@ def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock from core.graphql_client import DotDict mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6530,7 +6524,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6547,7 +6541,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock def test_upload_module_type_images_discards_missing_attachment_before_failed_upload( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=False) @@ -6575,7 +6569,7 @@ def test_upload_module_type_images_discards_missing_attachment_before_failed_upl def test_upload_module_type_images_skips_changed_image_when_delete_fails( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=True) @@ -6825,7 +6819,7 @@ def test_filter_actionable_module_types_skips_unchanged_existing_module( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6874,7 +6868,7 @@ def test_filter_actionable_module_types_includes_module_with_missing_component( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6922,7 +6916,7 @@ def test_missing_netbox_field_is_not_treated_as_change( ): """When existing module lacks an attribute, it's skipped — no false positive change.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6979,7 +6973,7 @@ def test_remove_components_is_called_when_flag_set( ): """When remove_components=True and there are component changes, remove_components is called.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7037,7 +7031,7 @@ def test_component_reconciliation_continues_when_scalar_patch_fails( skipped entirely. """ mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7106,7 +7100,7 @@ def _netbox(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_rack_type_create_failure_is_reported(self, mock_settings, mock_pynetbox, mock_handle): @@ -7190,7 +7184,7 @@ def test_device_type_create_failure_is_not_called_an_update_failure( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b'{"manufacturer":["Related object not found using the provided attributes: slug ribbon"]}' ) @@ -7223,7 +7217,7 @@ def test_module_type_create_failure_is_not_called_an_update_failure( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.module_types.create.side_effect = _request_error( b'{"model":["This field may not be blank."]}' ) @@ -7247,7 +7241,7 @@ class TestSkippedComponentReasonReachesTheReport: def _dt(self, mock_pynetbox, make_device_types, parent_id=1): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) return dt @@ -7260,7 +7254,7 @@ def test_unresolvable_power_port_reaches_the_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) inlet = MagicMock() @@ -7351,7 +7345,7 @@ def test_component_update_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) existing_iface = MagicMock() @@ -7410,7 +7404,7 @@ def test_component_removal_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) stale = MagicMock() @@ -7439,7 +7433,7 @@ def test_failed_component_create_reports_the_netbox_message( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 6119, {}) @@ -7484,7 +7478,7 @@ def _nb(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_new_device_type_with_failing_component_is_partial( @@ -7495,7 +7489,7 @@ def test_new_device_type_with_failing_component_is_partial( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 900, {}) mock_nb_api.dcim.interface_templates.create.side_effect = _request_error( @@ -7532,7 +7526,7 @@ def test_new_device_type_with_failing_component_is_partial( def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, graphql_client, make_device_types): """Errors buffered for one entity must not surface in the next entity's reason.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt._log_component_error("stale error from an earlier entity") @@ -7547,7 +7541,7 @@ def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, grap def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_client, make_device_types): """The buffer is empty after a scope closes, so the next scope starts clean.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) with dt.collect_component_errors() as first: @@ -7557,3 +7551,16 @@ def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_cl with dt.collect_component_errors() as second: pass assert second == [] + + +def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_settings, mock_pynetbox, mock_handle): + """A GraphQL schema error is a poor way to learn the server is too old.""" + from core.netbox_api import NetBoxError + + for version in ("4.1", "4.2", "3.5"): + mock_pynetbox.api.return_value.version = version + with pytest.raises(NetBoxError) as exc: + NetBox(mock_settings, mock_handle) + message = str(exc.value) + assert "4.3" in message, f"{version}: the message must name the minimum" + assert version in message, f"{version}: the message must name what was found" diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py new file mode 100644 index 00000000..7dac8f0f --- /dev/null +++ b/tests/test_relation_scope.py @@ -0,0 +1,188 @@ +"""A relation assigned from the wrong manufacturer scope must be detected as a change. + +The catalog resolves a reference name in the owning manufacturer's scope first and only +then in Generic. If NetBox already holds the Generic object where the owner has one of +its own, comparing names alone reports equality and the wrong object survives. +""" + +import pytest + +from core.change_detector import ChangeDetector +from core.module_bay_types import ModuleBayTypeCatalog + + +class Handle: + """Capture what the detector logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +class Related: + """A module bay type as NetBox returns it inside a relation.""" + + def __init__(self, name, slug, manufacturer_slug): + """Store the identity fields the comparison needs.""" + self.name = name + self.slug = slug + self.manufacturer = type("M", (), {"slug": manufacturer_slug})() + + +class NetBoxBay: + """A module bay template as the cache hands it to the detector.""" + + def __init__(self, name, module_bay_types): + """Store the bay name and its assigned classes.""" + self.name = name + self.module_bay_types = module_bay_types + + +@pytest.fixture +def two_scope_catalog(tmp_path): + """Build a catalog where the same class name exists under Acme and under Generic.""" + for manufacturer, slug in (("Acme", "acme-x"), ("Generic", "generic-x")): + directory = tmp_path / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{slug}.yaml").write_text( + f"name: X\nslug: {slug}\nmanufacturer: {manufacturer}\n", encoding="utf-8" + ) + return str(tmp_path) + + +def _detector(catalog, handle=None): + """Build a detector whose device_types exposes the catalog, as the real one does.""" + device_types = type("DeviceTypes", (), {"module_bay_types": catalog, "module_bay_types_supported": True})() + return ChangeDetector(device_types, handle or Handle()) + + +def test_wrong_scope_assignment_is_detected(two_scope_catalog): + """Acme owns X, but NetBox assigned Generic's X. The names match; the objects do not.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert [c.property_name for c in changes] == ["module_bay_types"], ( + "a bay owned by Acme holding Generic's X must be corrected to Acme's X" + ) + + +def test_right_scope_assignment_is_left_alone(two_scope_catalog): + """The same bay already holding Acme's X is correct and must not be rewritten.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert changes == [] + + +def test_generic_fallback_is_correct_when_the_owner_has_no_such_class(two_scope_catalog): + """A Nokia bay resolves X to Generic's X, so holding Generic's X is right.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="nokia" + ) + assert changes == [] + + +def _changes(detector, yaml_comp, netbox_comp, manufacturer="acme"): + """Run the real property comparison for the relation and return what it found.""" + return detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer=manufacturer + ) + + +class TestUnmanagedRelations: + """A relation is only rewritten when both sides say something about it.""" + + def test_an_omitted_key_leaves_the_relation_alone(self, two_scope_catalog): + """A definition that never mentions the relation is not asking for it to be cleared.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0"}, netbox_comp) == [] + + def test_a_bare_key_leaves_the_relation_alone_but_says_so(self, two_scope_catalog): + """Parses as None, so nothing is cleared; a typo must still not be invisible.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": None}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): + """Reading an absent field as empty would report a change on every run.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = type("Bay", (), {"name": "Slot 0"})() + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_an_unresolvable_reference_reaches_the_write_path(self, two_scope_catalog): + """Reporting "no change" leaves the bay unrestricted in silence. + + The write path is the only thing that logs and records an unresolvable name, and it + only ever sees a component the detector reported as changed. + """ + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["NO-SUCH-CLASS"]}, netbox_comp) + + assert [(c.property_name, c.new_value) for c in changes] == [("module_bay_types", ["NO-SUCH-CLASS"])] + + +class TestNameComparisonFallback: + """Where an identity cannot be had, comparing names is still better than doing nothing.""" + + def test_names_are_compared_when_netbox_returned_no_slug(self, two_scope_catalog): + """A read path returning only id and name cannot answer the scope question.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", None, None)]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.property_name, c.old_value, c.new_value) for c in changes] == [("module_bay_types", ["Y"], ["X"])] + + def test_matching_names_are_left_alone_when_netbox_returned_no_slug(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", None, None)]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_names_are_compared_without_a_catalog(self): + """A caller that has not wired a catalog still gets the name comparison.""" + detector = _detector(None) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", "generic-y", "generic")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["Y"], ["X"])] + + def test_an_empty_list_clears_the_relation(self, two_scope_catalog): + """`module_bay_types: []` is an explicit instruction to hold no classes.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": []}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["X"], [])] diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index dd8a46bc..e85d411e 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -82,6 +82,25 @@ def test_integration_collection_reads_credentials_from_a_local_env_file(tmp_path assert "1 passed" in result.stdout, result.stdout + result.stderr +def test_the_fake_netbox_releases_its_listening_socket(): + """shutdown() only stops the serve loop. One server per test leaks a descriptor each.""" + import socket + + from helpers import FakeNetBox + + server = FakeNetBox(manufacturers=[{"id": 1, "name": "Juniper", "slug": "juniper"}]) + port = server._server.server_port + server.close() + + probe = socket.socket() + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: # pragma: no cover - only reached when close() leaks + raise AssertionError(f"FakeNetBox.close() left port {port} bound: {exc}") from exc + finally: + probe.close() + + def test_no_test_function_contains_an_orphaned_docstring(): """A dropped ``def`` header silently merges two tests into one. From a0ee29b5c7751bcb0ad9a0b4ac882e49ea920fc1 Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:07:07 +0200 Subject: [PATCH 04/19] feat(export): write the NetBox 4.5 port-mappings stanza (#134) Closes #78. NetBox 4.5 replaced FrontPortTemplate's rear_port foreign key and rear_port_position integer with a through table, so one front port can occupy several positions across several rear ports. The export wrote only the first mapping and warned about the rest, and it wrote them inline on the front-port entry. The device-type library has since taken the same model: a front-port entry carries no rear port at all, the linkage lives in a top-level port-mappings stanza, and additionalProperties is false, so the old shape no longer validates. The export now writes every mapping into that stanza with all four fields. A server below 4.5 answers with the rear_port and rear_port_position scalars instead; those describe one mapping, so they are written as one entry rather than dropped, because the output format follows the library schema and not the server release. The library also made positions required on front and rear ports, and writes it even when it is 1, so it is no longer omitted at its default and the front port row selects it. positions arrived with the mapping model, so the pre-4.5 tiers of the front-port query fallback must not ask for it: leaving it in made every tier fail the same way and took the whole front-port preload down on a server that would have answered the older shape. Verified the output against the real schema/devicetype.json from the library, and round-tripped it back through normalize_port_mappings, which reads the stanza and rebuilds the same per-port mappings. --- core/component_registry.py | 3 +- core/graphql_client.py | 5 +- core/nb_serializer.py | 81 ++++---- tests/test_component_registry.py | 1 + tests/test_graphql_client.py | 18 ++ tests/test_nb_serializer.py | 314 ++++++++++++------------------- 6 files changed, 198 insertions(+), 224 deletions(-) diff --git a/core/component_registry.py b/core/component_registry.py index a1fc18e7..fe414278 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -119,7 +119,8 @@ def create_label(self, parent_type): yaml_key="front-ports", endpoint="front_port_templates", label="Front Port", - fields=("name", "type", "label", "description", "color"), + # positions arrived with the 4.5 mapping model; the query drops it on older servers. + fields=("name", "type", "label", "description", "color", "positions"), graphql_extra=("mappings { id front_port_position rear_port_position rear_port { id name } }",), compare_extra=("_mappings",), link=LINK_REAR_PORTS, diff --git a/core/graphql_client.py b/core/graphql_client.py index 4a8fe786..88cd53e3 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -744,7 +744,7 @@ def get_module_type_image_details(self): def _front_port_field_variants(fields): """Yield successive field-list tiers for the front_port_templates fallback. - Tier 1: mappings block (NetBox 4.5+) + Tier 1: mappings block and positions (NetBox 4.5+) Tier 2: rear_port_position scalar (<4.5) Tier 3: neither (field removed entirely) """ @@ -753,7 +753,8 @@ def _front_port_field_variants(fields): for f in fields: if "mappings" in f: fallback.extend(["rear_port_position", "rear_port { id name }"]) - else: + elif f != "positions": + # positions arrived with the mapping model, so no pre-4.5 tier may ask for it. fallback.append(f) yield fallback stripped = [f for f in fallback if f != "rear_port_position" and "rear_port" not in f] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 5ba5b2ca..5e899758 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -4,7 +4,6 @@ comparison against existing repo YAML files. """ -import warnings from typing import Any, Sequence from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS @@ -26,7 +25,6 @@ "feed_leg": None, "maximum_draw": None, "allocated_draw": None, - "positions": 1, # rear port default; include only when > 1 } # Device type scalar field order for output. @@ -146,37 +144,53 @@ def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: def _serialize_front_port(record: Any) -> dict: - """Serialize a front port template, including rear_port mapping.""" - result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) - mappings = getattr(record, "mappings", None) or [] - if mappings: - if len(mappings) > 1: - port_name = getattr(record, "name", "") - warnings.warn( - f"Front port '{port_name}' has {len(mappings)} mappings; " - "only the first will be exported. " - "Full multi-mapping support requires DTL schema update (see issue #78).", - UserWarning, - stacklevel=4, + """Serialize a front port template's own fields. + + The rear-port linkage is no longer written here: NetBox 4.5 moved it to a through + table and the library schema follows, carrying it in a top-level ``port-mappings`` + stanza built by :func:`_port_mappings`. + """ + return _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + + +def _port_mappings(records: list) -> list: + """Return the ``port-mappings`` stanza for a type's front port templates. + + Every mapping is written, not just the first: one front port may occupy several + positions across rear ports, which is what the through table exists to express. + + A server below 4.5 has no through table and answers with ``rear_port`` and + ``rear_port_position`` scalars instead. Those describe one mapping, so they are + written as one entry rather than dropped. + """ + stanza = [] + for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): + name = getattr(record, "name", None) + for mapping in getattr(record, "mappings", None) or []: + rear_port = getattr(mapping, "rear_port", None) + if not rear_port: + continue + stanza.append( + { + "front_port": name, + "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, + "rear_port": rear_port.name, + "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, + } ) - m = mappings[0] - rear_port = getattr(m, "rear_port", None) - if rear_port: - result["rear_port"] = rear_port.name - rear_pos = getattr(m, "rear_port_position", None) - rear_pos = _coerce_numeric(rear_pos) - if rear_pos is not None and rear_pos > 1: - result["rear_port_position"] = rear_pos - else: - # Legacy: pre-4.5 NetBox returns rear_port / rear_port_position as direct scalar fields - legacy_rp = getattr(record, "rear_port", None) - if legacy_rp: - result["rear_port"] = legacy_rp.name - legacy_pos = getattr(record, "rear_port_position", None) - legacy_pos = _coerce_numeric(legacy_pos) - if legacy_pos is not None and legacy_pos > 1: - result["rear_port_position"] = legacy_pos - return result + if getattr(record, "mappings", None): + continue + legacy = getattr(record, "rear_port", None) + if legacy: + stanza.append( + { + "front_port": name, + "front_port_position": 1, + "rear_port": legacy.name, + "rear_port_position": _coerce_numeric(getattr(record, "rear_port_position", None)) or 1, + } + ) + return stanza def _serialize_component_list(endpoint_name: str, records: list) -> list: @@ -200,6 +214,9 @@ def _add_components(result: dict, type_id: int, components_by_id: dict) -> None: records = type_components.get(component.endpoint, []) if records: result[component.yaml_key] = _serialize_component_list(component.endpoint, records) + mappings = _port_mappings(type_components.get("front_port_templates", [])) + if mappings: + result["port-mappings"] = mappings def serialize_device_type(nb_record: Any, components_by_dt_id: dict) -> dict: diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 7e536f53..f1d05dbe 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -128,6 +128,7 @@ def test_the_query_selects_exactly_these_fields(self): "label", "description", "color", + "positions", _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index b72ddae6..5cfb8004 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -1152,6 +1152,24 @@ def _query_for(supported): assert "module_bay_types" in _query_for(True) assert "module_bay_types" not in _query_for(False) + def test_the_front_port_fallback_drops_every_45_only_field(self, mock_post): + """Positions arrived with the mapping model, so a pre-4.5 tier must not ask for it. + + Leaving it in a fallback tier makes every tier fail the same way, and the whole + front-port preload dies on a server that would answer the older shape. + """ + from core.component_registry import BY_ENDPOINT + from core.graphql_client import NetBoxGraphQLClient + + fields = list(BY_ENDPOINT["front_port_templates"].graphql_fields) + assert "positions" in fields, "guard: the registry is expected to select positions" + + tiers = list(NetBoxGraphQLClient._front_port_field_variants(fields)) + + assert any("positions" in f for f in tiers[0]), "the 4.5 tier keeps it" + for tier in tiers[1:]: + assert not any("positions" in f for f in tier), f"pre-4.5 tier still asks for it: {tier}" + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 3d5edf70..fce437b1 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -441,195 +441,7 @@ def test_rack_type_manufacturer_as_name_string(self): class TestFrontPortSerialization: - """Tests for front port rear_port extraction.""" - - def test_front_port_rear_port_extracted_from_mapping(self): - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] - - def test_front_port_rear_port_position_included_when_gt_1(self): - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=3) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port_position"] == 3 - - def test_front_port_rear_port_position_zero_omitted(self): - """Position 0 is not a valid DTL value and should be omitted.""" - from types import SimpleNamespace - - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=0) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert "rear_port_position" not in result["front-ports"][0] - - def test_front_port_multiple_mappings_warns_and_uses_first(self): - """When a front port has >1 mappings a UserWarning is raised and only the first is used.""" - import warnings - from types import SimpleNamespace - - m1 = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - m2 = SimpleNamespace(rear_port=SimpleNamespace(name="RP2"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[m1, m2]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert len(caught) == 1 - assert issubclass(caught[0].category, UserWarning) - assert "FP1" in str(caught[0].message) - assert "2 mappings" in str(caught[0].message) - assert "issue #78" in str(caught[0].message) - - def test_front_port_legacy_rear_port_scalars(self): - """pre-4.5 NetBox: record has rear_port/rear_port_position as direct attrs (no mappings).""" - from types import SimpleNamespace - - fp = SimpleNamespace( - name="FP1", - type="8p8c", - label="", - description="", - color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=3, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert result["front-ports"][0]["rear_port_position"] == 3 - - def test_front_port_legacy_rear_port_position_1_omitted(self): - """pre-4.5: rear_port_position == 1 should be omitted (same as mappings path).""" - from types import SimpleNamespace - - fp = SimpleNamespace( - name="FP1", - type="8p8c", - label="", - description="", - color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=1, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] + """Front port scalars. The rear-port linkage lives in TestPortMappingsStanza.""" def test_components_sorted_by_name(self): from types import SimpleNamespace @@ -743,3 +555,127 @@ def test_a_device_type_bay_exports_its_classes_too(self): result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] + + +class TestPortMappingsStanza: + """Export writes the NetBox 4.5 port-mappings stanza the DTL schema now requires.""" + + @staticmethod + def _mapping(rear_port, rear_position=1, front_position=1): + from types import SimpleNamespace + + return SimpleNamespace( + rear_port=SimpleNamespace(name=rear_port), + rear_port_position=rear_position, + front_port_position=front_position, + ) + + @staticmethod + def _front_port(name, mappings=None, positions=1, **extra): + from types import SimpleNamespace + + return SimpleNamespace( + name=name, + type="lc-upc", + label="", + description="", + color="", + positions=positions, + mappings=mappings or [], + **extra, + ) + + def _device(self): + return _dotdict( + id=1, + model="X", + slug="acme-x", + manufacturer=_make_mfr(), + u_height=1, + is_full_depth=True, + part_number=None, + airflow=None, + weight=None, + weight_unit=None, + description="", + comments="", + subdevice_role=None, + front_image=None, + rear_image=None, + ) + + def test_a_front_port_carries_positions_and_no_inline_rear_port(self): + """The schema dropped rear_port from front-port entries and made positions required.""" + fp = self._front_port("FP1", [self._mapping("RP1")], positions=1) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "lc-upc", "positions": 1}] + + def test_every_mapping_reaches_the_stanza_not_just_the_first(self): + """The issue: a crossover front port mapped to two rear ports lost the second.""" + fp = self._front_port("FP1", [self._mapping("RP1", 1), self._mapping("RP2", 3, front_position=2)], positions=2) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 1}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2", "rear_port_position": 3}, + ] + + def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): + """The shape the library actually carries: many front ports onto one MPO rear port.""" + ports = [self._front_port(f"FP{i}", [self._mapping("MPO1", i)]) for i in (1, 2, 3)] + + result = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert [(m["front_port"], m["rear_port_position"]) for m in result["port-mappings"]] == [ + ("FP1", 1), + ("FP2", 2), + ("FP3", 3), + ] + + def test_a_pre_45_server_still_exports_its_mappings(self): + """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" + from types import SimpleNamespace + + fp = SimpleNamespace( + name="FP1", + type="8p8c", + label="", + description="", + color="", + rear_port=SimpleNamespace(name="RP1"), + rear_port_position=4, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} + ] + + def test_a_front_port_with_no_mapping_adds_no_stanza(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) + + assert "port-mappings" not in result + + def test_a_type_without_front_ports_adds_no_stanza(self): + assert "port-mappings" not in serialize_device_type(self._device(), {1: {}}) + + def test_the_importer_reads_back_what_the_export_wrote(self): + """Serializer to normalizer, both real: the stanza is the seam between them.""" + from core.repo import normalize_port_mappings + + ports = [ + self._front_port("1", [self._mapping("MPO1", 1)]), + self._front_port("2", [self._mapping("MPO1", 2)]), + ] + exported = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert normalize_port_mappings(exported) is None + assert [fp["_mappings"] for fp in exported["front-ports"]] == [ + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 1}], + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 2}], + ] + assert "port-mappings" not in exported, "the normalizer consumes the stanza" From 43480e70e63237d15820f0aa964eb2a9ac9748ab Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 09:25:37 +0200 Subject: [PATCH 05/19] fix(export): default front-port positions for pre-4.5 servers The DTL front-port schema makes positions required, but the field arrived with the NetBox 4.5 mapping model. The pre-4.5 query tiers do not select it, so _serialize_component found no value and omitted the key, and every front-port entry exported from an older server failed schema validation. Default an absent positions to 1, which is what a pre-4.5 front port occupies: one rear-port position. positions is the last registry field, so the key lands in the same slot the 4.5 path produces and the YAML order does not change with the server version. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/134#discussion_r3965186646 --- core/nb_serializer.py | 5 ++++- tests/test_nb_serializer.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 5e899758..9b89619e 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -150,7 +150,10 @@ def _serialize_front_port(record: Any) -> dict: table and the library schema follows, carrying it in a top-level ``port-mappings`` stanza built by :func:`_port_mappings`. """ - return _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) + # positions is schema-required but arrived in 4.5, so a pre-4.5 record has none. + result.setdefault("positions", 1) + return result def _port_mappings(records: list) -> list: diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index fce437b1..fdbbf119 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -654,6 +654,17 @@ def test_a_pre_45_server_still_exports_its_mappings(self): assert result["port-mappings"] == [ {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} ] + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_legacy_front_port_carries_the_schema_required_positions(self): + """The schema requires positions, but it arrived in 4.5, so a pre-4.5 record needs the default.""" + from types import SimpleNamespace + + fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] def test_a_front_port_with_no_mapping_adds_no_stanza(self): result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) From a9fed7212a8a6c82a4976ee3a80376dfc9d4e426 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 09:45:57 +0200 Subject: [PATCH 06/19] feat(export): report module bays exported without a position The library schema requires a position on every module bay, but NetBox leaves the field blank when a bay names no physical slot. A blank string writes no key, so the exported entry silently omits a required field and the file fails validation only later, in the library CI. Name the offending bays as each file is written, so the operator sees which type needs a position before opening a library PR. The export still writes the file: NetBox is the source of truth here, and there is no correct value to invent, unlike front-port positions where one position is implied. module_bays_missing_position keeps the schema fact next to the serializer that produces the entries, and the exporter owns the logging as it does for the other write-time warnings. --- core/export.py | 7 +++++ core/nb_serializer.py | 9 +++++++ tests/test_exporter.py | 59 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/core/export.py b/core/export.py index f0911ee6..b14ebc7b 100644 --- a/core/export.py +++ b/core/export.py @@ -23,6 +23,7 @@ from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.nb_serializer import ( COMPONENT_ENDPOINT_NAMES, + module_bays_missing_position, serialize_device_type, serialize_module_type, serialize_rack_type, @@ -466,6 +467,12 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: continue written_count += 1 + bays = module_bays_missing_position(to_write) + if bays: + self.handle.log( + f"[yellow]{item.mfr_name}/{item.filename}: module bay(s) {', '.join(bays)} have " + f"no position, which the library schema requires[/yellow]" + ) images_ok = self._download_type_images(item) if images_ok: update_entry(manifest, f"{item.kind}s", item.manifest_key, item.nb_record.last_updated) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index 9b89619e..a822a130 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -196,6 +196,15 @@ def _port_mappings(records: list) -> list: return stanza +def module_bays_missing_position(serialized: dict) -> list: + """Return the names of module bays the library schema would reject. + + NetBox leaves ``position`` blank on a bay that names no physical slot, and a blank + string writes no key, but the schema requires one on every module bay. + """ + return [bay.get("name", "?") for bay in serialized.get("module-bays", []) if "position" not in bay] + + def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" component = BY_ENDPOINT[endpoint_name] diff --git a/tests/test_exporter.py b/tests/test_exporter.py index d9eebe29..343d9c20 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1565,3 +1565,62 @@ def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_ with pytest.raises(FileNotFoundError, match="No device-type library found"): self._exporter(tmp_path, repo)._verify_repo_available() + + +class TestModuleBayPositionWarning: + """NetBox allows a blank module-bay position; the DTL schema requires one.""" + + @staticmethod + def _item(module_bays, kind="device-type"): + return ExportItem( + kind=kind, + nb_record=_make_dt(), + repo_yaml=None, + serialized={"model": "7750-SR-7s", "module-bays": module_bays}, + reason="absent", + mfr_name="Nokia", + filename="nokia-7750-sr-7s.yaml", + manifest_key="Nokia/nokia-7750-sr-7s", + ) + + @staticmethod + def _write(tmp_path, item): + """Drive the real write path with a real LogHandler, which prints to stdout.""" + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + exporter._get_module_image_details = lambda: {} + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): + item = self._item([{"name": "Slot 0"}, {"name": "Slot 1", "position": "1"}]) + + self._write(tmp_path, item) + + out = capsys.readouterr().out + assert "Slot 0" in out + assert "position" in out + assert "nokia-7750-sr-7s.yaml" in out + assert "Slot 1" not in out, "a bay that has a position is not a problem" + + def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): + """'0' is a real position: the MX304 PSU bays use '0' and '1'.""" + item = self._item([{"name": "Slot 0", "position": "0"}]) + + self._write(tmp_path, item) + + assert "no position" not in capsys.readouterr().out + + def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + item = replace(self._item([]), serialized={"model": "7750-SR-7s"}) + + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + assert "no position" not in capsys.readouterr().out + + def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): + """Device types and module types share the module-bay schema definition.""" + item = self._item([{"name": "Sub 0"}], kind="module-type") + + self._write(tmp_path, item) + + assert "Sub 0" in capsys.readouterr().out From c20153d772c24e8a6d9e5c6629b25284ab00f55c Mon Sep 17 00:00:00 2001 From: Marcin Zieba <49913098+marcinpsk@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:10:26 +0200 Subject: [PATCH 07/19] refactor: drop version branches unreachable at the NetBox 4.3 floor (#135) Closes #133. The importer refuses anything below NetBox 4.3, so three of the five feature flags can no longer be false on a server it will talk to: modules is a 3.2 threshold, new_filters and rack_types are 4.1 thresholds. The branches behind them were still carried, threaded through three constructors and asserted on. Removed the three flags and kept the true side of every branch they guarded. m2m_front_ports and module_bay_types still vary across supported releases and are untouched. core/compat.py keeps its version helpers, which #132 added and both entry points use; only the four filter-key helpers go, because the devicetype_id and moduletype_id spelling they chose between is unreachable at 4.1 and later. The issue predates those helpers and called for deleting the module outright. Tests that asserted the removed behaviour are deleted rather than adapted: a test that pins the legacy filter name or the rack-types skip is asserting what no supported deployment executes. Four filter-key tests collapse into two that say the supported key is the only one ever sent, which still catches the failure that mattered, since the wrong key does not raise but silently returns every row. --- core/compat.py | 58 +---------------- core/component_cache.py | 20 ++---- core/import_run.py | 93 +++++++++++---------------- core/netbox_api.py | 28 +------- core/update_failure_resolver.py | 24 ++----- tests/test_change_detector.py | 2 +- tests/test_component_cache.py | 9 --- tests/test_import_run.py | 7 +- tests/test_module_bay_type_sync.py | 1 - tests/test_nb_dt_import.py | 33 ++-------- tests/test_netbox_api.py | 70 ++++---------------- tests/test_update_failure_resolver.py | 53 ++------------- 12 files changed, 83 insertions(+), 315 deletions(-) diff --git a/core/compat.py b/core/compat.py index 97c59bf5..8ebcbf03 100644 --- a/core/compat.py +++ b/core/compat.py @@ -1,15 +1,7 @@ """NetBox version-compatibility helpers. -Centralises filter parameter names that changed between NetBox releases so -that every caller uses the same logic and drift is impossible. - -NetBox 4.1 renamed several filter keys on the DCIM endpoints: - - devicetype_id → device_type_id - moduletype_id → module_type_id - -Any code that constructs endpoint filter kwargs should call the helpers -here rather than inlining ``"device_type_id" if new_filters else "devicetype_id"``. +Centralises the version line each feature sits behind so that every caller uses the +same logic and drift is impossible. """ from __future__ import annotations @@ -33,49 +25,3 @@ def parse_netbox_version(version) -> tuple[int, int]: def supports_module_bay_types(version) -> bool: """Return True when this NetBox release exposes the module bay type relation.""" return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION - - -def device_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for device-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"device_type_id"``); - ``False`` for older releases (returns ``"devicetype_id"``). - """ - return "device_type_id" if new_filters else "devicetype_id" - - -def module_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for module-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"module_type_id"``); - ``False`` for older releases (returns ``"moduletype_id"``). - """ - return "module_type_id" if new_filters else "moduletype_id" - - -def device_type_filter_kwargs(device_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a device type. - - Args: - device_type_id: NetBox ID of the device type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {device_type_filter_key(new_filters): device_type_id} - - -def module_type_filter_kwargs(module_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a module type. - - Args: - module_type_id: NetBox ID of the module type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {module_type_filter_key(new_filters): module_type_id} diff --git a/core/component_cache.py b/core/component_cache.py index d8dc4a22..f314ecf7 100644 --- a/core/component_cache.py +++ b/core/component_cache.py @@ -14,12 +14,6 @@ import threading from typing import Any -from core.compat import ( - device_type_filter_key, - device_type_filter_kwargs, - module_type_filter_key, - module_type_filter_kwargs, -) from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError @@ -118,14 +112,13 @@ class ComponentCache: filter, which happens for types created during this run and after an invalidation. """ - def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_record=None): + def __init__(self, netbox, graphql, handle, max_threads, wrap_record=None): """Build a cache over the *netbox* REST client and the *graphql* client. Args: netbox: pynetbox API object, used for REST fallbacks and count checks. graphql: GraphQL client used for the bulk fetch. handle: Log handler. - new_filters (bool): Whether this NetBox takes the newer filter parameter names. max_threads (int): Upper bound on concurrent endpoint fetches. wrap_record (callable | None): Applied to every front-port record, so change detection sees one mappings shape across NetBox versions. @@ -133,7 +126,6 @@ def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_recor self.netbox = netbox self.graphql = graphql self.handle = handle - self.new_filters = new_filters self.max_threads = max_threads self._wrap_record = wrap_record or (lambda record: record) @@ -313,9 +305,9 @@ def get(self, endpoint_name, parent_type, parent_id, endpoint): return cached[key] if parent_type == "device": - filter_kwargs = device_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"device_type_id": parent_id} else: - filter_kwargs = module_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"module_type_id": parent_id} result = {item.name: item for item in endpoint.filter(**filter_kwargs)} self.record(endpoint_name, parent_type, parent_id, result) return result @@ -479,8 +471,6 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc Raises: GraphQLCountMismatchError: When an endpoint holds fewer records than REST reports. """ - dt_filter_key = device_type_filter_key(self.new_filters) - mt_filter_key = module_type_filter_key(self.new_filters) dt_ids = list(device_type_ids) mt_ids = list(module_type_ids) @@ -495,9 +485,9 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc rest_endpoint = getattr(self.netbox.dcim, endpoint_name) rest_count = 0 if dt_ids: - rest_count += self._rest_count(rest_endpoint, dt_filter_key, dt_ids) + rest_count += self._rest_count(rest_endpoint, "device_type_id", dt_ids) if mt_ids and component.module_types: - rest_count += self._rest_count(rest_endpoint, mt_filter_key, mt_ids) + rest_count += self._rest_count(rest_endpoint, "module_type_id", mt_ids) if cached_count != rest_count: if vendor_scope_valid: diff --git a/core/import_run.py b/core/import_run.py index f81fd7bf..cdac353b 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -49,8 +49,6 @@ class RunSummary: """Snapshot of the result of one completed import run.""" counter: Counter - modules: bool - rack_types: bool outcome_counts: dict failure_lines: tuple duplicate_definitions: tuple @@ -66,8 +64,6 @@ def capture(cls, netbox, repo, started_at): """ return cls( counter=Counter(netbox.counter), - modules=netbox.modules, - rack_types=netbox.rack_types, outcome_counts=netbox.outcomes.summary_by_kind(), failure_lines=tuple(netbox.outcomes.render_failure_report()), duplicate_definitions=tuple(repo.duplicate_definitions), @@ -455,10 +451,6 @@ def _process_rack_types(config, netbox, handle, progress, rack_types, vendor_nam if not rack_types: return - if not netbox.rack_types: - handle.log("Rack types require NetBox >= 4.1. Skipping rack type import.") - return - handle.verbose_log(f"{len(rack_types)} Rack-Types Found") all_rack_types = netbox.get_existing_rack_types() @@ -521,21 +513,19 @@ def _log_run_summary(handle, summary): handle.log(f"{counter['components_removed']} components removed") handle.verbose_log(f"{counter['images']} images uploaded") handle.log(f"{counter['manufacturer']} manufacturers created") - if summary.modules: - handle.log(f"{counter['module_added']} modules created") - handle.log(f"{counter['module_updated']} modules updated") - module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) - if module_failed: - handle.log(f"{module_failed} modules failed to create or update") - module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) - if module_partial: - handle.log(f"{module_partial} modules partially updated") - if summary.rack_types: - handle.log(f"{counter['rack_type_added']} rack types created") - handle.log(f"{counter['rack_type_updated']} rack types updated") - rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) - if rack_failed: - handle.log(f"{rack_failed} rack types failed") + handle.log(f"{counter['module_added']} modules created") + handle.log(f"{counter['module_updated']} modules updated") + module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) + if module_failed: + handle.log(f"{module_failed} modules failed to create or update") + module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) + if module_partial: + handle.log(f"{module_partial} modules partially updated") + handle.log(f"{counter['rack_type_added']} rack types created") + handle.log(f"{counter['rack_type_updated']} rack types updated") + rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) + if rack_failed: + handle.log(f"{rack_failed} rack types failed") for line in summary.failure_lines: handle.log(line) @@ -673,30 +663,24 @@ def plan_vendor(self, selection, vendor): self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] ) - if self.netbox.modules: - module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None - if module_hint is not None and vendor["slug"] not in module_hint: - module_types = [] - else: - module_types = _parse_vendor_files( - self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] - ) - else: + module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None + if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] - - if self.netbox.rack_types: - rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None - if rack_hint is not None and vendor["slug"] not in rack_hint: - rack_types = [] - else: - rack_types = _parse_vendor_files( - self.repo, - selection.racks_path, - vendor["name"], - self.config.slugs or [], - ) else: + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) + + rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None + if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] + else: + rack_types = _parse_vendor_files( + self.repo, + selection.racks_path, + vendor["name"], + self.config.slugs or [], + ) return VendorPlan( vendor=vendor, @@ -738,17 +722,16 @@ def apply(self, plan): ) cache.pump() - if self.netbox.modules: - _process_module_types( - self.config, - self.netbox, - self.reporter, - self.progress, - plan.module_types, - vendor_name=plan.vendor["name"], - task_registry=self.task_registry, - ) - cache.pump() + _process_module_types( + self.config, + self.netbox, + self.reporter, + self.progress, + plan.module_types, + vendor_name=plan.vendor["name"], + task_registry=self.task_registry, + ) + cache.pump() _process_rack_types( self.config, diff --git a/core/netbox_api.py b/core/netbox_api.py index 979e704d..7bdacf92 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -492,11 +492,8 @@ def __init__(self, config, handle): self.handle = handle self.netbox: Any = None self.ignore_ssl = config.ignore_ssl_errors - self.modules = False - self.new_filters = False self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) - self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts self.remove_unmanaged_types = config.remove_unmanaged_types self.verify_images = config.verify_images @@ -538,7 +535,6 @@ def __init__(self, config, handle): self.handle, self.counter, self.ignore_ssl, - self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, module_bay_types_supported=self.module_bay_types, @@ -603,10 +599,10 @@ def connect_api(self): raise UnknownError("NetBox API Error", cause=e) from e def verify_compatibility(self): - """Check the connected NetBox version and configure feature flags accordingly. + """Refuse a server below the supported floor and set the feature flags above it. - Sets ``self.modules = True`` for NetBox >= 3.2 and ``self.new_filters = True`` - for >= 4.1. Logs the detected version when the new-filter flag is enabled. + Only the flags that still vary across supported releases are set here: the 4.3 + floor makes everything introduced at or below 4.1 unconditional. """ # nb.version should be the version in the form '3.2' # Strip non-numeric suffixes (e.g. "4.1-beta") before converting to int. @@ -644,17 +640,6 @@ def verify_compatibility(self): f"Older releases fail part way through with a GraphQL schema error rather than here." ) - # Later than 3.2 - # Might want to check for the module-types entry as well? - if version_split[0] > 3 or (version_split[0] == 3 and version_split[1] >= 2): - self.modules = True - - # check if version >= 4.1 in order to use new filter names (https://github.com/netbox-community/netbox/issues/15410) - if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 1): - self.new_filters = True - self.rack_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using new filters.") - # NetBox 4.5 replaced FrontPortTemplate.rear_port (FK) + rear_port_position (int) # with a ManyToMany through table (PortMapping). The creation and read APIs differ. # https://github.com/netbox-community/netbox/issues/20564 @@ -777,7 +762,6 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er netbox=self.netbox, device_type_id=dt.id, device_type_yaml=device_type, - new_filters=self.new_filters, ) except Exception as exc: # defensive: classifier must never break the run self.handle.verbose_log(f"Failure classifier raised {type(exc).__name__}: {exc}") @@ -1204,8 +1188,6 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima yaml_key = component.yaml_key if yaml_key not in device_type: continue - if yaml_key == "module-bays" and not self.modules: - continue self.device_types.create_components( yaml_key, device_type[yaml_key], @@ -2320,7 +2302,6 @@ def __init__( handle, counter, ignore_ssl, - new_filters, *, graphql, repo_path, @@ -2338,7 +2319,6 @@ def __init__( handle (LogHandler): Sink for creation and error messages. counter (Counter): Shared operation counter updated during creation. ignore_ssl (bool): Whether SSL certificate verification is disabled. - new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. @@ -2349,7 +2329,6 @@ def __init__( self.handle = handle self.counter = counter self.ignore_ssl = ignore_ssl - self.new_filters = new_filters self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports @@ -2359,7 +2338,6 @@ def __init__( netbox, graphql, handle, - new_filters, max_threads, wrap_record=_FrontPortRecordWithMappings, ) diff --git a/core/update_failure_resolver.py b/core/update_failure_resolver.py index 7e2eb1f6..82543a5d 100644 --- a/core/update_failure_resolver.py +++ b/core/update_failure_resolver.py @@ -24,8 +24,6 @@ from enum import Enum from typing import Any, Callable, List, Optional -from core.compat import device_type_filter_kwargs - class FailureKind(str, Enum): """High-level classification of a NetBox update failure.""" @@ -122,7 +120,7 @@ def _matches_subdevice_role_constraint(payload: Any) -> bool: return False -def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> tuple[int, List[str]]: +def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, List[str]]: """Query NetBox for devices using *device_type_id*. Returns ``(count, sample_names)`` where ``sample_names`` is up to 5 names @@ -133,10 +131,8 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ - filter_kwargs = device_type_filter_kwargs(device_type_id, new_filters=new_filters) + filter_kwargs = {"device_type_id": device_type_id} try: devices = list(netbox.dcim.devices.filter(**filter_kwargs, limit=5)) except Exception: @@ -152,7 +148,7 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b return total, sample -def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> Optional[List[Any]]: +def _list_device_bay_templates(netbox: Any, device_type_id: int) -> Optional[List[Any]]: """Return all ``DeviceBayTemplate`` records attached to *device_type_id*. Returns ``None`` when the NetBox query itself fails (network error, 5xx, etc.) @@ -161,15 +157,9 @@ def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ try: - return list( - netbox.dcim.device_bay_templates.filter( - **device_type_filter_kwargs(device_type_id, new_filters=new_filters) - ) - ) + return list(netbox.dcim.device_bay_templates.filter(device_type_id=device_type_id)) except Exception: return None @@ -180,7 +170,6 @@ def classify_device_type_update_failure( netbox: Any, device_type_id: int, device_type_yaml: dict, - new_filters: bool = False, ) -> FailureResolution: """Classify a ``pynetbox.RequestError`` raised while updating a device type. @@ -192,7 +181,6 @@ def classify_device_type_update_failure( device_type_yaml: Parsed YAML dict for this device-type (used to detect whether the YAML *also* lists device bays — in which case we cannot blindly delete them). - new_filters: When True, use updated filter parameter names (NetBox ≥ 4.1). Returns: A :class:`FailureResolution` describing the constraint and (when safe) @@ -207,7 +195,7 @@ def classify_device_type_update_failure( ) # SUBDEVICE_ROLE_FLIP path ------------------------------------------------- - blocking_templates = _list_device_bay_templates(netbox, device_type_id, new_filters=new_filters) + blocking_templates = _list_device_bay_templates(netbox, device_type_id) if blocking_templates is None: return FailureResolution( kind=FailureKind.MANUAL_REQUIRED, @@ -216,7 +204,7 @@ def classify_device_type_update_failure( ) blocking_names = [getattr(t, "name", str(getattr(t, "id", "?"))) for t in blocking_templates] - dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id, new_filters=new_filters) + dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id) # YAML must NOT redefine device-bays — otherwise deleting them would just # cause our own component-creation step to fail or thrash. This catches diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index 3a9b6434..bf9d6a92 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -19,7 +19,7 @@ def _cache(**records): Populating through the real object means these tests read the same index the importer builds, rather than a dict that merely looks like it. """ - cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), new_filters=True, max_threads=1) + cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), max_threads=1) for endpoint_name, items in records.items(): cache.populate(endpoint_name, items) return cache diff --git a/tests/test_component_cache.py b/tests/test_component_cache.py index a6d9e330..a915936f 100644 --- a/tests/test_component_cache.py +++ b/tests/test_component_cache.py @@ -187,7 +187,6 @@ def make_cache(netbox=None, graphql=None, handle=None, **kwargs): netbox or FakeNetBox(), graphql or FakeGraphQL(), handle or FakeHandle(), - kwargs.pop("new_filters", True), kwargs.pop("max_threads", 4), **kwargs, ) @@ -279,14 +278,6 @@ def test_a_miss_filters_by_module_type_for_a_module_parent(self): assert endpoint.filter_calls == [{"module_type_id": 5}] - def test_old_netbox_filter_names_are_used_when_asked(self): - cache = make_cache(new_filters=False) - endpoint = FakeEndpoint() - - cache.get("interface_templates", "device", 1, endpoint) - - assert endpoint.filter_calls == [{"devicetype_id": 1}] - def test_an_empty_result_still_becomes_a_hit(self): """Otherwise every parent with no components is re-read on each lookup.""" cache = make_cache() diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 1aa14152..9c9d5694 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -59,7 +59,6 @@ class _NetBoxBoundary: """Expose run state and fail if planning starts an import.""" def __init__(self): - self.modules = True self.rack_types = True self.device_types = _DeviceTypes() self.outcomes = _Outcomes() @@ -271,8 +270,6 @@ def test_execute_returns_snapshot_and_owns_console_lifecycle(make_config, tmp_pa assert isinstance(summary, RunSummary) assert summary.counter["added"] == 2 - assert summary.modules is True - assert summary.rack_types is True assert progress_factory.entered is True assert progress_factory.exited is True assert handle.console is None @@ -434,11 +431,9 @@ def test_banners_omit_the_separator_when_no_vendor_is_given(make_config): class _OutcomeNetBox: """Run-state stand-in carrying a real OutcomeRegistry.""" - def __init__(self, *, modules=True, rack_types=True, counter=None): + def __init__(self, *, counter=None): from core.outcomes import OutcomeRegistry - self.modules = modules - self.rack_types = rack_types self.outcomes = OutcomeRegistry() base = dict( added=0, diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index ed2a27e1..6030b764 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -84,7 +84,6 @@ def _make(module_bay_types_supported=True): handle, {}, False, - True, graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), repo_path=str(catalog_root), module_bay_types_supported=module_bay_types_supported, diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index a358c6ca..fea59e41 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -335,13 +335,11 @@ def _make_mock_repo(device_types=None): return mock_repo -def _make_mock_netbox(modules=False, rack_types=False): +def _make_mock_netbox(): """Return a pre-configured NetBox mock.""" from collections import Counter mock_nb = MagicMock() - mock_nb.modules = modules - mock_nb.rack_types = rack_types mock_nb.device_types.existing_device_types = {} mock_nb.device_types.existing_device_types_by_slug = {} mock_nb.count_device_type_images.return_value = 0 @@ -884,7 +882,7 @@ def test_modules_with_types_to_process(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([module_type], {}, []) MockNetBox.return_value = mock_nb MockNetBox.filter_new_module_types.return_value = [] @@ -912,7 +910,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([], {}, change_log) mock_nb.filter_new_module_types.return_value = [] MockNetBox.return_value = mock_nb @@ -930,7 +928,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): mock_nb.log_module_type_changes.assert_called_once_with(change_log) def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): - """When netbox.modules is True, module_added/updated counters are logged.""" + """Module counters are always logged: every supported release has module types.""" with ( patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), patch("nb_dt_import.DTLRepo") as MockRepo, @@ -938,7 +936,7 @@ def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): patch("nb_dt_import.LogHandler") as MockLogHandler, ): MockRepo.return_value = _make_mock_repo() - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() MockNetBox.return_value = mock_nb nb_dt_import.main() @@ -987,19 +985,6 @@ class TestProcessRackTypes: def _make_args(self, only_new=False): return SimpleNamespace(only_new=only_new) - def test_rack_types_disabled_logs_warning_and_returns(self, nb_dt_import): - """netbox.rack_types=False with actual rack types: warning logged, no further processing.""" - handle = MagicMock() - netbox = MagicMock() - netbox.rack_types = False - - rack_type = {"manufacturer": {"slug": "apc"}, "model": "AR1300", "slug": "apc-ar1300"} - import_run_module._process_rack_types(self._make_args(), netbox, handle, None, [rack_type]) - - handle.log.assert_called_once() - assert "4.1" in handle.log.call_args[0][0] - netbox.get_existing_rack_types.assert_not_called() - def test_empty_rack_types_returns_early(self, nb_dt_import): """rack_types=[]: returns immediately without any logging or API calls.""" handle = MagicMock() @@ -1252,7 +1237,7 @@ def test_module_type_only_vendor_uses_scoped_preload(self, nb_dt_import): """ mt = {"manufacturer": {"slug": "acbel"}, "model": "M1", "slug": "acbel-m1"} - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Acbel", "slug": "acbel"}] @@ -1371,8 +1356,6 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = True from collections import Counter mock_nb.counter = Counter( @@ -1404,8 +1387,6 @@ def test_duplicate_definitions_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = False from collections import Counter mock_nb.counter = Counter( @@ -1988,7 +1969,7 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c if files == ["cisco-module-types.yaml"] else [] ) - netbox = _make_mock_netbox(modules=True) + netbox = _make_mock_netbox() slug_resolved = { "device_files": {"empty": [], "cisco": ["resolved.yaml"]}, "module_vendors": {"cisco"}, diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index eb535191..68549282 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -80,7 +80,6 @@ def _factory(nb_api=None, handle=None, counter=None, **kwargs): handle if handle is not None else mock_handle, counter if counter is not None else MagicMock(), False, - False, graphql=kwargs.pop("graphql", graphql_client), repo_path=kwargs.pop("repo_path", mock_settings.repo_path), **kwargs, @@ -118,7 +117,6 @@ def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): assert nb.url == "http://mock-netbox" assert nb.token == "mock-token" # Verify module support detection - assert nb.modules def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, mock_handle): @@ -142,7 +140,6 @@ def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): mock_pynetbox.api.return_value.version = version nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters, version assert nb.module_bay_types is module_bay_types, version @@ -2479,7 +2476,6 @@ def test_creates_all_component_types( nb = NetBox(mock_settings, mock_handle) nb.device_types = dt - nb.modules = True created_dt = MagicMock() created_dt.id = 1 @@ -4174,40 +4170,6 @@ def test_image_file_not_found_logs_error( nb.create_device_types([device_type]) assert any("Error locating image file" in str(c) for c in mock_handle.log.call_args_list) - def test_module_bays_not_created_when_modules_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle - ): - """module-bays are only created when self.modules is True.""" - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - dt.existing_device_types = {} - dt.existing_device_types_by_slug = {} - dt.components.record("module_bay_templates", "device", 1, {}) - - nb = NetBox(mock_settings, mock_handle) - nb.device_types = dt - nb.modules = False # explicitly disabled - - created_dt = MagicMock() - created_dt.id = 1 - created_dt.manufacturer.name = "Cisco" - created_dt.model = "TestSwitch" - mock_nb_api.dcim.device_types.create.return_value = created_dt - - device_type = { - "manufacturer": {"slug": "cisco"}, - "model": "TestSwitch", - "slug": "testswitch", - "module-bays": [{"name": "MB1"}], - "src": "/tmp/device-types/cisco/testswitch.yaml", - } - nb.create_device_types([device_type]) - mock_nb_api.dcim.module_bay_templates.create.assert_not_called() - - -class TestCreateModuleTypesCornerCases: - """Corner-case tests for create_module_types (cognitive complexity 16).""" - def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" mock_pynetbox.api.return_value.version = "4.3" @@ -4605,49 +4567,45 @@ class TestVerifyCompatibility: """Tests for NetBox.verify_compatibility() version thresholds.""" @pytest.mark.parametrize( - "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", + "version_str, expected_m2m, expected_module_bay_types", [ - ("4.3", True, True, True, False), - ("4.4", True, True, True, False), - ("4.5", True, True, True, True), - ("4.6", True, True, True, True), - ("4.7", True, True, True, True), - ("5.0", True, True, True, True), + ("4.3", False, False), + ("4.4", False, False), + ("4.5", True, False), + ("4.6", True, False), + ("4.7", True, True), + ("5.0", True, True), # Version strings with non-numeric suffixes - ("4.5-beta", True, True, True, True), - ("4.3.0", True, True, True, False), + ("4.5-beta", True, False), + ("4.3.0", False, False), ], ) def test_version_thresholds( self, version_str, - expected_modules, - expected_new_filters, - expected_rack_types, expected_m2m, + expected_module_bay_types, mock_settings, mock_pynetbox, mock_handle, ): + """Only the flags that still vary above the 4.3 floor are set.""" mock_pynetbox.api.return_value.version = version_str nb = NetBox(mock_settings, mock_handle) - assert nb.modules == expected_modules, f"modules mismatch for {version_str}" - assert nb.new_filters == expected_new_filters, f"new_filters mismatch for {version_str}" - assert nb.rack_types == expected_rack_types, f"rack_types mismatch for {version_str}" assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" + assert nb.module_bay_types == expected_module_bay_types, f"module_bay_types mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): """A version string with only a major component (e.g. '5') does not crash.""" mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True + assert nb.m2m_front_ports is True def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( self, mock_settings, mock_pynetbox, mock_handle ): mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True assert nb.m2m_front_ports is False assert nb.module_bay_types is False @@ -6808,7 +6766,6 @@ def _device_types(url, handle): handle, MagicMock(), False, - False, graphql=NetBoxGraphQLClient(url, "token", page_size=10), repo_path="/tmp/repo", max_threads=2, @@ -7222,7 +7179,6 @@ def test_module_type_create_failure_is_not_called_an_update_failure( b'{"model":["This field may not be blank."]}' ) nb = NetBox(mock_settings, mock_handle) - nb.modules = True nb._process_single_module_type( {"manufacturer": {"slug": "panduit"}, "model": "FAP6WBUSC", "slug": "fap6wbusc"}, "/repo/module-types/Panduit/FAP6WBUSC.yaml", diff --git a/tests/test_update_failure_resolver.py b/tests/test_update_failure_resolver.py index a6135626..93a4bf2f 100644 --- a/tests/test_update_failure_resolver.py +++ b/tests/test_update_failure_resolver.py @@ -314,57 +314,18 @@ def test_classifier_count_fallback_when_count_query_fails(): assert res.dependent_devices_count == 5 -def test_new_filters_uses_device_type_id_key(): - """new_filters=True must call filter(device_type_id=...) not devicetype_id=... +def test_the_device_bay_template_lookup_uses_the_supported_filter_key(): + """The 4.1 rename is below the supported floor, so only device_type_id is ever sent. - This matters because NetBox >= 4.1 changed the query param name. - Passing the wrong key causes pynetbox to silently return ALL templates. + The wrong key does not raise: pynetbox silently returns every template, so the + classifier would report an unrelated device type as the blocker. """ nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=99, device_type_yaml={}) nb.dcim.device_bay_templates.filter.assert_called_once_with(device_type_id=99) -def test_old_filters_uses_devicetype_id_key(): - """new_filters=False (default) must call filter(devicetype_id=...) for NetBox < 4.1.""" - nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.device_bay_templates.filter.assert_called_once_with(devicetype_id=99) - - -def test_count_dependent_devices_uses_new_filter_key(): - """When new_filters=True, dcim.devices must be queried with device_type_id= not devicetype_id=.""" +def test_the_dependent_device_count_uses_the_supported_filter_key(): nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=77, device_type_yaml={}) nb.dcim.devices.filter.assert_called_once_with(device_type_id=77, limit=5) - - -def test_count_dependent_devices_uses_legacy_filter_key(): - """When new_filters=False (default), dcim.devices must be queried with devicetype_id=.""" - nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.devices.filter.assert_called_once_with(devicetype_id=77, limit=5) From 7b3c9e5b673fad5017ee28aea0c937393b9b408e Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 14:07:20 +0200 Subject: [PATCH 08/19] fix(export): treat an omitted front-port positions as the default The export now always writes positions, because the library schema requires it. _is_subset needs every NetBox leaf to be present in the repo YAML, so a library entry that omits positions no longer matched the serialized positions: 1 and the whole definition read as differing. On a library that omits the field, an export would rewrite every such file. Fill an absent front-port positions with 1 on both sides before comparing. The default is what an omitted value means, so this restores the old comparison without giving up the field the schema requires. A non-default value still differs: the merge puts the default first so an explicit value wins. Rejected the alternative of adding positions to _OMIT_IF_EQUAL: that would drop the key from the output again and reintroduce the schema violation fixed in 43480e7. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3967814233 --- core/export.py | 13 +++++++++++-- tests/test_exporter.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/core/export.py b/core/export.py index b14ebc7b..b01ede70 100644 --- a/core/export.py +++ b/core/export.py @@ -162,8 +162,17 @@ def _norm_mfr(d: dict) -> dict: return d return {**d, "manufacturer": _canon_mfr_slug(d["manufacturer"])} - nrepo = _normalize_for_compare(_norm_mfr(repo_yaml)) - nnb = _normalize_for_compare(_norm_mfr(nb_serialized)) + # The serializer always writes the schema-required positions; an entry that omits it + # means the default, so filling it here keeps those definitions from reading as differing. + def _default_positions(d: dict) -> dict: + ports = d.get("front-ports") + if not isinstance(ports, list): + return d + filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] + return {**d, "front-ports": filled} + + nrepo = _normalize_for_compare(_default_positions(_norm_mfr(repo_yaml))) + nnb = _normalize_for_compare(_default_positions(_norm_mfr(nb_serialized))) return _is_subset(nnb, nrepo) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 343d9c20..e5184fb5 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -259,6 +259,32 @@ def test_an_empty_relation_does_not_make_every_definition_differ(self): assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + def test_a_default_positions_does_not_make_every_front_port_differ(self): + """The export writes the schema-required positions; it must not re-export the library. + + A library entry that omits positions means the default, 1. _is_subset requires every + NetBox leaf to be present in the repo YAML, so a serialized positions: 1 compared + against an entry that omits it would report every such definition as differing. + """ + from types import SimpleNamespace + from core.nb_serializer import _serialize_front_port + + legacy = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + serialized = _serialize_front_port(legacy) + assert serialized == {"name": "FP1", "type": "8p8c", "positions": 1} + + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [serialized]} + + assert _repo_supersedes(repo, nb), "an omitted positions is the default, not a difference" + + def test_a_non_default_positions_still_differs(self): + """Only the default may be treated as absent, or a real change would be suppressed.""" + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c", "positions": 4}]} + + assert _repo_supersedes(repo, nb) is False + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} From ac319ceaa9f39112f727c692b1bba4b799421a15 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 14:08:36 +0200 Subject: [PATCH 09/19] fix(config): report a NETBOX_URL that sends the token in cleartext The API token travels in an Authorization header on every request, including the /api/status/ probe added for module bay type detection. An http:// NETBOX_URL pointing off this machine puts that token on the wire in cleartext (CWE-319). Add a startup notice through the existing notices channel, so it covers the importer and the export alike from the one place NETBOX_URL is resolved. Warn rather than reject. Plain HTTP to NetBox on a trusted network is a common deployment, the tool already carries IGNORE_SSL_ERRORS for imperfect transport, and a hard failure would break those users with no upgrade path. Loopback is exempt: it never leaves the host, and the integration tests run against it. Review finding: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3967814227 --- core/config.py | 24 +++++++++++++++++++++++- tests/test_config.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/core/config.py b/core/config.py index d37778e6..fe7fa307 100644 --- a/core/config.py +++ b/core/config.py @@ -1,9 +1,11 @@ """Resolution of one run's configuration from the command line and the environment.""" +import ipaddress import os import re from argparse import ArgumentParser from dataclasses import dataclass, field +from urllib.parse import urlparse from dotenv import load_dotenv @@ -22,6 +24,20 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def _sends_token_in_cleartext(url): + """Return True when *url* would send the API token over plain HTTP off this host.""" + parsed = urlparse(str(url or "").strip()) + if parsed.scheme != "http": + return False + host = (parsed.hostname or "").casefold() + if host in {"", "localhost"}: + return False + try: + return not ipaddress.ip_address(host).is_loopback + except ValueError: + return True + + def is_local_repo_url(url): """Return True when *url* is the sentinel that turns off every git operation.""" return str(url or "").strip().casefold() == LOCAL_REPO_URL @@ -271,6 +287,12 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + netbox_url = _text(env, "NETBOX_URL") + if _sends_token_in_cleartext(netbox_url): + notices.append( + "NETBOX_URL uses http:// on a remote host, so the API token is sent in cleartext. " + "Use https:// unless NetBox is on this machine." + ) if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: notices.append( f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " @@ -278,7 +300,7 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: ) return RunConfig( - netbox_url=_text(env, "NETBOX_URL"), + netbox_url=netbox_url, netbox_token=_text(env, "NETBOX_TOKEN"), ignore_ssl_errors=(_text(env, "IGNORE_SSL_ERRORS", "False") or "False").casefold() in {"true", "1", "yes"}, graphql_page_size=_positive_int(env, "GRAPHQL_PAGE_SIZE", DEFAULT_GRAPHQL_PAGE_SIZE), diff --git a/tests/test_config.py b/tests/test_config.py index 1aae1838..d9db0907 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -139,3 +139,39 @@ def test_a_branch_set_beside_a_real_url_needs_no_notice(self): config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + +class TestInsecureNetboxUrlIsReported: + """The API token travels in an Authorization header, so cleartext transport leaks it.""" + + @pytest.mark.parametrize( + "url", + [ + "http://netbox.example.com", + "http://netbox.example.com:8000/", + "http://10.0.0.5:8000", + ], + ) + def test_http_to_a_remote_host_is_reported(self, url): + config = _resolve(NETBOX_URL=url) + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + @pytest.mark.parametrize( + "url", + [ + "http://localhost:8000", + "http://127.0.0.1:8000/", + "http://[::1]:8000", + ], + ) + def test_http_to_loopback_needs_no_notice(self, url): + """A loopback URL never leaves the host, and the integration tests rely on it.""" + config = _resolve(NETBOX_URL=url) + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_https_needs_no_notice(self): + config = _resolve(NETBOX_URL="https://netbox.example.com") + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices From 2f0bd8016ed7b0d28a024cf235f4227097134894 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 9 Sep 2026 18:12:05 +0200 Subject: [PATCH 10/19] fix(module-bay-types): separate a catalog load failure from a name miss ModuleBayTypeError carried two failures of different severity. A name that does not resolve is per definition, and every caller recovers from it to report that one definition and continue. An unreadable, malformed or duplicated catalog file is terminal: nothing in the catalog can be trusted. Because both raised the same type, those per-definition handlers caught the terminal one too. A single broken catalog file was reported as one skipped component after another, and _load_catalog's own docstring already claimed the failure "is terminal for the run" while the code made it anything but. ModuleBayCatalogError is a sibling of ModuleBayTypeError under FatalError, never a subclass, so the existing `except ModuleBayTypeError` handlers let it through and the run ends once with the real reason. The three load failures in _read_catalog raise it; per-name lookups and per-definition NetBox rejections keep ModuleBayTypeError. The four catalog-reading tests that asserted the old shared type now assert the new one, which is the behaviour change this makes. Also reuse the already-fetched nb_version in the two version log lines. Each pynetbox `version` access is another HTTP request, and both of these sat outside the try that maps transport failures to NetBoxError, so a connection drop there escaped unmapped. Review findings: https://github.com/marcinpsk/Device-Type-Library-Import/pull/136#discussion_r3970510857 https://github.com/marcinpsk/Device-Type-Library-Import/pull/136 (review body, core/netbox_api.py:652) --- core/module_bay_types.py | 19 ++++++++--- core/netbox_api.py | 4 +-- tests/test_module_bay_types.py | 62 +++++++++++++++++++++++++++++++--- tests/test_netbox_api.py | 15 ++++++++ 4 files changed, 89 insertions(+), 11 deletions(-) diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 68a23fbc..308c999e 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -46,6 +46,15 @@ class ModuleBayTypeError(FatalError): """ +class ModuleBayCatalogError(FatalError): + """The catalog itself could not be read, so no name in it can be trusted. + + A sibling of :class:`ModuleBayTypeError`, never a subclass: every caller recovers from + that one per definition, which would turn a single unreadable catalog into one skipped + component after another instead of ending the run once. + """ + + class ModuleBayTypeCatalog: """Turn module-bay-type names into NetBox ids, creating what is missing. @@ -137,7 +146,7 @@ def _load_catalog(self): raise self._load_error try: self._entries = self._read_catalog() - except ModuleBayTypeError as exc: + except ModuleBayCatalogError as exc: self._load_error = exc raise return self._entries @@ -154,7 +163,9 @@ def _read_catalog(self): with open(path, encoding="utf-8") as handle: data = yaml.safe_load(handle) except (OSError, yaml.YAMLError) as exc: - raise ModuleBayTypeError(f"Module bay type catalog file {path!r} could not be read: {exc}") from exc + raise ModuleBayCatalogError( + f"Module bay type catalog file {path!r} could not be read: {exc}" + ) from exc if not isinstance(data, dict): continue # Reject a half-written entry here; the readers index on name and dereference slug. @@ -164,12 +175,12 @@ def _read_catalog(self): if not isinstance(data.get(field), str) or not data[field].strip() ] if invalid: - raise ModuleBayTypeError( + raise ModuleBayCatalogError( f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" ) key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) if key in entries: - raise ModuleBayTypeError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + raise ModuleBayCatalogError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") entries[key] = data return entries diff --git a/core/netbox_api.py b/core/netbox_api.py index 7bdacf92..bf2ab197 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -645,11 +645,11 @@ def verify_compatibility(self): # https://github.com/netbox-community/netbox/issues/20564 if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 5): self.m2m_front_ports = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + self.handle.log(f"Netbox version {nb_version} found. Using M2M front/rear port mappings.") if supports_module_bay_types(nb_version): self.module_bay_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Module bay types are supported.") + self.handle.log(f"Netbox version {nb_version} found. Module bay types are supported.") def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index bdf1581d..7fa40c05 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -8,7 +8,7 @@ import pytest -from core.module_bay_types import ModuleBayTypeCatalog, ModuleBayTypeError +from core.module_bay_types import ModuleBayCatalogError, ModuleBayTypeCatalog, ModuleBayTypeError from helpers import FakeNetBox, write_module_bay_type @@ -172,7 +172,7 @@ def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, ca (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.identities_for("generic", ["SFP"]) assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) @@ -183,7 +183,7 @@ def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog) (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.identities_for("generic", ["SFP"]) assert "sfp.yaml" in str(exc.value) @@ -200,7 +200,7 @@ def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catal monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) for _ in range(3): - with pytest.raises(ModuleBayTypeError): + with pytest.raises(ModuleBayCatalogError): cat.identities_for("generic", ["SFP"]) assert len(walks) == 1 @@ -209,7 +209,7 @@ def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") cat, _ = catalog(root=tmp_path) - with pytest.raises(ModuleBayTypeError) as exc: + with pytest.raises(ModuleBayCatalogError) as exc: cat.ids_for("generic", ["SFP"]) assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) @@ -248,3 +248,55 @@ def test_duplicate_names_collapse(self, catalog): assert len(once) == 1 assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once + + +@pytest.mark.real_http +class TestACatalogFailureIsNotAPerComponentSkip: + """A broken catalog is terminal for the run, but every caller recovers per component.""" + + @staticmethod + def _broken_catalog(tmp_path, catalog): + root = tmp_path / "broken" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "fine") + # A half-written entry: the readers index on name and dereference slug. + (root / "module-bay-types" / "Juniper" / "bad.yaml").write_text( + "name: 123\nslug: bad\nmanufacturer: Juniper\n", encoding="utf-8" + ) + return catalog(root=root)[0] + + def test_a_malformed_entry_is_not_reported_as_a_relation_change(self, tmp_path, catalog): + """_relation_change recovers from ModuleBayTypeError, so a load failure must not be one.""" + from types import SimpleNamespace + + from core.change_detector import _relation_change + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + netbox_comp = SimpleNamespace(name="RE0", module_bay_types=[]) + + with pytest.raises(ModuleBayCatalogError): + _relation_change( + "module_bay_types", + {"name": "RE0", "module_bay_types": ["MX304-RE"]}, + netbox_comp, + catalog=broken, + manufacturer="Juniper", + ) + + def test_the_catalog_failure_is_not_a_module_bay_type_error(self, tmp_path, catalog): + """Sibling, not subclass: an `except ModuleBayTypeError` must not swallow it.""" + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + + with pytest.raises(ModuleBayCatalogError) as caught: + broken.identities_for("Juniper", ["MX304-RE"]) + + assert not isinstance(caught.value, ModuleBayTypeError), "per-name catches would swallow it" + + def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, catalog): + """The split must not promote a per-name miss into a run-ending failure.""" + resolver, _server = catalog() + + with pytest.raises(ModuleBayTypeError): + resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 68549282..895c2c62 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -143,6 +143,21 @@ def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): assert nb.module_bay_types is module_bay_types, version +def test_the_netbox_version_is_fetched_once(mock_settings, mock_pynetbox, mock_handle): + """Each pynetbox `version` access is another HTTP request, and only the first is error-mapped.""" + from unittest.mock import PropertyMock + + api_type = type(mock_pynetbox.api.return_value) + version = PropertyMock(return_value="4.7.0") + api_type.version = version + try: + NetBox(mock_settings, mock_handle) + + assert version.call_count == 1, "the log lines must reuse the version already fetched" + finally: + del api_type.version + + def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] From 0b12aca23142736a041ef550bd09d2557ee16111 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 10:10:33 +0200 Subject: [PATCH 11/19] fix: close the paths that resolve a short catalog to the wrong entry Six findings from review, five of them variations on the same shape: a boundary that accepts or skips something quietly, so the failure surfaces later as wrong data rather than as an error. - os.walk suppresses a directory it cannot read. A manufacturer directory that fails to open produced a short catalog, and a same-named Generic entry then answered in its place, writing the wrong module_bay_types relation. An onerror callback makes the traversal error terminal. - A catalog document that parses to a list or a scalar was skipped in the same silent way. Only an empty document is skipped now; any other non-mapping is refused. - The /api/status/ probe read a missing netbox-version as "", which answers "module bay types unsupported" for a 4.7 server and exports without the relation. A non-mapping body raised a bare AttributeError. Both are now GraphQLError. - _is_relation_list and _type_relation_changes accepted a whitespace-only name that the catalog rejects later, which skipped the whole component's update. Both strip before testing, matching the catalog. - The export manifest recorded only last_updated. An unchanged NetBox record keeps that timestamp forever, so an old manifest skipped exactly the types a new output shape was added for. Entries now carry EXPORT_SCHEMA_REVISION and an older revision is not fresh. - The cleartext-token notice parsed the URL with urlparse, which reads "http://10.0.0.1\@localhost" as loopback where requests targets 10.0.0.1. Backslashes are normalized before parsing. The GraphQL session fixture never configured session.get, so the status probe read a synthesized MagicMock and any shape check on the payload saw something no NetBox would return. It now serves a real status body, which is what surfaced the probe finding in eight existing tests. --- core/change_detector.py | 8 +++++-- core/config.py | 4 +++- core/export_manifest.py | 11 ++++++++-- core/graphql_client.py | 7 ++++++- core/module_bay_types.py | 16 ++++++++++++-- core/netbox_api.py | 2 +- tests/conftest.py | 7 +++++++ tests/test_config.py | 10 +++++++++ tests/test_export_manifest.py | 38 +++++++++++++++++++++++++++++----- tests/test_graphql_client.py | 38 ++++++++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 36 ++++++++++++++++++++++++++++++++ tests/test_relation_scope.py | 9 ++++++++ 12 files changed, 172 insertions(+), 14 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 12ea6f0d..e232de17 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -45,8 +45,12 @@ def _relation_properties(comp_type): def _is_relation_list(value): - """Return True when *value* is a list of non-empty strings, the only shape a reference takes.""" - return isinstance(value, list) and all(isinstance(item, str) and item for item in value) + """Return True when *value* is a list of non-empty strings, the only shape a reference takes. + + Blank is checked after stripping, matching the catalog: accepting " " here only defers + the rejection to the write path, where it skips the whole component's update. + """ + return isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value) def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): diff --git a/core/config.py b/core/config.py index fe7fa307..4e1b0068 100644 --- a/core/config.py +++ b/core/config.py @@ -26,7 +26,9 @@ def _sends_token_in_cleartext(url): """Return True when *url* would send the API token over plain HTTP off this host.""" - parsed = urlparse(str(url or "").strip()) + # requests treats a backslash in the authority as a delimiter and urlparse does not, so + # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. + parsed = urlparse(str(url or "").strip().replace("\\", "/")) if parsed.scheme != "http": return False host = (parsed.hostname or "").casefold() diff --git a/core/export_manifest.py b/core/export_manifest.py index b959ddeb..d9ae91c0 100644 --- a/core/export_manifest.py +++ b/core/export_manifest.py @@ -11,6 +11,11 @@ _EMPTY: dict = {"device-types": {}, "module-types": {}, "rack-types": {}} +# Bump whenever the serialized output shape changes (a new stanza, a field that starts or +# stops being written). An unchanged NetBox record has the same last_updated forever, so +# without this an old manifest skips the very types the new shape was added for. +EXPORT_SCHEMA_REVISION = 2 + def load_manifest(path: Path) -> dict: """Load manifest from *path*. Returns an empty manifest on any error.""" @@ -33,13 +38,15 @@ def save_manifest(path: Path, data: dict) -> None: def is_entry_fresh(manifest: dict, kind: str, key: str, last_updated: str) -> bool: - """Return True if the manifest entry for *key* matches *last_updated*.""" + """Return True if *key* was written by this exporter and the record has not changed.""" section = manifest.get(kind) if not isinstance(section, dict): return False entry = section.get(key) if not isinstance(entry, dict): return False + if entry.get("schema") != EXPORT_SCHEMA_REVISION: + return False return entry.get("last_updated") == last_updated @@ -48,4 +55,4 @@ def update_entry(manifest: dict, kind: str, key: str, last_updated: str) -> None section = manifest.get(kind) if not isinstance(section, dict): manifest[kind] = {} - manifest[kind][key] = {"last_updated": last_updated} + manifest[kind][key] = {"last_updated": last_updated, "schema": EXPORT_SCHEMA_REVISION} diff --git a/core/graphql_client.py b/core/graphql_client.py index 88cd53e3..f6e9e8f5 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -212,12 +212,17 @@ def detect_module_bay_type_support(self): try: response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) response.raise_for_status() - version = response.json().get("netbox-version", "") + payload = response.json() except requests.RequestException as exc: raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc except ValueError as exc: # A proxy error page answers 200 with HTML, so the body is not JSON. raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + # Reading an absent version as "" would answer "unsupported" for a 4.7 server and + # export without the relation, so demand the field rather than defaulting it. + version = payload.get("netbox-version") if isinstance(payload, dict) else None + if not isinstance(version, str) or not version.strip(): + raise GraphQLError(f"No netbox-version in the status payload from {status_url}: {payload!r}") self.supports_module_bay_types = supports_module_bay_types(version) return self.supports_module_bay_types diff --git a/core/module_bay_types.py b/core/module_bay_types.py index 308c999e..76697cd6 100644 --- a/core/module_bay_types.py +++ b/core/module_bay_types.py @@ -153,8 +153,16 @@ def _load_catalog(self): def _read_catalog(self): """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + + def _unreadable(exc): + # os.walk skips a directory it cannot read; a short catalog resolves to the + # wrong entry rather than failing, so make the traversal error terminal. + raise ModuleBayCatalogError( + f"Module bay type catalog directory {exc.filename!r} could not be read: {exc}" + ) from exc + entries = {} - for root, _dirs, files in os.walk(self._catalog_dir): + for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable): for filename in sorted(files): if not filename.endswith((".yaml", ".yml")): continue @@ -166,8 +174,12 @@ def _read_catalog(self): raise ModuleBayCatalogError( f"Module bay type catalog file {path!r} could not be read: {exc}" ) from exc + if data is None: + continue # an empty or comment-only document is not an entry if not isinstance(data, dict): - continue + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is not a mapping, got {type(data).__name__}" + ) # Reject a half-written entry here; the readers index on name and dereference slug. invalid = [ field diff --git a/core/netbox_api.py b/core/netbox_api.py index bf2ab197..1e09f712 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1626,7 +1626,7 @@ def _type_relation_changes(self, module_type, existing_module): continue related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] declared = module_type[field] - if not isinstance(declared, list) or any(not isinstance(x, str) or not x for x in declared): + if not isinstance(declared, list) or any(not isinstance(x, str) or not x.strip() for x in declared): # Malformed or bare key: leave the relation unmanaged rather than clear it. continue wanted = sorted(set(declared)) diff --git a/tests/conftest.py b/tests/conftest.py index a1df5ff4..a2d55c4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,6 +114,13 @@ def mock_graphql_requests(request): } } mock_session.post.return_value = response + # /api/status/ is a GET; without this the probe reads a synthesized MagicMock and + # any shape check on the payload sees something no NetBox would ever return. + status = MagicMock() + status.status_code = 200 + status.raise_for_status = MagicMock() + status.json.return_value = {"netbox-version": "4.7.0"} + mock_session.get.return_value = status yield mock_session.post diff --git a/tests/test_config.py b/tests/test_config.py index d9db0907..2db730f7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -175,3 +175,13 @@ def test_https_needs_no_notice(self): config = _resolve(NETBOX_URL="https://netbox.example.com") assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + +class TestCleartextCheckMatchesWhatRequestsWillDo: + """The notice is worthless if the URL it parses is not the URL the token is sent to.""" + + def test_a_backslash_authority_is_not_treated_as_loopback(self): + """The host reads as localhost here, but requests targets the address before the backslash.""" + config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_export_manifest.py b/tests/test_export_manifest.py index 8ffeb796..c646a241 100644 --- a/tests/test_export_manifest.py +++ b/tests/test_export_manifest.py @@ -93,11 +93,10 @@ class TestIsEntryFresh: """Tests for is_entry_fresh function.""" def test_fresh_when_last_updated_matches(self): - manifest = { - "device-types": {"Nokia/acme-x": {"last_updated": "2024-01-01T00:00:00Z"}}, - "module-types": {}, - "rack-types": {}, - } + """Built through update_entry: a hand-written entry cannot carry the schema revision.""" + manifest = {"device-types": {}, "module-types": {}, "rack-types": {}} + update_entry(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") + assert is_entry_fresh(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") is True def test_stale_when_last_updated_differs(self): @@ -125,3 +124,32 @@ def test_updates_existing_entry(self): manifest = {"device-types": {"Nokia/acme-x": {"last_updated": "old"}}, "module-types": {}, "rack-types": {}} update_entry(manifest, "device-types", "Nokia/acme-x", "2024-02-01T00:00:00Z") assert manifest["device-types"]["Nokia/acme-x"]["last_updated"] == "2024-02-01T00:00:00Z" + + +class TestExportSchemaRevision: + """last_updated alone cannot see a change in what the exporter writes.""" + + def test_an_entry_from_an_older_exporter_is_not_fresh(self): + """The record did not change, but the serialized shape did, so it must be rewritten.""" + from core.export_manifest import is_entry_fresh + + manifest = {"device-types": {"Acme/x": {"last_updated": "2026-01-01T00:00:00Z"}}} + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is False + + def test_an_entry_this_exporter_wrote_is_fresh(self): + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is True + + def test_a_changed_record_is_still_not_fresh(self): + """The revision must not paper over the timestamp check it sits beside.""" + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-02-02T00:00:00Z") is False diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 5cfb8004..e5ec70c7 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -201,6 +201,44 @@ def log_message(self, *args): server.shutdown() server.server_close() + @pytest.mark.real_http + @pytest.mark.parametrize( + ("body", "why"), + [ + (b"[]", "a JSON list has no .get, so the probe raised a bare AttributeError"), + (b"{}", "no netbox-version read as '' and silently disabled the relation"), + (b'{"netbox-version": ""}', "an empty version is not a version"), + (b'{"netbox-version": null}', "a null version is not a version"), + ], + ) + def test_a_wrong_shaped_status_body_fails_the_probe(self, body, why): + """Silently deciding 'unsupported' on a 4.7 server exports without the relation.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + @pytest.mark.real_http def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index 7fa40c05..ab0a1a32 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -165,6 +165,17 @@ def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog) cat, _ = catalog(root=tmp_path) assert len(cat.ids_for("generic", ["SFP"])) == 1 + def test_a_yaml_document_that_is_not_a_mapping_is_refused(self, tmp_path, catalog): + """Silently skipping it shrinks the catalog, and a Generic entry then answers instead.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + directory = tmp_path / "module-bay-types" / "Juniper" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("- name: SFP\n slug: sfp\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("Juniper", ["SFP"]) + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" directory = tmp_path / "module-bay-types" / "Generic" @@ -300,3 +311,28 @@ def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, c with pytest.raises(ModuleBayTypeError): resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) + + +@pytest.mark.real_http +class TestUnreadableCatalogDirectory: + """os.walk swallows a directory it cannot read, which silently shrinks the catalog.""" + + def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, catalog): + """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" + import os + + if os.geteuid() == 0: + pytest.skip("root ignores the permission bits this test relies on") + + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "the owner-scoped entry") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "the fallback entry") + vendor_dir = root / "module-bay-types" / "Juniper" + os.chmod(vendor_dir, 0o000) + try: + resolver, _server = catalog(root=root) + + with pytest.raises(ModuleBayCatalogError): + resolver.identities_for("Juniper", ["QSFP-DD"]) + finally: + os.chmod(vendor_dir, 0o755) diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py index 7dac8f0f..ca69e25e 100644 --- a/tests/test_relation_scope.py +++ b/tests/test_relation_scope.py @@ -138,6 +138,15 @@ def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + def test_a_whitespace_only_entry_leaves_the_relation_alone(self, two_scope_catalog): + """The catalog rejects it later, which would skip the whole component's update.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [" "]}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): """Reading an absent field as empty would report a change on every run.""" detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) From 51dc2a8a43d04b14316ce39e44c63b83171c51ff Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 10:13:46 +0200 Subject: [PATCH 12/19] test(module-bay-types): restore the vendor directory owner-only The unreadable-directory test only needs the directory readable again so tmp_path cleanup can remove it, so 0o700 does the job. 0o755 hands the group and world access the test never needed, which bandit's S103 flags. --- tests/test_module_bay_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index ab0a1a32..66f52a70 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -335,4 +335,4 @@ def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, with pytest.raises(ModuleBayCatalogError): resolver.identities_for("Juniper", ["QSFP-DD"]) finally: - os.chmod(vendor_dir, 0o755) + os.chmod(vendor_dir, 0o700) From 5d80883b6495070b3adff5db609bd734dbe744f0 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 12:15:41 +0200 Subject: [PATCH 13/19] fix: stop three silent data paths the adversarial review found All three lose or skip data without saying anything. - Adding a mapping to a front port that NetBox 4.5+ reports as unmapped did nothing at all. mappings=[] carries no rear port name, and the comparison inferred the model from the data, so an empty list read as a pre-4.5 record and produced a positions-only tuple. _build_mappings_patch returns None for that, so no PATCH was sent and nothing was logged. The record wrapper now records which model it read instead of leaving it to be guessed. - Exporting from a server below 4.7 stripped module_bay_types from every definition it wrote. The relation is never queried there, so its absence means "not asked", not "cleared", but the writer preserved only absent non-list top-level fields and took component lists wholly from NetBox. Change one description and the restriction was gone. Relations the server cannot answer for are now carried back from the repo, matched by name; a server that does answer stays authoritative, which a second test pins. - The cleartext-token notice could abort the run. urlparse rejects some authorities with ValueError, which escapes the ConfigError handler and reaches the user as a traceback on a configuration that worked before. An authority this tool cannot read is also one it cannot clear as loopback, so it warns rather than raising. Also guards os.geteuid, which does not exist on Windows, so the permission test skips instead of erroring during collection. --- core/change_detector.py | 5 +++- core/config.py | 7 ++++- core/export.py | 43 ++++++++++++++++++++++++++++--- core/netbox_api.py | 6 ++++- tests/test_change_detector.py | 25 ++++++++++++++++++ tests/test_config.py | 6 +++++ tests/test_exporter.py | 47 ++++++++++++++++++++++++++++++++++ tests/test_module_bay_types.py | 2 ++ 8 files changed, 134 insertions(+), 7 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index e232de17..5ea7666f 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -518,7 +518,10 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - has_names = any(m.get("rear_port_name") is not None for m in canonical) + # The wrapper records the model it read; fall back to inference only for a + # caller that did not wrap the record. + m2m = getattr(netbox_comp, "_mappings_m2m", None) + has_names = m2m if m2m is not None else any(m.get("rear_port_name") is not None for m in canonical) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( diff --git a/core/config.py b/core/config.py index 4e1b0068..fd4dd6c8 100644 --- a/core/config.py +++ b/core/config.py @@ -28,7 +28,12 @@ def _sends_token_in_cleartext(url): """Return True when *url* would send the API token over plain HTTP off this host.""" # requests treats a backslash in the authority as a delimiter and urlparse does not, so # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. - parsed = urlparse(str(url or "").strip().replace("\\", "/")) + try: + parsed = urlparse(str(url or "").strip().replace("\\", "/")) + except ValueError: + # urlparse rejects some authorities outright. An authority this tool cannot read is + # also one it cannot clear as loopback, and a notice must never abort the run. + return True if parsed.scheme != "http": return False host = (parsed.hostname or "").casefold() diff --git a/core/export.py b/core/export.py index b01ede70..21670232 100644 --- a/core/export.py +++ b/core/export.py @@ -14,6 +14,7 @@ import requests import yaml +from core.component_registry import COMPONENT_TYPES, MODULE_TYPE_RELATIONS from core.export_manifest import ( is_entry_fresh, load_manifest, @@ -141,6 +142,40 @@ def _yaml_equal(a: dict, b: dict) -> bool: return _normalize_for_compare(a) == _normalize_for_compare(b) +def _relation_names() -> set: + """Every relation key the registry knows, so this does not become a second source.""" + return {relation for component in COMPONENT_TYPES for relation in component.relations} | set(MODULE_TYPE_RELATIONS) + + +def _carry_unqueried_relations(repo_yaml: dict, serialized: dict) -> dict: + """Return *serialized* with relations the server never answered for taken from the repo. + + Below NetBox 4.7 module_bay_types is not queried at all, so its absence means "not asked", + not "cleared". Writing NetBox's answer as it stands would delete the restriction from + every definition the export touches for any other reason. + """ + relations = _relation_names() + result = dict(serialized) + for relation in relations: + if relation in repo_yaml and relation not in result: + result[relation] = repo_yaml[relation] + for key, repo_value in repo_yaml.items(): + if not isinstance(repo_value, list) or not isinstance(result.get(key), list): + continue + by_name = {e.get("name"): e for e in repo_value if isinstance(e, dict)} + merged = [] + for entry in result[key]: + if isinstance(entry, dict): + repo_entry = by_name.get(entry.get("name")) + if isinstance(repo_entry, dict): + carried = {r: repo_entry[r] for r in relations if r in repo_entry and r not in entry} + if carried: + entry = {**entry, **carried} + merged.append(entry) + result[key] = merged + return result + + def _repo_supersedes(repo_yaml: dict, nb_serialized: dict) -> bool: """Return True when *repo_yaml* already contains every field NetBox would write. @@ -456,15 +491,15 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: # top-level fields (e.g. comments, profile) that NetBox does not return # in its serialized output. Component lists are left as NB authoritative. to_write = item.serialized + if item.repo_yaml and not self.graphql.supports_module_bay_types: + to_write = _carry_unqueried_relations(item.repo_yaml, to_write) if item.reason == "differs" and item.repo_yaml: # Only preserve scalar/metadata repo fields not present in the NB output. # Exclude list-valued keys (component sections such as interfaces, power-ports, # console-ports, etc.) so that NB remains authoritative for all components. - extra = { - k: v for k, v in item.repo_yaml.items() if k not in item.serialized and not isinstance(v, list) - } + extra = {k: v for k, v in item.repo_yaml.items() if k not in to_write and not isinstance(v, list)} if extra: - to_write = {**item.serialized, **extra} + to_write = {**to_write, **extra} written = self._write_yaml(dest, to_write) if not written: skipped_overwrite += 1 diff --git a/core/netbox_api.py b/core/netbox_api.py index 1e09f712..63565c6c 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -2241,7 +2241,7 @@ class _FrontPortRecordWithMappings: All other attribute accesses are forwarded to the underlying record. """ - __slots__ = ("_record", "_mappings_canonical") + __slots__ = ("_record", "_mappings_canonical", "_mappings_m2m") def __init__(self, record): """Wrap *record* and pre-compute a canonical mappings list for ChangeDetector compatibility. @@ -2287,6 +2287,10 @@ def __init__(self, record): else None # Both mappings and rear_port_position absent; skip comparison. ) object.__setattr__(self, "_mappings_canonical", canonical) + # Which model the record came from, recorded rather than inferred: an empty M2M list + # carries no names to infer from, and reading it as pre-4.5 drops the rear port name + # the patch needs. + object.__setattr__(self, "_mappings_m2m", mappings_raw is not None) def __getattr__(self, name): """Delegate attribute access to the wrapped record.""" diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index bf9d6a92..d03ffb75 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -428,6 +428,31 @@ def _make_netbox_comp(self, canonical, **attrs): """Build a netbox component with _mappings_canonical and explicit attributes.""" return SimpleNamespace(_mappings_canonical=canonical, **attrs) + def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): + """mappings=[] is 4.5+ saying "none", not a pre-4.5 record without the field. + + Inferring the model from the data cannot tell those apart when the list is empty, and + a positions-only tuple makes _build_mappings_patch return None, so the mapping is + never added and nothing is logged. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + netbox_comp = _FrontPortRecordWithMappings(SimpleNamespace(name="FP1", type="8p8c", mappings=[])) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "adding a mapping is a change" + tup = next(iter(changes[0].new_value)) + assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" + assert tup[0] == "RP1" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { diff --git a/tests/test_config.py b/tests/test_config.py index 2db730f7..e289960c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -185,3 +185,9 @@ def test_a_backslash_authority_is_not_treated_as_loopback(self): config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_an_unparseable_url_is_reported_not_raised(self): + """Urlparse raises on some authorities; an advisory notice must not abort the run.""" + config = _resolve(NETBOX_URL="http://user[foo]@198.18.0.1") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_exporter.py b/tests/test_exporter.py index e5184fb5..456f688c 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1650,3 +1650,50 @@ def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): self._write(tmp_path, item) assert "Sub 0" in capsys.readouterr().out + + +class TestUnqueriedRelationsSurviveTheExport: + """A pre-4.7 server never returns module_bay_types, so the export must not strip it.""" + + @staticmethod + def _item(repo_yaml, serialized): + return ExportItem( + kind="device-type", + nb_record=_make_dt(), + repo_yaml=repo_yaml, + serialized=serialized, + reason="differs", + mfr_name="Juniper", + filename="mx304.yaml", + manifest_key="Juniper/mx304", + ) + + def _write(self, tmp_path, item, supported): + exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = supported + exporter._get_module_image_details = lambda: {} + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) + + def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path): + """Only the description changed; the relation must not be collateral damage.""" + repo = { + "model": "MX304", + "description": "old", + "module-bays": [{"name": "RE0", "position": "0", "module_bay_types": ["MX304-RE"]}], + } + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0", "position": "0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=False) + + assert written["description"] == "new", "the real change still lands" + assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] + + def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): + """On 4.7 an absent relation means NetBox cleared it, so it must not be resurrected.""" + repo = {"model": "MX304", "module-bays": [{"name": "RE0", "module_bay_types": ["MX304-RE"]}]} + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=True) + + assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py index 66f52a70..82a2e4d1 100644 --- a/tests/test_module_bay_types.py +++ b/tests/test_module_bay_types.py @@ -321,6 +321,8 @@ def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" import os + if not hasattr(os, "geteuid"): + pytest.skip("no POSIX ownership, so the permission bits mean nothing here") if os.geteuid() == 0: pytest.skip("root ignores the permission bits this test relies on") From c4baac89a5d7fb9aed8fda4810ac1b94fbeafdd9 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 13:09:48 +0200 Subject: [PATCH 14/19] fix(mappings): report legacy truncation and detect a legacy rear-port move Three more silent paths from the same review, all on the pre-4.5 mapping model, plus the stanza case that could not express a removal. - A front port moved from RP1 to RP2 produced no change, no PATCH and no warning. The pre-4.5 query does ask for rear_port { id name }, but the record wrapper discarded the name, leaving only positions to compare, and the positions had not changed. The wrapper keeps the name now, and the comparison uses names whenever one is available. - The update path truncated several mappings to one in silence, where the create path already said so. Both now report it through _log_component_error, so the text reaches the outcome reason instead of scrolling past. This does not reclassify the outcome itself. - Which mapping survived that truncation was arbitrary: the winner came from next(iter(frozenset)). It is the sorted first now, so two runs against the same library agree. - port-mappings: [] could not clear anything. An empty stanza fell back to the inline format and no front port received _mappings: [], so a removal was unrepresentable. An explicitly empty stanza is authoritative now, and an empty stanza beside a surviving inline rear_port is a conflict rather than a silent preference. An absent key still means "no opinion", which a test pins, because reading it as empty would clear every file. normalize_port_mappings passed the complexity gate as a result, so the inline collection and the conflict check moved into helpers. Also asserts the written file in two export warning tests. They checked only that no warning appeared, which would hold if the writer never ran. --- core/change_detector.py | 9 ++-- core/netbox_api.py | 18 +++++-- core/repo.py | 99 ++++++++++++++++++++++------------- tests/test_change_detector.py | 23 ++++++++ tests/test_exporter.py | 5 ++ tests/test_netbox_api.py | 34 ++++++++++++ tests/test_repo.py | 41 +++++++++++++++ 7 files changed, 187 insertions(+), 42 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 5ea7666f..afc914f1 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -518,10 +518,11 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - # The wrapper records the model it read; fall back to inference only for a - # caller that did not wrap the record. - m2m = getattr(netbox_comp, "_mappings_m2m", None) - has_names = m2m if m2m is not None else any(m.get("rear_port_name") is not None for m in canonical) + # Compare by name whenever one is available: an empty M2M list has no name to + # infer from but still needs the named path, and the pre-4.5 query returns one. + has_names = bool(getattr(netbox_comp, "_mappings_m2m", False)) or any( + m.get("rear_port_name") is not None for m in canonical + ) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( diff --git a/core/netbox_api.py b/core/netbox_api.py index 63565c6c..e4d9b253 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -2275,10 +2275,18 @@ def __init__(self, record): else: # NetBox < 4.5: rear_port_position is a direct scalar field rp_pos = getattr(record, "rear_port_position", None) + # The pre-4.5 query asks for rear_port { id name }; keeping the name is what lets + # a move to a different rear port be seen at all. + legacy_rp = getattr(record, "rear_port", None) + legacy_name = ( + (legacy_rp.get("name") if isinstance(legacy_rp, dict) else getattr(legacy_rp, "name", None)) + if legacy_rp is not None + else None + ) canonical = ( [ { - "rear_port_name": None, + "rear_port_name": legacy_name, "front_port_position": 1, "rear_port_position": rp_pos, } @@ -2547,7 +2555,11 @@ def _apply_mappings_change(self, comp_name, new_mappings, yaml_mappings, update_ update_data["rear_port"] = None update_data["rear_port_position"] = None return - first = next(iter(new_mappings)) + if len(new_mappings) > 1: + self._log_component_error( + f'Multiple mappings for front port "{comp_name}" on NetBox < 4.5: only first mapping applied' + ) + first = sorted(new_mappings)[0] if len(first) != 3: # Legacy NetBox (<4.5): ChangeDetector emits 2-tuples (fp_pos, rp_pos) # because rear port names are unavailable via the GraphQL API. @@ -2917,7 +2929,7 @@ def link_rear_ports(items, pid): else: if len(resolved) > 1: ctx = f" (Context: {context})" if context else "" - self.handle.log( + self._log_component_error( f'Multiple mappings for {label} "{port["name"]}" on NetBox < 4.5: ' f"only first mapping applied{ctx}" ) diff --git a/core/repo.py b/core/repo.py index 2022cb57..ec7cf0c3 100644 --- a/core/repo.py +++ b/core/repo.py @@ -272,6 +272,51 @@ def validate_repo_path(repo_path): return True, "" +def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): + """Return ``({front_port_name: [mapping, ...]}, error)`` for the pre-4.5 inline format. + + The inline keys are removed from each entry as they are read, so the caller is left with + one representation to reason about. + """ + inline_mappings: dict = {} + for fp in front_ports: + rp_name = fp.get("rear_port") + if rp_name is None: + continue + fp_name = fp.get("name") + if rear_ports_declared and rp_name not in rear_by_name: + return {}, f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" + rp_pos = fp.pop("rear_port_position", 1) + fp.pop("rear_port") + inline_mappings.setdefault(fp_name, []).append( + {"rear_port": rp_name, "front_port_position": 1, "rear_port_position": rp_pos} + ) + return inline_mappings, None + + +def _conflicting_mapping(inline_mappings, stanza_mappings): + """Return an error when both formats describe the same front port differently. + + Carrying both is allowed only while they agree, which is what a half-finished migration + looks like; disagreeing is the case where guessing a winner would silently pick one. + """ + if not (inline_mappings and stanza_mappings): + return None + + def _shape(mappings): + return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) + + for name in set(inline_mappings) | set(stanza_mappings): + inline = _shape(inline_mappings.get(name, [])) + stanza = _shape(stanza_mappings.get(name, [])) + if inline != stanza: + return ( + f"Error: front port '{name}' has conflicting mapping definitions " + f"(inline: {inline}, port-mappings stanza: {stanza})" + ) + return None + + def normalize_port_mappings(data): """Normalize port mapping definitions in a parsed YAML device/module type dict. @@ -320,27 +365,14 @@ def normalize_port_mappings(data): # --- Old inline format --- # Collect rear_port references declared directly on front-port entries. - inline_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - for fp in front_ports: - rp_name = fp.get("rear_port") - if rp_name is None: - continue - fp_name = fp.get("name") - if rear_ports_declared and rp_name not in rear_by_name: - return f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" - rp_pos = fp.pop("rear_port_position", 1) - fp.pop("rear_port") - inline_mappings.setdefault(fp_name, []).append( - { - "rear_port": rp_name, - "front_port_position": 1, - "rear_port_position": rp_pos, - } - ) + inline_mappings, error = _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared) + if error: + return error # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - if "port-mappings" in data: + stanza_present = "port-mappings" in data + if stanza_present: for entry in port_mappings_stanza or []: fp_name = entry.get("front_port") rp_name = entry.get("rear_port") @@ -359,24 +391,21 @@ def normalize_port_mappings(data): ) del data["port-mappings"] - # --- Conflict detection --- - # Accept both formats simultaneously only when they describe identical mappings. - if inline_mappings and stanza_mappings: - all_names = set(inline_mappings) | set(stanza_mappings) - for name in all_names: - inline = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in inline_mappings.get(name, []) - ) - stanza = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in stanza_mappings.get(name, []) + conflict = _conflicting_mapping(inline_mappings, stanza_mappings) + if conflict: + return conflict + + # An explicitly empty stanza states there are no mappings. An absent key states nothing, + # so only the first may clear what NetBox already holds. + if stanza_present and not stanza_mappings: + if inline_mappings: + return ( + "Error: port-mappings is empty but front port(s) " + f"{sorted(inline_mappings)} still declare an inline rear_port" ) - if inline != stanza: - return ( - f"Error: front port '{name}' has conflicting mapping definitions " - f"(inline: {inline}, port-mappings stanza: {stanza})" - ) + for fp in front_ports: + fp["_mappings"] = [] + return None effective = stanza_mappings if stanza_mappings else inline_mappings for fp_name, mappings in effective.items(): diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index d03ffb75..f56450d3 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -453,6 +453,29 @@ def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" assert tup[0] == "RP1" + def test_a_legacy_rear_port_change_is_detected_by_name(self): + """The pre-4.5 query asks for rear_port { id name }, so the name is available. + + Discarding it left only positions to compare, and RP1/1 -> RP2/1 produced no change, + no PATCH and no warning: the definition silently never synced. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + legacy = SimpleNamespace(name="FP1", type="8p8c", rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) + netbox_comp = _FrontPortRecordWithMappings(legacy) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "moving the front port to a different rear port is a change" + assert next(iter(changes[0].new_value))[0] == "RP2" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 456f688c..49d605d3 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1633,6 +1633,9 @@ def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): self._write(tmp_path, item) + # Assert the file was written too: silence alone would also hold if nothing ran. + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] assert "no position" not in capsys.readouterr().out def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): @@ -1641,6 +1644,8 @@ def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written == {"model": "7750-SR-7s"}, "the write path ran, it simply had nothing to warn about" assert "no position" not in capsys.readouterr().out def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 895c2c62..095689bf 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -518,6 +518,40 @@ def test_update_components_legacy_mapping_two_tuple_warns_and_skips( assert any("NetBox < 4.5" in str(c) for c in mock_handle.log.call_args_list) +def test_update_components_legacy_truncation_is_reported( + mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle +): + """The create path says "only first mapping applied"; the update path said nothing at all.""" + from core.change_detector import ChangeType, ComponentChange, PropertyChange + + mock_nb_api = MagicMock() + dt = make_device_types(nb_api=mock_nb_api) + dt.m2m_front_ports = False + + existing_fp = MagicMock(id=10, name="FP1") + rp1 = MagicMock(id=21, name="RP1") + rp2 = MagicMock(id=22, name="RP2") + dt.components.record("front_port_templates", "device", 1, {"FP1": existing_fp}) + dt.components.record("rear_port_templates", "device", 1, {"RP1": rp1, "RP2": rp2}) + + # Two mappings: the legacy model can hold only one of them. + new_mappings_set = frozenset({("RP1", 1, 1), ("RP2", 2, 1)}) + changes = [ + ComponentChange( + component_type="front-ports", + component_name="FP1", + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange("_mappings", frozenset(), new_mappings_set)], + ), + ] + + mock_handle.log.reset_mock() + dt.update_components({}, 1, changes, parent_type="device") + + logged = " ".join(str(c) for c in mock_handle.log.call_args_list) + assert "FP1" in logged and "4.5" in logged, f"truncation must be reported, got: {logged}" + + def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( mock_settings, mock_pynetbox, graphql_client, make_device_types ): diff --git a/tests/test_repo.py b/tests/test_repo.py index ab4e695a..457f1a4f 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1796,3 +1796,44 @@ def test_corrupted_device_json_returns_none(self, tmp_path): repo.cwd = "" assert repo.resolve_slug_files(["nokia"]) is None + + +class TestAnExplicitlyEmptyStanza: + """`port-mappings: []` is an author saying "none", which is not the same as saying nothing.""" + + def test_an_empty_stanza_clears_every_front_port_mapping(self): + """Without _mappings: [] the change detector cannot express removing a mapping.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c"}, {"name": "FP2", "type": "8p8c"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 2}], + "port-mappings": [], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [] + assert data["front-ports"][1]["_mappings"] == [] + + def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): + """Silently preferring the inline linkage ignores the newer, explicit statement.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 1}], + "port-mappings": [], + } + + result = normalize_port_mappings(data) + + assert result is not None and result.startswith("Error:"), result + + def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): + """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" + from core.repo import normalize_port_mappings + + data = {"front-ports": [{"name": "FP1", "type": "8p8c"}], "rear-ports": []} + + assert normalize_port_mappings(data) is None + assert "_mappings" not in data["front-ports"][0] From 26f24e96a57986ecd4a44019a460d2d1710136f5 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 13:14:32 +0200 Subject: [PATCH 15/19] test: split composite assertions and drop a useless lambda Each half of a composite assert reports which condition failed, and dict is the callable the stub wanted. Caught by the widened rule set on the branch that enables PT018 and PIE807. --- tests/test_exporter.py | 4 ++-- tests/test_netbox_api.py | 3 ++- tests/test_repo.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 49d605d3..b2e1fa4e 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -1613,7 +1613,7 @@ def _item(module_bays, kind="device-type"): def _write(tmp_path, item): """Drive the real write path with a real LogHandler, which prints to stdout.""" exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) - exporter._get_module_image_details = lambda: {} + exporter._get_module_image_details = dict exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): @@ -1676,7 +1676,7 @@ def _item(repo_yaml, serialized): def _write(self, tmp_path, item, supported): exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) exporter.graphql.supports_module_bay_types = supported - exporter._get_module_image_details = lambda: {} + exporter._get_module_image_details = dict exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 095689bf..b2cdbabc 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -549,7 +549,8 @@ def test_update_components_legacy_truncation_is_reported( dt.update_components({}, 1, changes, parent_type="device") logged = " ".join(str(c) for c in mock_handle.log.call_args_list) - assert "FP1" in logged and "4.5" in logged, f"truncation must be reported, got: {logged}" + assert "FP1" in logged, f"truncation must name the port, got: {logged}" + assert "4.5" in logged, f"truncation must say why, got: {logged}" def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( diff --git a/tests/test_repo.py b/tests/test_repo.py index 457f1a4f..b661dc96 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1827,7 +1827,8 @@ def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): result = normalize_port_mappings(data) - assert result is not None and result.startswith("Error:"), result + assert result is not None, "an empty stanza beside an inline linkage must not pass silently" + assert result.startswith("Error:"), result def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" From 15ac9a9aa9f1f16044ba95853bb1260961dabf6b Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 10 Sep 2026 17:47:43 +0200 Subject: [PATCH 16/19] fix(mappings): name the stanza as authoritative instead of blaming a conflict `_conflicting_mapping` compared the union of front-port names, so any name that appeared in only one of the two formats was reported as a conflict against an empty shape. Two consequences. A front port the stanza never mentions was reported as "conflicting mapping definitions (inline: [('RP1', 1, 1)], port-mappings stanza: [])", which names a stanza that has no opinion on that port. The rejection is still correct: `port-mappings: []` clears every front port, so a stanza speaks for the whole file and an inline linkage it omits cannot be honoured. Only the diagnostic was wrong, so it now says the stanza is authoritative and how to resolve it. A front port the stanza alone named was also rejected, which blocked the half-finished migration the two formats exist to allow: FP1 kept inline and FP2 moved to the stanza. Such a port had no inline linkage, so nothing disagrees. The loop now walks only the inline names. Iteration was over a set, so with several offending ports the reported name varied between runs. It is sorted now. Selecting `effective = stanza_mappings` stays safe: once the check passes, every inline name is present in the stanza with an equal shape, so no inline mapping can be dropped. --- core/repo.py | 16 ++++++++--- tests/test_repo.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/core/repo.py b/core/repo.py index ec7cf0c3..074d19ae 100644 --- a/core/repo.py +++ b/core/repo.py @@ -295,10 +295,12 @@ def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): def _conflicting_mapping(inline_mappings, stanza_mappings): - """Return an error when both formats describe the same front port differently. + """Return an error when the two formats cannot both be honoured. Carrying both is allowed only while they agree, which is what a half-finished migration looks like; disagreeing is the case where guessing a winner would silently pick one. + A stanza speaks for the whole file, so an inline linkage it omits cannot be honoured + either. A port the stanza alone names is not in question: it had no inline linkage. """ if not (inline_mappings and stanza_mappings): return None @@ -306,9 +308,15 @@ def _conflicting_mapping(inline_mappings, stanza_mappings): def _shape(mappings): return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) - for name in set(inline_mappings) | set(stanza_mappings): - inline = _shape(inline_mappings.get(name, [])) - stanza = _shape(stanza_mappings.get(name, [])) + for name in sorted(inline_mappings): + if name not in stanza_mappings: + return ( + f"Error: front port '{name}' declares an inline rear_port but the port-mappings " + f"stanza does not list it; the stanza is authoritative, so add '{name}' to it " + f"or remove the inline rear_port keys" + ) + inline = _shape(inline_mappings[name]) + stanza = _shape(stanza_mappings[name]) if inline != stanza: return ( f"Error: front port '{name}' has conflicting mapping definitions " diff --git a/tests/test_repo.py b/tests/test_repo.py index b661dc96..45c819f2 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1838,3 +1838,75 @@ def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): assert normalize_port_mappings(data) is None assert "_mappings" not in data["front-ports"][0] + + +class TestAStanzaThatDoesNotListAFrontPort: + """A stanza speaks for the whole file, so a port it omits has no mapping.""" + + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): + """The old wording blamed a conflict against a stanza that never mentioned the port.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP2", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an inline linkage the stanza omits must not pass silently" + assert "conflicting mapping definitions" not in result, result + assert "FP1" in result, result + assert "does not list it" in result, result + + def test_a_disagreement_on_a_shared_front_port_still_reads_as_a_conflict(self): + """Both formats naming one port differently is a real conflict, not an omission.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP1", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None + assert "conflicting mapping definitions" in result, result + + def test_a_stanza_may_add_a_port_the_inline_format_never_linked(self): + """The half-finished migration the two formats exist to allow: both are kept.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [ + {"front_port": "FP1", "rear_port": "RP1"}, + {"front_port": "FP2", "rear_port": "RP2"}, + ], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] From e0fb671423bb31af65259bb536565a1cfb6f07dc Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 12:41:11 +0200 Subject: [PATCH 17/19] refactor: remove two ways for the same fact to be stated twice Both changes are mechanical. No behavior changes. `DeviceTypes.module_bay_types` returned the catalog while `NetBox.module_bay_types` is the boolean support flag, and `_type_relation_changes` reads both in one function. The code was correct, but a mistaken receiver silently disabled the relation instead of failing, because a falsy catalog and a False flag read the same at the call site. The catalog is now `module_bay_type_catalog`. No alias is left behind. `get_module_types` hand-rolled the GraphQL selection for `module_bay_types`, which `ComponentType.graphql_relation_fields` already builds for component templates. Two spellings of one relation shape drift as soon as the relation gains a field. Both now come from `relation_selection` in the component registry, and the module type query derives its relation list from `MODULE_TYPE_RELATIONS` rather than a literal. GraphQL ignores the whitespace the extracted form changes. `test_relation_selections_use_the_shared_helper` asserted on call names, so it still passed with `relation_selection` returning an empty string. It now compares the generated selection text and the query actually sent. --- core/change_detector.py | 2 +- core/component_registry.py | 8 ++++-- core/graphql_client.py | 5 ++-- core/netbox_api.py | 21 +++++++------- tests/test_graphql_client.py | 46 ++++++++++++++++++++++++++++++ tests/test_module_bay_type_sync.py | 5 ++-- tests/test_relation_scope.py | 2 +- 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index afc914f1..1a7de71d 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -474,7 +474,7 @@ def _relation_catalog(self): """ from core.module_bay_types import ModuleBayTypeCatalog - catalog = getattr(self.device_types, "module_bay_types", None) + catalog = getattr(self.device_types, "module_bay_type_catalog", None) return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None def _compare_component_properties( diff --git a/core/component_registry.py b/core/component_registry.py index fe414278..8fb4e474 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -24,6 +24,11 @@ MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) +def relation_selection(name): + """Return the GraphQL fields that identify a related object.""" + return f"{name} {{ id name slug manufacturer {{ slug }} }}" + + @dataclass(frozen=True) class ComponentType: """One kind of component template, described once.""" @@ -50,8 +55,7 @@ def graphql_fields(self): @property def graphql_relation_fields(self): """GraphQL selections for this row's relations, one nested block per relation.""" - # slug plus owning manufacturer is the identity; the name alone is ambiguous. - return [f"{name} {{ id name slug manufacturer {{ slug }} }}" for name in self.relations] + return [relation_selection(name) for name in self.relations] @property def compare_properties(self): diff --git a/core/graphql_client.py b/core/graphql_client.py index f6e9e8f5..a188ea6e 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -13,7 +13,7 @@ import requests from core.compat import supports_module_bay_types -from core.component_registry import BY_ENDPOINT +from core.component_registry import BY_ENDPOINT, MODULE_TYPE_RELATIONS, relation_selection # Module-level dedup: tracks (url, requested_page_size) pairs that have already # emitted the page-size clamping warning so the message appears at most once @@ -523,8 +523,7 @@ def get_module_types(self, manufacturer_slugs=None): """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) module_bay_type_selection = ( - "module_bay_types {\n id\n name\n slug\n" - " manufacturer {\n slug\n }\n }\n " + "".join(f"{relation_selection(name)}\n " for name in MODULE_TYPE_RELATIONS) if self.supports_module_bay_types else "" ) diff --git a/core/netbox_api.py b/core/netbox_api.py index e4d9b253..544ce538 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1632,7 +1632,7 @@ def _type_relation_changes(self, module_type, existing_module): wanted = sorted(set(declared)) current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) if _relation_identities_differ( - self.device_types.module_bay_types, + self.device_types.module_bay_type_catalog, self.device_types._manufacturer_slug(module_type.get("manufacturer")), declared, related, @@ -1655,7 +1655,8 @@ def _resolve_type_relations(self, payload): return {k: v for k, v in payload.items() if k not in names} manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) resolved = { - field: self.device_types.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + field: self.device_types.module_bay_type_catalog.ids_for(manufacturer, value) + for field, value in names.items() } return {**payload, **resolved} @@ -1680,7 +1681,7 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): try: - updates[field] = self.device_types.module_bay_types.ids_for( + updates[field] = self.device_types.module_bay_type_catalog.ids_for( self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted ) except ModuleBayTypeError as exc: @@ -2354,7 +2355,7 @@ def __init__( wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None - self._module_bay_types = None + self._module_bay_type_catalog = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2638,7 +2639,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, from core.module_bay_types import ModuleBayTypeError try: - update_data[pc.property_name] = self.module_bay_types.ids_for( + update_data[pc.property_name] = self.module_bay_type_catalog.ids_for( self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value ) except ModuleBayTypeError as exc: @@ -2989,13 +2990,13 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): ) @property - def module_bay_types(self): + def module_bay_type_catalog(self): """The module-bay-type catalog, built once per run from the library checkout.""" - if self._module_bay_types is None: + if self._module_bay_type_catalog is None: from core.module_bay_types import ModuleBayTypeCatalog - self._module_bay_types = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) - return self._module_bay_types + self._module_bay_type_catalog = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_type_catalog @staticmethod def _manufacturer_slug(manufacturer): @@ -3039,7 +3040,7 @@ def _resolve_relations(self, component, items, manufacturer): continue try: replacements = { - field: self.module_bay_types.ids_for(manufacturer, value) for field, value in names.items() + field: self.module_bay_type_catalog.ids_for(manufacturer, value) for field, value in names.items() } except ModuleBayTypeError as exc: self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index e5ec70c7..378a83e1 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -1156,6 +1156,52 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + @pytest.mark.real_http + @pytest.mark.parametrize("caller", ["graphql_relation_fields", "get_module_types"]) + def test_relation_selections_use_the_shared_helper(self, caller): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + expected = "module_bay_types { id name slug manufacturer { slug } }" + if caller == "graphql_relation_fields": + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [expected] + else: + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + assert expected in " ".join(query.split()) + finally: + server.close() + + @pytest.mark.real_http + def test_module_type_query_contains_the_complete_relation_selection(self): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + finally: + server.close() + + selection = "module_bay_types { id name slug manufacturer { slug } }" + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [selection] + assert " ".join(query.split()) == " ".join( + """ +query($pagination: OffsetPaginationInput) { + module_type_list(pagination: $pagination) { + id model part_number airflow description comments weight weight_unit last_updated + module_bay_types { id name slug manufacturer { slug } } + manufacturer { id name slug } + } +} +""".split() + ) + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): """The prefetch runs on clones, not on the client it was cloned from. diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py index 6030b764..afe27626 100644 --- a/tests/test_module_bay_type_sync.py +++ b/tests/test_module_bay_type_sync.py @@ -569,8 +569,9 @@ class TestCatalogWiring: def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): device_types, _ = make_device_types() - catalog = device_types.module_bay_types + catalog = device_types.module_bay_type_catalog assert isinstance(catalog, ModuleBayTypeCatalog) - assert device_types.module_bay_types is catalog + assert device_types.module_bay_type_catalog is catalog + assert not hasattr(device_types, "module_bay_types") assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py index ca69e25e..caee3dd0 100644 --- a/tests/test_relation_scope.py +++ b/tests/test_relation_scope.py @@ -60,7 +60,7 @@ def two_scope_catalog(tmp_path): def _detector(catalog, handle=None): """Build a detector whose device_types exposes the catalog, as the real one does.""" - device_types = type("DeviceTypes", (), {"module_bay_types": catalog, "module_bay_types_supported": True})() + device_types = type("DeviceTypes", (), {"module_bay_type_catalog": catalog, "module_bay_types_supported": True})() return ChangeDetector(device_types, handle or Handle()) From d704a718a36701dbe4140a322a2d602bd82f3866 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 12:45:00 +0200 Subject: [PATCH 18/19] fix(mappings): only an explicit list may clear a relation A bare `port-mappings:` key parses as None. `normalize_port_mappings` decided the stanza was present from key presence alone, so None read the same as an explicit `port-mappings: []` and every front port was assigned `_mappings: []`. The update path writes that to NetBox as a clear, so a one-character typo deleted every front-to-rear mapping on the type. `core/change_detector.py` already stated the opposite rule for the identical YAML shape, that a bare `module_bay_types:` leaves the relation unmanaged, so the two sides disagreed. The rule is now `is_explicit_list` in `core/normalization.py` and both sides call it. A stanza that is neither null nor a list was silently discarded and reported success, and a list holding a non-mapping raised AttributeError out of a function whose contract is to return an "Error:" string. Both now return an error naming the offending value. This is deliberately stricter than `_relation_change`, which logs and ignores the same shape: that path skips one property, this one rejects the file. A nonempty stanza left a front port it omits unmanaged, so an existing linkage on that port could never be removed. `port-mappings: []` already cleared every declared front port, which means a present stanza speaks for the whole file, and a nonempty one now does too. A half-migrated file still fails the existing conflict check before reaching the clearing path, which is what stops a migration from deleting a mapping. Measured against the library: 348 files carry a stanza and every one already lists every front port it declares, so no current definition changes. Mapping order was not canonical. `_is_subset` compares lists of unnamed dicts positionally, so the same mappings in a different order reported "differs" and re-exported unchanged data. The serializer now emits them sorted, and `_repo_supersedes` sorts both inputs, so a hand-written stanza in another order also compares equal. Sorting keeps duplicates, so multiplicity still differs. Three tests could not fail: two compared a value against a list the same emptiness made equal, and one bypassed export selection by supplying its result. Each now fails against a mutation of the code it covers. --- core/change_detector.py | 4 +- core/export.py | 11 +++- core/nb_serializer.py | 6 +- core/normalization.py | 5 ++ core/repo.py | 25 ++++--- tests/test_exporter.py | 127 ++++++++++++++++++++++++++++++++---- tests/test_nb_serializer.py | 25 ++++++- tests/test_repo.py | 92 ++++++++++++++++++++++++++ 8 files changed, 262 insertions(+), 33 deletions(-) diff --git a/core/change_detector.py b/core/change_detector.py index 1a7de71d..5e2ecf78 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -11,7 +11,7 @@ from enum import Enum from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES -from core.normalization import normalize_values +from core.normalization import is_explicit_list, normalize_values from core.formatting import log_property_diffs from core.schema_reader import load_properties_for_type @@ -50,7 +50,7 @@ def _is_relation_list(value): Blank is checked after stripping, matching the catalog: accepting " " here only defers the rejection to the write path, where it skips the whole component's update. """ - return isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value) + return is_explicit_list(value) and all(isinstance(item, str) and item.strip() for item in value) def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): diff --git a/core/export.py b/core/export.py index 21670232..9dd48507 100644 --- a/core/export.py +++ b/core/export.py @@ -206,8 +206,15 @@ def _default_positions(d: dict) -> dict: filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] return {**d, "front-ports": filled} - nrepo = _normalize_for_compare(_default_positions(_norm_mfr(repo_yaml))) - nnb = _normalize_for_compare(_default_positions(_norm_mfr(nb_serialized))) + def _sort_port_mappings(d: dict) -> dict: + mappings = d.get("port-mappings") + if not isinstance(mappings, list) or not all(isinstance(mapping, dict) for mapping in mappings): + return d + fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") + return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} + + nrepo = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml)))) + nnb = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized)))) return _is_subset(nnb, nrepo) diff --git a/core/nb_serializer.py b/core/nb_serializer.py index a822a130..4977553b 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -169,11 +169,12 @@ def _port_mappings(records: list) -> list: stanza = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): name = getattr(record, "name", None) + mappings = [] for mapping in getattr(record, "mappings", None) or []: rear_port = getattr(mapping, "rear_port", None) if not rear_port: continue - stanza.append( + mappings.append( { "front_port": name, "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, @@ -181,6 +182,9 @@ def _port_mappings(records: list) -> list: "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, } ) + stanza.extend( + sorted(mappings, key=lambda m: (m["front_port_position"], m["rear_port_position"], m["rear_port"] or "")) + ) if getattr(record, "mappings", None): continue legacy = getattr(record, "rear_port", None) diff --git a/core/normalization.py b/core/normalization.py index c8846ec9..d807f38a 100644 --- a/core/normalization.py +++ b/core/normalization.py @@ -1,6 +1,11 @@ """Shared value-normalization helpers for YAML-vs-NetBox comparisons.""" +def is_explicit_list(value): + """Only an explicit YAML list manages a relation; null leaves it unmanaged.""" + return isinstance(value, list) + + def normalize_values(yaml_val, nb_val): """Normalize a YAML/NetBox value pair for comparison. diff --git a/core/repo.py b/core/repo.py index 074d19ae..62472f7a 100644 --- a/core/repo.py +++ b/core/repo.py @@ -13,6 +13,7 @@ from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +from core.normalization import is_explicit_list # Top-level directories that make a checkout a device-type library. LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") @@ -362,9 +363,9 @@ def normalize_port_mappings(data): """ front_ports = data.get("front-ports") or [] port_mappings_stanza = data.get("port-mappings") - - if not front_ports and "port-mappings" not in data: - return None + stanza_authoritative = is_explicit_list(port_mappings_stanza) + if port_mappings_stanza is not None and not stanza_authoritative: + return f"Error: port-mappings must be a list: {port_mappings_stanza!r}" front_by_name = {fp["name"]: fp for fp in front_ports if fp.get("name")} rear_ports_declared = "rear-ports" in data @@ -379,9 +380,10 @@ def normalize_port_mappings(data): # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - stanza_present = "port-mappings" in data - if stanza_present: + if stanza_authoritative: for entry in port_mappings_stanza or []: + if not isinstance(entry, dict): + return f"Error: port-mappings entry must be a mapping: {entry!r}" fp_name = entry.get("front_port") rp_name = entry.get("rear_port") if not fp_name or not rp_name: @@ -397,26 +399,23 @@ def normalize_port_mappings(data): "rear_port_position": entry.get("rear_port_position", 1), } ) - del data["port-mappings"] + data.pop("port-mappings", None) conflict = _conflicting_mapping(inline_mappings, stanza_mappings) if conflict: return conflict - # An explicitly empty stanza states there are no mappings. An absent key states nothing, - # so only the first may clear what NetBox already holds. - if stanza_present and not stanza_mappings: - if inline_mappings: + if stanza_authoritative: + if not stanza_mappings and inline_mappings: return ( "Error: port-mappings is empty but front port(s) " f"{sorted(inline_mappings)} still declare an inline rear_port" ) for fp in front_ports: - fp["_mappings"] = [] + fp["_mappings"] = stanza_mappings.get(fp.get("name"), []) return None - effective = stanza_mappings if stanza_mappings else inline_mappings - for fp_name, mappings in effective.items(): + for fp_name, mappings in inline_mappings.items(): if fp_name in front_by_name: front_by_name[fp_name]["_mappings"] = mappings diff --git a/tests/test_exporter.py b/tests/test_exporter.py index b2e1fa4e..8e063884 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -247,17 +247,36 @@ def test_an_empty_relation_does_not_make_every_definition_differ(self): "module_bay_types: []" would therefore be absent from every library definition and re-export the whole library on a NetBox 4.7 server. """ - from core.nb_serializer import _serialize_relations - - bay = type("Bay", (), {"name": "FPC 0", "module_bay_types": []})() - assert _serialize_relations(bay, ("module_bay_types",)) == {} + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type - repo = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} - as_serialized = {"model": "MX304", "module-bays": [{"name": "FPC 0"}]} - with_empty_key = {"model": "MX304", "module-bays": [{"name": "FPC 0", "module_bay_types": []}]} + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +module-bays: + - {name: Slot 0, position: '0'} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + components = {1: {"module_bay_templates": [bay]}} + serialized = serialize_device_type(record, components) - assert _repo_supersedes(repo, as_serialized), "an unchanged definition must not re-export" - assert not _repo_supersedes(repo, with_empty_key), "which is exactly what the empty key would do" + assert serialized["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert _repo_supersedes(repo, serialized), "an unchanged definition must not re-export" + bay.module_bay_types = [DotDict(name="Control")] + populated = serialize_device_type(record, components) + assert populated["module-bays"][0]["module_bay_types"] == ["Control"] + assert not _repo_supersedes(repo, populated), "a new restriction must trigger export" def test_a_default_positions_does_not_make_every_front_port_differ(self): """The export writes the schema-required positions; it must not re-export the library. @@ -285,6 +304,54 @@ def test_a_non_default_positions_still_differs(self): assert _repo_supersedes(repo, nb) is False + @pytest.mark.parametrize("reverse_repo", [False, True]) + @pytest.mark.parametrize("reverse_netbox", [False, True]) + def test_reordered_netbox_mappings_do_not_trigger_export(self, reverse_repo, reverse_netbox): + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Panel +slug: acme-panel +u_height: 1 +is_full_depth: false +front-ports: + - {name: FP1, type: lc-upc, positions: 2} +port-mappings: + - {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} + - {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 3} +""") + record = DotDict( + id=1, manufacturer=DotDict(name="Acme"), model="Panel", slug="acme-panel", u_height=1, is_full_depth=False + ) + port = DotDict( + name="FP1", + type="lc-upc", + positions=2, + mappings=[ + DotDict(rear_port=DotDict(name="RP2"), front_port_position=2, rear_port_position=3), + DotDict(rear_port=DotDict(name="RP1"), front_port_position=1, rear_port_position=1), + ], + ) + if reverse_repo: + repo["port-mappings"].reverse() + if reverse_netbox: + port.mappings.reverse() + serialized = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + + assert _repo_supersedes(repo, serialized), "unchanged mappings must not trigger export" + repo["port-mappings"].append(dict(repo["port-mappings"][0])) + assert not _repo_supersedes(repo, serialized), "duplicate mappings must keep their multiplicity" + repo["port-mappings"].pop() + port.mappings.append(port.mappings[0]) + duplicated = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, duplicated), "NetBox duplicates must keep their multiplicity" + port.mappings.pop() + port.mappings[0].rear_port_position = 4 + changed = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, changed) + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -1695,10 +1762,42 @@ def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): - """On 4.7 an absent relation means NetBox cleared it, so it must not be resurrected.""" - repo = {"model": "MX304", "module-bays": [{"name": "RE0", "module_bay_types": ["MX304-RE"]}]} - serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0"}]} - - written = self._write(tmp_path, self._item(repo, serialized), supported=True) + """An export selected for a changed description must retain the supported server's answer.""" + from core.graphql_client import DotDict + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +description: old +module-bays: + - {name: Slot 0, position: '0', module_bay_types: [Control]} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme", slug="acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + description="new", + front_image=None, + rear_image=None, + last_updated="2026-01-01T00:00:00Z", + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = True + items = exporter._determine_export_set_for_device_types( + [record], {("acme", "acme-chassis"): repo}, {1: {"module_bay_templates": [bay]}} + ) + assert len(items) == 1 + assert items[0].reason == "differs" + exporter._write_export_items(items, {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Acme" / "Chassis.yaml").read_text()) + assert written["description"] == "new" + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index fdbbf119..95e9452d 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -488,7 +488,7 @@ def test_components_sorted_by_name(self): components = {1: {"interface_templates": [iface_z, iface_a]}} result = serialize_device_type(record, components) names = [i["name"] for i in result["interfaces"]] - assert names == sorted(names) + assert names == ["eth0", "eth9"] class TestRelationSerialization: @@ -635,6 +635,29 @@ def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): ("FP3", 3), ] + def test_mappings_sort_by_numeric_positions_then_rear_port_name(self): + import yaml + + fp = self._front_port( + "FP1", + [ + self._mapping("RP2", "10.0", "2.0"), + self._mapping("RP2", 2, 2), + self._mapping("RP1", "2.0", 2), + self._mapping("RP1", None, None), + ], + positions=2, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == yaml.safe_load(""" +- {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} +- {front_port: FP1, front_port_position: 2, rear_port: RP1, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 10} +""") + def test_a_pre_45_server_still_exports_its_mappings(self): """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" from types import SimpleNamespace diff --git a/tests/test_repo.py b/tests/test_repo.py index 45c819f2..09714d48 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,6 +1,7 @@ import os import pytest +import yaml from unittest.mock import MagicMock, call, mock_open, patch from git import Actor, Repo as GitRepo, exc as git_exc from core.repo import ( @@ -1104,6 +1105,44 @@ def test_null_stanza_is_deleted(self): assert err is None assert "port-mappings" not in data + @pytest.mark.parametrize("stanza", ["", "port-mappings:", "port-mappings: []"]) + def test_only_an_explicit_list_manages_front_port_mappings(self, stanza): + data = yaml.safe_load(f""" +front-ports: + - name: FP1 + type: 8p8c + - name: FP2 + type: 8p8c +{stanza} +""") + + assert normalize_port_mappings(data) is None + assert "port-mappings" not in data + for port in data["front-ports"]: + if stanza == "port-mappings: []": + assert port["_mappings"] == [] + else: + assert "_mappings" not in port + + @pytest.mark.parametrize( + ("stanza", "expected"), + [ + ("RP1", "Error: port-mappings must be a list: 'RP1'"), + ("{FP1: RP1}", "Error: port-mappings must be a list: {'FP1': 'RP1'}"), + ("[RP1]", "Error: port-mappings entry must be a mapping: 'RP1'"), + ], + ) + def test_malformed_stanza_returns_an_error(self, stanza, expected): + data = yaml.safe_load(f""" +front-ports: + - {{name: FP1, type: 8p8c}} +rear-ports: + - {{name: RP1, type: 8p8c}} +port-mappings: {stanza} +""") + + assert normalize_port_mappings(data) == expected + def test_empty_stanza_no_front_ports_still_deleted(self): """Empty port-mappings stanza with no front-ports is cleaned up (not silently skipped).""" data = { @@ -1843,6 +1882,59 @@ def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): class TestAStanzaThatDoesNotListAFrontPort: """A stanza speaks for the whole file, so a port it omits has no mapping.""" + def test_a_nonempty_stanza_clears_an_omitted_front_port_mapping(self): + from types import SimpleNamespace + from core.change_detector import ChangeDetector + + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP1, rear_port: RP1} +""") + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [] + + existing = SimpleNamespace( + name="FP2", + _mappings_canonical=[{"rear_port_name": "RP2", "front_port_position": 1, "rear_port_position": 1}], + ) + detector = ChangeDetector(SimpleNamespace(), LogHandler(False)) + changes = detector._compare_component_properties( + data["front-ports"][1], existing, ["_mappings"], comp_type="front-ports" + ) + assert len(changes) == 1 + assert changes[0].property_name == "_mappings" + assert changes[0].old_value == {("RP2", 1, 1)} + assert changes[0].new_value == set() + + def test_a_half_migrated_file_errors_before_assigning_mappings(self): + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c, rear_port: RP1} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP2, rear_port: RP2} +""") + + assert normalize_port_mappings(data) == ( + "Error: front port 'FP1' declares an inline rear_port but the port-mappings " + "stanza does not list it; the stanza is authoritative, so add 'FP1' to it " + "or remove the inline rear_port keys" + ) + assert all("_mappings" not in port for port in data["front-ports"]) + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): """The old wording blamed a conflict against a stanza that never mentioned the port.""" from core.repo import normalize_port_mappings From c2c16cf813bb4c93fdac29c144e193a5834f6fee Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 11 Sep 2026 20:36:38 +0200 Subject: [PATCH 19/19] fix(mappings): clearing a mapping now obeys --remove-components With the flag off the importer tells the user "will not remove components from existing models". Clearing a front-to-rear linkage removes data, but it reaches NetBox as a `_mappings` property change on an existing component, not as a COMPONENT_REMOVED, so it never passed the gate and applied under a plain --update. `_without_gated_mapping_clears` drops a `_mappings` change whose new value is a strict subset of the old one, which is the case where mappings only go away. Repointing a front port to another rear port still applies, and a component that also carries other property changes keeps them. The filter runs before `_count_actionable_component_changes` at both call sites, so the reported count and the applied changes cannot disagree. Affected ports are named once per type, with the same hint the startup notice uses. This closes a gap that predates the omission change: `port-mappings: []` could already clear every front port without the flag. `update_components` still clears when handed such a change, which is what the two existing write-boundary tests cover. The gate sits above it, with the run configuration, rather than at the write. --- core/netbox_api.py | 47 ++++++++++++++++++++-- tests/test_netbox_api.py | 84 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/core/netbox_api.py b/core/netbox_api.py index 544ce538..3321d45d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1,6 +1,7 @@ """NetBox REST and GraphQL API client for importing device and module type libraries.""" from collections import Counter +from dataclasses import replace from contextlib import contextmanager from functools import lru_cache import hashlib @@ -434,6 +435,42 @@ def _image_dir_for_yaml(src_file: str, src_segment: str, dst_segment: str) -> "P # from pynetbox import RequestError as APIRequestError +def _is_mapping_removal(prop_change): + """Return True when a ``_mappings`` change only takes mappings away.""" + return ( + prop_change.property_name == "_mappings" + and isinstance(prop_change.old_value, (set, frozenset)) + and isinstance(prop_change.new_value, (set, frozenset)) + and prop_change.new_value < prop_change.old_value + ) + + +def _without_gated_mapping_clears(changes, remove_components, handle=None): + """Drop mapping clears unless removal is enabled. + + Clearing a front-to-rear linkage deletes data, but it reaches NetBox as a property + change rather than a COMPONENT_REMOVED, so it would otherwise ignore the flag and + contradict the "will not remove components" notice. Filtering here rather than at the + write boundary keeps the actionable count and the applied changes in agreement. + """ + if remove_components: + return changes + kept, gated = [], [] + for change in changes: + remaining = [pc for pc in change.property_changes if not _is_mapping_removal(pc)] + if len(remaining) == len(change.property_changes): + kept.append(change) + continue + gated.append(change.component_name) + if remaining: + kept.append(replace(change, property_changes=remaining)) + if gated and handle is not None: + handle.log( + f"Kept existing port mappings on {sorted(gated)}; use --remove-components with --update to clear them." + ) + return kept + + def _count_actionable_component_changes(changes, remove_components): """Return the count of changes in *changes* that will issue an API call. @@ -1071,7 +1108,10 @@ def _handle_existing_device_type( # Apply component changes component_errors = [] if dt_change.component_changes: - actionable_count = _count_actionable_component_changes(dt_change.component_changes, remove_components) + component_changes = _without_gated_mapping_clears( + dt_change.component_changes, remove_components, self.handle + ) + actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1081,11 +1121,11 @@ def _handle_existing_device_type( self.device_types.update_components( device_type, dt.id, - dt_change.component_changes, + component_changes, parent_type="device", ) if remove_components: - self.device_types.remove_components(dt.id, dt_change.component_changes, parent_type="device") + self.device_types.remove_components(dt.id, component_changes, parent_type="device") after_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1754,6 +1794,7 @@ def _apply_module_type_component_updates( self.device_types.ensure_components_ready(manufacturer_slug=curr_mt["manufacturer"]["slug"]) identity = f"{module_type_res.manufacturer.name}/{module_type_res.model}" component_changes = self.change_detector._compare_components(curr_mt, module_type_res.id, parent_type="module") + component_changes = _without_gated_mapping_clears(component_changes, remove_components, self.handle) if component_changes: actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_updated = self.counter["components_updated"] diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index b2cdbabc..2423e106 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -1,5 +1,6 @@ import os import threading +from types import SimpleNamespace from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -7570,3 +7571,86 @@ def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_setti message = str(exc.value) assert "4.3" in message, f"{version}: the message must name the minimum" assert version in message, f"{version}: the message must name what was found" + + +class TestAMappingClearNeedsTheRemovalFlag: + """Clearing a front-port mapping removes data, so it obeys --remove-components. + + The tool tells the user "will not remove components from existing models" when the flag + is off. A mapping clear reaches NetBox as a COMPONENT_CHANGED property change, so it + bypassed that promise and cleared the linkage under a plain --update. + """ + + def _module_type_losing_a_mapping(self, nb): + """Record an existing FP1 mapped to RP1, and return YAML whose stanza omits it.""" + existing_module = MagicMock() + existing_module.id = 55 + existing_module.manufacturer.name = "Cisco" + existing_module.model = "CM-Map" + + existing_fp = SimpleNamespace( + name="FP1", + _mappings_canonical=[{"rear_port_name": "RP1", "front_port_position": 1, "rear_port_position": 1}], + _mappings_m2m=True, + ) + nb.device_types.components.record("front_port_templates", "module", 55, {"FP1": existing_fp}) + _mark_cache_ready(nb.device_types) + + curr_mt = { + "manufacturer": {"slug": "cisco"}, + "model": "CM-Map", + "slug": "cm-map", + # normalize_port_mappings assigns [] to a front port the stanza omits + "front-ports": [{"name": "FP1", "type": "8p8c", "_mappings": []}], + } + return {"cisco": {"CM-Map": existing_module}}, curr_mt + + def _mapping_clears_sent(self, nb): + """Return the _mappings property changes that reached update_components.""" + from core.change_detector import ChangeType + + sent = [] + for call_args in nb.device_types.update_components.call_args_list: + for change in call_args.args[2]: + if change.change_type is not ChangeType.COMPONENT_CHANGED: + continue + sent += [pc for pc in change.property_changes if pc.property_name == "_mappings"] + return sent + + @pytest.mark.parametrize("remove_components, expected", [(False, 0), (True, 1)]) + def test_the_flag_decides_whether_a_clear_reaches_netbox( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle, remove_components, expected + ): + """Without the flag the clear must never be sent; with it, it must.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=remove_components + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset()] * expected + + def test_a_changed_mapping_still_applies_without_the_flag( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle + ): + """Only removal is gated. Repointing FP1 to another rear port is an ordinary update.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + curr_mt["front-ports"][0]["_mappings"] = [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=False + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset({("RP2", 1, 1)})]