diff --git a/core/change_detector.py b/core/change_detector.py index 24fbd214..5e2ecf78 100644 --- a/core/change_detector.py +++ b/core/change_detector.py @@ -10,8 +10,8 @@ from typing import Any, List, Optional from enum import Enum -from core.component_registry import COMPONENT_TYPES -from core.normalization import normalize_values +from core.component_registry import BY_YAML_KEY, COMPONENT_TYPES +from core.normalization import is_explicit_list, normalize_values from core.formatting import log_property_diffs from core.schema_reader import load_properties_for_type @@ -26,6 +26,114 @@ class ChangeType(Enum): COMPONENT_REMOVED = "component_removed" +def _manufacturer_slug_of(yaml_data): + """Return the owning manufacturer's slug from a parsed definition. + + ``core.repo`` rewrites every YAML manufacturer to ``{"slug": ...}`` before the importer + sees it, so that is the usual shape here. + """ + manufacturer = (yaml_data or {}).get("manufacturer") + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or "" + return manufacturer or "" + + +def _relation_properties(comp_type): + """Return the relation field names the registry declares for *comp_type*.""" + component = BY_YAML_KEY.get(comp_type) + return component.relations if component else () + + +def _is_relation_list(value): + """Return True when *value* is a list of non-empty strings, the only shape a reference takes. + + Blank is checked after stripping, matching the catalog: accepting " " here only defers + the rejection to the write path, where it skips the whole component's update. + """ + return is_explicit_list(value) and all(isinstance(item, str) and item.strip() for item in value) + + +def _relation_change(prop, yaml_comp, netbox_comp, catalog=None, manufacturer=None, handle=None): + """Return a PropertyChange when a relation differs, or None when there is nothing to do. + + A relation is an unordered set of related objects. Where the catalog is available the + comparison is between *identities*, ``(manufacturer slug, slug)``, because two + manufacturers may define the same class name and a bay holding the wrong one reads as + equal on names alone. Without a catalog it falls back to comparing names, which is + what a caller that has not wired one can still do. + + An omitted key leaves the relation unmanaged and an empty list clears it, but a bare + ``module_bay_types:`` parses as None and a malformed entry is not a name: those are left + unmanaged, because reading them as empty would clear a restriction nobody removed. A + field the query did not return is skipped for the same reason. + """ + if prop not in yaml_comp: + return None + declared = yaml_comp.get(prop) + if not _is_relation_list(declared): + if handle is not None: + handle.log( + f"Ignored {prop} on {yaml_comp.get('name', 'Unknown')!r}: expected a list of names, got {declared!r}" + ) + return None + netbox_value = getattr(netbox_comp, prop, _MISSING) + if netbox_value is _MISSING: + return None + + if catalog is not None and manufacturer: + from core.module_bay_types import ModuleBayTypeError + + try: + wanted = catalog.identities_for(manufacturer, declared) + except ModuleBayTypeError: + # Reported as a change so the write path sees it; "no change" is silent. + return PropertyChange( + property_name=prop, + old_value=sorted(_relation_names(netbox_value)), + new_value=sorted(set(declared)), + ) + current = _relation_identities(netbox_value) + if current is not None and wanted == current: + return None + if current is not None: + return PropertyChange( + property_name=prop, old_value=sorted(_relation_names(netbox_value)), new_value=sorted(set(declared)) + ) + + yaml_names = frozenset(declared) + netbox_names = frozenset(_relation_names(netbox_value)) + if yaml_names == netbox_names: + return None + return PropertyChange(property_name=prop, old_value=sorted(netbox_names), new_value=sorted(yaml_names)) + + +def _relation_identities(value): + """Return {(manufacturer slug, slug)} for a relation, or None if the shape lacks them. + + A read path that returns only ``id`` and ``name`` cannot answer the scope question, so + the caller falls back to comparing names rather than inventing an identity. + """ + identities = set() + for item in value or []: + slug = item.get("slug") if isinstance(item, dict) else getattr(item, "slug", None) + manufacturer = item.get("manufacturer") if isinstance(item, dict) else getattr(item, "manufacturer", None) + owner = manufacturer.get("slug") if isinstance(manufacturer, dict) else getattr(manufacturer, "slug", None) + if not slug or not owner: + return None + identities.add((owner, slug)) + return frozenset(identities) + + +def _relation_names(value): + """Return the ``name`` of each related object NetBox returned for a relation.""" + names = [] + for item in value or []: + name = item.get("name") if isinstance(item, dict) else getattr(item, "name", None) + if name is not None: + names.append(name) + return names + + @dataclass class PropertyChange: """Represents a single property change.""" @@ -339,7 +447,11 @@ def _compare_components( # Check for property changes on existing component existing = existing_components[comp_name] prop_changes = self._compare_component_properties( - yaml_comp, existing, component.compare_properties, comp_type=yaml_key + yaml_comp, + existing, + component.compare_properties, + comp_type=yaml_key, + manufacturer=_manufacturer_slug_of(yaml_data), ) if prop_changes: changes.append( @@ -353,14 +465,31 @@ def _compare_components( return changes + def _relation_catalog(self): + """Return the module bay type catalog, or None when this detector has no real one. + + Tests build the detector around a stand-in for DeviceTypes, and a stand-in cannot + answer what a reference means. Returning None there keeps the name comparison, + which is what those tests are asserting on. + """ + from core.module_bay_types import ModuleBayTypeCatalog + + catalog = getattr(self.device_types, "module_bay_type_catalog", None) + return catalog if isinstance(catalog, ModuleBayTypeCatalog) else None + def _compare_component_properties( self, yaml_comp: dict, netbox_comp, properties: List[str], comp_type: str = "", + manufacturer: str = "", ) -> List[PropertyChange]: - """Compare properties between YAML and NetBox component.""" + """Compare properties between YAML and NetBox component. + + *manufacturer* is the owning manufacturer's slug, used to resolve a relation + reference to the object it means rather than the name it is written as. + """ changes = [] for prop in properties: @@ -389,7 +518,11 @@ def _compare_component_properties( # GraphQL response lacked both mappings and rear_port_position; # treat as unmanaged to avoid a false COMPONENT_CHANGED. continue - has_names = any(m.get("rear_port_name") is not None for m in canonical) + # Compare by name whenever one is available: an empty M2M list has no name to + # infer from but still needs the named path, and the pre-4.5 query returns one. + has_names = bool(getattr(netbox_comp, "_mappings_m2m", False)) or any( + m.get("rear_port_name") is not None for m in canonical + ) if has_names: # NetBox >= 4.5: compare with rear port names netbox_set: frozenset = frozenset( @@ -426,6 +559,14 @@ def _compare_component_properties( ) continue + if prop in _relation_properties(comp_type): + relation_change = _relation_change( + prop, yaml_comp, netbox_comp, self._relation_catalog(), manufacturer, self.handle + ) + if relation_change is not None: + changes.append(relation_change) + continue + # Only compare properties explicitly present in the YAML component; # an omitted property means the YAML doesn't manage it (absent key != removal). if prop not in yaml_comp: diff --git a/core/compat.py b/core/compat.py index 62c33bf1..8ebcbf03 100644 --- a/core/compat.py +++ b/core/compat.py @@ -1,61 +1,27 @@ """NetBox version-compatibility helpers. -Centralises filter parameter names that changed between NetBox releases so -that every caller uses the same logic and drift is impossible. - -NetBox 4.1 renamed several filter keys on the DCIM endpoints: - - devicetype_id → device_type_id - moduletype_id → module_type_id - -Any code that constructs endpoint filter kwargs should call the helpers -here rather than inlining ``"device_type_id" if new_filters else "devicetype_id"``. +Centralises the version line each feature sits behind so that every caller uses the +same logic and drift is impossible. """ from __future__ import annotations +import re -def device_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for device-type component queries. +# Selecting or sending the relation below this release fails the whole query. +MODULE_BAY_TYPE_MINIMUM_VERSION = (4, 7) - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"device_type_id"``); - ``False`` for older releases (returns ``"devicetype_id"``). - """ - return "device_type_id" if new_filters else "devicetype_id" +def parse_netbox_version(version) -> tuple[int, int]: + """Return ``(major, minor)`` from a NetBox version string. -def module_type_filter_key(new_filters: bool) -> str: - """Return the correct filter parameter name for module-type component queries. - - Args: - new_filters: ``True`` for NetBox ≥ 4.1 (returns ``"module_type_id"``); - ``False`` for older releases (returns ``"moduletype_id"``). + Padded to two parts so a single-component string cannot raise, and tolerant of the + suffixes NetBox ships ("4.7.0-beta2"). """ - return "module_type_id" if new_filters else "moduletype_id" - - -def device_type_filter_kwargs(device_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a device type. + raw = [int(re.sub(r"\D.*", "", part.strip()) or "0") for part in str(version).split(".")] + return tuple((raw + [0, 0])[:2]) # type: ignore[return-value] - Args: - device_type_id: NetBox ID of the device type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {device_type_filter_key(new_filters): device_type_id} - - -def module_type_filter_kwargs(module_type_id: int, *, new_filters: bool) -> dict: - """Return filter kwargs for querying components of a module type. - - Args: - module_type_id: NetBox ID of the module type. - new_filters: ``True`` for NetBox ≥ 4.1; ``False`` for older releases. - - Returns: - A dict suitable for unpacking into ``endpoint.filter(**kwargs)``. - """ - return {module_type_filter_key(new_filters): module_type_id} +def supports_module_bay_types(version) -> bool: + """Return True when this NetBox release exposes the module bay type relation.""" + return parse_netbox_version(version) >= MODULE_BAY_TYPE_MINIMUM_VERSION diff --git a/core/component_cache.py b/core/component_cache.py index d8dc4a22..f314ecf7 100644 --- a/core/component_cache.py +++ b/core/component_cache.py @@ -14,12 +14,6 @@ import threading from typing import Any -from core.compat import ( - device_type_filter_key, - device_type_filter_kwargs, - module_type_filter_key, - module_type_filter_kwargs, -) from core.component_registry import COMPONENT_TYPES from core.graphql_client import GraphQLCountMismatchError, GraphQLSchemaError @@ -118,14 +112,13 @@ class ComponentCache: filter, which happens for types created during this run and after an invalidation. """ - def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_record=None): + def __init__(self, netbox, graphql, handle, max_threads, wrap_record=None): """Build a cache over the *netbox* REST client and the *graphql* client. Args: netbox: pynetbox API object, used for REST fallbacks and count checks. graphql: GraphQL client used for the bulk fetch. handle: Log handler. - new_filters (bool): Whether this NetBox takes the newer filter parameter names. max_threads (int): Upper bound on concurrent endpoint fetches. wrap_record (callable | None): Applied to every front-port record, so change detection sees one mappings shape across NetBox versions. @@ -133,7 +126,6 @@ def __init__(self, netbox, graphql, handle, new_filters, max_threads, wrap_recor self.netbox = netbox self.graphql = graphql self.handle = handle - self.new_filters = new_filters self.max_threads = max_threads self._wrap_record = wrap_record or (lambda record: record) @@ -313,9 +305,9 @@ def get(self, endpoint_name, parent_type, parent_id, endpoint): return cached[key] if parent_type == "device": - filter_kwargs = device_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"device_type_id": parent_id} else: - filter_kwargs = module_type_filter_kwargs(parent_id, new_filters=self.new_filters) + filter_kwargs = {"module_type_id": parent_id} result = {item.name: item for item in endpoint.filter(**filter_kwargs)} self.record(endpoint_name, parent_type, parent_id, result) return result @@ -479,8 +471,6 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc Raises: GraphQLCountMismatchError: When an endpoint holds fewer records than REST reports. """ - dt_filter_key = device_type_filter_key(self.new_filters) - mt_filter_key = module_type_filter_key(self.new_filters) dt_ids = list(device_type_ids) mt_ids = list(module_type_ids) @@ -495,9 +485,9 @@ def _check_counts_against_rest(self, device_type_ids, module_type_ids, vendor_sc rest_endpoint = getattr(self.netbox.dcim, endpoint_name) rest_count = 0 if dt_ids: - rest_count += self._rest_count(rest_endpoint, dt_filter_key, dt_ids) + rest_count += self._rest_count(rest_endpoint, "device_type_id", dt_ids) if mt_ids and component.module_types: - rest_count += self._rest_count(rest_endpoint, mt_filter_key, mt_ids) + rest_count += self._rest_count(rest_endpoint, "module_type_id", mt_ids) if cached_count != rest_count: if vendor_scope_valid: diff --git a/core/component_registry.py b/core/component_registry.py index ed991052..8fb4e474 100644 --- a/core/component_registry.py +++ b/core/component_registry.py @@ -17,6 +17,17 @@ LINK_POWER_PORT = "power_port" LINK_REAR_PORTS = "rear_ports" +# Fields holding names that must become NetBox ids before a POST. +RELATION_MODULE_BAY_TYPES = "module_bay_types" + +# Relation fields a module type carries itself, rather than through one of its components. +MODULE_TYPE_RELATIONS = (RELATION_MODULE_BAY_TYPES,) + + +def relation_selection(name): + """Return the GraphQL fields that identify a related object.""" + return f"{name} {{ id name slug manufacturer {{ slug }} }}" + @dataclass(frozen=True) class ComponentType: @@ -27,19 +38,29 @@ class ComponentType: label: str fields: tuple[str, ...] module_types: bool = True + relations: tuple[str, ...] = field(default_factory=tuple) graphql_extra: tuple[str, ...] = field(default_factory=tuple) compare_extra: tuple[str, ...] = field(default_factory=tuple) link: Optional[str] = None @property def graphql_fields(self): - """Fields to select in a GraphQL query, including the id every consumer needs.""" - return ["id", *self.fields, *self.graphql_extra] + """Fields to select in a GraphQL query, including the id every consumer needs. + + A relation is a list of related objects, so it is selected by name rather than + read as a scalar. + """ + return ["id", *self.fields, *self.graphql_extra, *self.graphql_relation_fields] + + @property + def graphql_relation_fields(self): + """GraphQL selections for this row's relations, one nested block per relation.""" + return [relation_selection(name) for name in self.relations] @property def compare_properties(self): """Properties change detection compares between YAML and NetBox.""" - return [*self.fields, *self.compare_extra] + return [*self.fields, *self.compare_extra, *self.relations] @property def list_key(self): @@ -102,7 +123,8 @@ def create_label(self, parent_type): yaml_key="front-ports", endpoint="front_port_templates", label="Front Port", - fields=("name", "type", "label", "description", "color"), + # positions arrived with the 4.5 mapping model; the query drops it on older servers. + fields=("name", "type", "label", "description", "color", "positions"), graphql_extra=("mappings { id front_port_position rear_port_position rear_port { id name } }",), compare_extra=("_mappings",), link=LINK_REAR_PORTS, @@ -121,6 +143,7 @@ def create_label(self, parent_type): endpoint="module_bay_templates", label="Module Bay", fields=("name", "position", "label", "description"), + relations=(RELATION_MODULE_BAY_TYPES,), ), ) diff --git a/core/config.py b/core/config.py index d37778e6..fd4dd6c8 100644 --- a/core/config.py +++ b/core/config.py @@ -1,9 +1,11 @@ """Resolution of one run's configuration from the command line and the environment.""" +import ipaddress import os import re from argparse import ArgumentParser from dataclasses import dataclass, field +from urllib.parse import urlparse from dotenv import load_dotenv @@ -22,6 +24,27 @@ _DEFAULT_REPO_PATH = f"{os.path.dirname(os.path.dirname(os.path.realpath(__file__)))}/repo" +def _sends_token_in_cleartext(url): + """Return True when *url* would send the API token over plain HTTP off this host.""" + # requests treats a backslash in the authority as a delimiter and urlparse does not, so + # "http://10.0.0.1\\@localhost" would otherwise look like loopback and skip the notice. + try: + parsed = urlparse(str(url or "").strip().replace("\\", "/")) + except ValueError: + # urlparse rejects some authorities outright. An authority this tool cannot read is + # also one it cannot clear as loopback, and a notice must never abort the run. + return True + if parsed.scheme != "http": + return False + host = (parsed.hostname or "").casefold() + if host in {"", "localhost"}: + return False + try: + return not ipaddress.ip_address(host).is_loopback + except ValueError: + return True + + def is_local_repo_url(url): """Return True when *url* is the sentinel that turns off every git operation.""" return str(url or "").strip().casefold() == LOCAL_REPO_URL @@ -271,6 +294,12 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: # Only the environment can reach here: an explicit --slugs is rejected above. notices.append("Ignoring SLUGS from the environment: --export-diff does not filter by slug.") slugs = () + netbox_url = _text(env, "NETBOX_URL") + if _sends_token_in_cleartext(netbox_url): + notices.append( + "NETBOX_URL uses http:// on a remote host, so the API token is sent in cleartext. " + "Use https:// unless NetBox is on this machine." + ) if is_local_repo_url(args.url) and args.branch != DEFAULT_REPO_BRANCH: notices.append( f"Ignoring REPO_BRANCH={args.branch}: REPO_URL={LOCAL_REPO_URL} reads REPO_PATH as it stands " @@ -278,7 +307,7 @@ def resolve_run_config(argv=None, env=None) -> RunConfig: ) return RunConfig( - netbox_url=_text(env, "NETBOX_URL"), + netbox_url=netbox_url, netbox_token=_text(env, "NETBOX_TOKEN"), ignore_ssl_errors=(_text(env, "IGNORE_SSL_ERRORS", "False") or "False").casefold() in {"true", "1", "yes"}, graphql_page_size=_positive_int(env, "GRAPHQL_PAGE_SIZE", DEFAULT_GRAPHQL_PAGE_SIZE), diff --git a/core/export.py b/core/export.py index c0dd57ac..9dd48507 100644 --- a/core/export.py +++ b/core/export.py @@ -14,6 +14,7 @@ import requests import yaml +from core.component_registry import COMPONENT_TYPES, MODULE_TYPE_RELATIONS from core.export_manifest import ( is_entry_fresh, load_manifest, @@ -23,6 +24,7 @@ from core.graphql_client import GraphQLError, NetBoxGraphQLClient from core.nb_serializer import ( COMPONENT_ENDPOINT_NAMES, + module_bays_missing_position, serialize_device_type, serialize_module_type, serialize_rack_type, @@ -140,6 +142,40 @@ def _yaml_equal(a: dict, b: dict) -> bool: return _normalize_for_compare(a) == _normalize_for_compare(b) +def _relation_names() -> set: + """Every relation key the registry knows, so this does not become a second source.""" + return {relation for component in COMPONENT_TYPES for relation in component.relations} | set(MODULE_TYPE_RELATIONS) + + +def _carry_unqueried_relations(repo_yaml: dict, serialized: dict) -> dict: + """Return *serialized* with relations the server never answered for taken from the repo. + + Below NetBox 4.7 module_bay_types is not queried at all, so its absence means "not asked", + not "cleared". Writing NetBox's answer as it stands would delete the restriction from + every definition the export touches for any other reason. + """ + relations = _relation_names() + result = dict(serialized) + for relation in relations: + if relation in repo_yaml and relation not in result: + result[relation] = repo_yaml[relation] + for key, repo_value in repo_yaml.items(): + if not isinstance(repo_value, list) or not isinstance(result.get(key), list): + continue + by_name = {e.get("name"): e for e in repo_value if isinstance(e, dict)} + merged = [] + for entry in result[key]: + if isinstance(entry, dict): + repo_entry = by_name.get(entry.get("name")) + if isinstance(repo_entry, dict): + carried = {r: repo_entry[r] for r in relations if r in repo_entry and r not in entry} + if carried: + entry = {**entry, **carried} + merged.append(entry) + result[key] = merged + return result + + def _repo_supersedes(repo_yaml: dict, nb_serialized: dict) -> bool: """Return True when *repo_yaml* already contains every field NetBox would write. @@ -161,8 +197,24 @@ def _norm_mfr(d: dict) -> dict: return d return {**d, "manufacturer": _canon_mfr_slug(d["manufacturer"])} - nrepo = _normalize_for_compare(_norm_mfr(repo_yaml)) - nnb = _normalize_for_compare(_norm_mfr(nb_serialized)) + # The serializer always writes the schema-required positions; an entry that omits it + # means the default, so filling it here keeps those definitions from reading as differing. + def _default_positions(d: dict) -> dict: + ports = d.get("front-ports") + if not isinstance(ports, list): + return d + filled = [{"positions": 1, **p} if isinstance(p, dict) else p for p in ports] + return {**d, "front-ports": filled} + + def _sort_port_mappings(d: dict) -> dict: + mappings = d.get("port-mappings") + if not isinstance(mappings, list) or not all(isinstance(mapping, dict) for mapping in mappings): + return d + fields = ("front_port", "front_port_position", "rear_port", "rear_port_position") + return {**d, "port-mappings": sorted(mappings, key=lambda m: tuple(str(m.get(field)) for field in fields))} + + nrepo = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(repo_yaml)))) + nnb = _sort_port_mappings(_normalize_for_compare(_default_positions(_norm_mfr(nb_serialized)))) return _is_subset(nnb, nrepo) @@ -241,6 +293,9 @@ def run(self, progress=None) -> None: ) self.handle.log(f"Export-diff: fetching NetBox device/module/rack types{scope}") + # Decided before the first query: the selection depends on the answer. + self.graphql.detect_module_bay_type_support() + # ── Fetch all types from NetBox ────────────────────────────────────── by_model, by_slug = self.graphql.get_device_types(manufacturer_slugs=self.vendor_slugs) all_mt = self.graphql.get_module_types(manufacturer_slugs=self.vendor_slugs) @@ -443,15 +498,15 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: # top-level fields (e.g. comments, profile) that NetBox does not return # in its serialized output. Component lists are left as NB authoritative. to_write = item.serialized + if item.repo_yaml and not self.graphql.supports_module_bay_types: + to_write = _carry_unqueried_relations(item.repo_yaml, to_write) if item.reason == "differs" and item.repo_yaml: # Only preserve scalar/metadata repo fields not present in the NB output. # Exclude list-valued keys (component sections such as interfaces, power-ports, # console-ports, etc.) so that NB remains authoritative for all components. - extra = { - k: v for k, v in item.repo_yaml.items() if k not in item.serialized and not isinstance(v, list) - } + extra = {k: v for k, v in item.repo_yaml.items() if k not in to_write and not isinstance(v, list)} if extra: - to_write = {**item.serialized, **extra} + to_write = {**to_write, **extra} written = self._write_yaml(dest, to_write) if not written: skipped_overwrite += 1 @@ -463,6 +518,12 @@ def _write_export_items(self, items, manifest, manifest_path, progress) -> None: continue written_count += 1 + bays = module_bays_missing_position(to_write) + if bays: + self.handle.log( + f"[yellow]{item.mfr_name}/{item.filename}: module bay(s) {', '.join(bays)} have " + f"no position, which the library schema requires[/yellow]" + ) images_ok = self._download_type_images(item) if images_ok: update_entry(manifest, f"{item.kind}s", item.manifest_key, item.nb_record.last_updated) @@ -624,13 +685,7 @@ def _fetch_vendor_components(self, mfr_slug: str) -> tuple: def _fetch_one(endpoint_name): if not getattr(_thread_local, "graphql", None): - client = NetBoxGraphQLClient( - self.graphql.url, - self.graphql.token, - self.graphql.ignore_ssl, - self.graphql.handle, - self.graphql.DEFAULT_PAGE_SIZE, - ) + client = self.graphql.clone() _thread_local.graphql = client with _clients_lock: _clients.append(client) diff --git a/core/export_manifest.py b/core/export_manifest.py index b959ddeb..d9ae91c0 100644 --- a/core/export_manifest.py +++ b/core/export_manifest.py @@ -11,6 +11,11 @@ _EMPTY: dict = {"device-types": {}, "module-types": {}, "rack-types": {}} +# Bump whenever the serialized output shape changes (a new stanza, a field that starts or +# stops being written). An unchanged NetBox record has the same last_updated forever, so +# without this an old manifest skips the very types the new shape was added for. +EXPORT_SCHEMA_REVISION = 2 + def load_manifest(path: Path) -> dict: """Load manifest from *path*. Returns an empty manifest on any error.""" @@ -33,13 +38,15 @@ def save_manifest(path: Path, data: dict) -> None: def is_entry_fresh(manifest: dict, kind: str, key: str, last_updated: str) -> bool: - """Return True if the manifest entry for *key* matches *last_updated*.""" + """Return True if *key* was written by this exporter and the record has not changed.""" section = manifest.get(kind) if not isinstance(section, dict): return False entry = section.get(key) if not isinstance(entry, dict): return False + if entry.get("schema") != EXPORT_SCHEMA_REVISION: + return False return entry.get("last_updated") == last_updated @@ -48,4 +55,4 @@ def update_entry(manifest: dict, kind: str, key: str, last_updated: str) -> None section = manifest.get(kind) if not isinstance(section, dict): manifest[kind] = {} - manifest[kind][key] = {"last_updated": last_updated} + manifest[kind][key] = {"last_updated": last_updated, "schema": EXPORT_SCHEMA_REVISION} diff --git a/core/graphql_client.py b/core/graphql_client.py index 70baabfd..a188ea6e 100644 --- a/core/graphql_client.py +++ b/core/graphql_client.py @@ -5,13 +5,15 @@ compatible with the existing REST-based code in ``netbox_api.py``. """ +import copy import threading import time from collections.abc import Sequence import requests -from core.component_registry import BY_ENDPOINT +from core.compat import supports_module_bay_types +from core.component_registry import BY_ENDPOINT, MODULE_TYPE_RELATIONS, relation_selection # Module-level dedup: tracks (url, requested_page_size) pairs that have already # emitted the page-size clamping warning so the message appears at most once @@ -102,6 +104,9 @@ def _to_dotdict(obj): # connections, which is transient during a long paginated run. _RETRYABLE_STATUSES = {429, 500, 502, 503, 504} +# One small metadata read; a run that cannot get it fails at the next query anyway. +_STATUS_TIMEOUT_SECONDS = 30 + def _response_body_detail(response): """Return the response body as a suffix for an error message, truncated and stripped.""" @@ -136,7 +141,7 @@ class NetBoxGraphQLClient: or raise the server's ``MAX_PAGE_SIZE`` setting to match. """ - def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): + def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000, supports_module_bay_types=False): """Store connection parameters for later use in :meth:`query`. Args: @@ -148,6 +153,8 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): ``print`` when not provided. page_size: Default number of items per GraphQL page (default: 5 000). + supports_module_bay_types: True when NetBox is >= 4.7 and its schema + exposes the module bay type relation. """ self.DEFAULT_PAGE_SIZE = page_size self.url = url.rstrip("/") @@ -155,22 +162,28 @@ def __init__(self, url, token, ignore_ssl=False, handle=None, page_size=5000): self.token = token self.ignore_ssl = ignore_ssl self._handle = handle + self.supports_module_bay_types = supports_module_bay_types + + self._session = self._new_session() - self._session = requests.Session() + def _new_session(self): + """Return an HTTP session carrying this client's auth and TLS settings.""" + session = requests.Session() # v2 tokens start with "nbt_" prefix (format: nbt_.); # v1 tokens are plain 40-char hex strings using legacy Token auth. auth_scheme = "Bearer" if self.token.startswith("nbt_") else "Token" - self._session.headers.update( + session.headers.update( { "Authorization": f"{auth_scheme} {self.token}", "Content-Type": "application/json", } ) - self._session.verify = not self.ignore_ssl + session.verify = not self.ignore_ssl if self.ignore_ssl: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + return session @property def handle(self): @@ -178,8 +191,40 @@ def handle(self): return self._handle def clone(self): - """Return an independent client with the same connection settings.""" - return type(self)(self.url, self.token, self.ignore_ssl, self.handle, self.DEFAULT_PAGE_SIZE) + """Return an independent client with the same settings and its own HTTP session. + + Copied rather than reconstructed, so a setting added to ``__init__`` travels with + the clone instead of quietly reverting to its default. The session is the one + thing a worker must not share, so it is the one thing rebuilt here. + """ + clone = copy.copy(self) + clone._session = self._new_session() + return clone + + def detect_module_bay_type_support(self): + """Ask NetBox for its version and record whether the relation can be selected. + + The importer learns this from its pynetbox client; the export entry point has no + such client, so it asks here. Both sides decide from core.compat, so the line + cannot drift between them. + """ + status_url = f"{self.url}/api/status/" + try: + response = self._session.get(status_url, timeout=_STATUS_TIMEOUT_SECONDS) + response.raise_for_status() + payload = response.json() + except requests.RequestException as exc: + raise GraphQLError(f"Could not read {status_url}: {exc}{_response_body_detail(exc.response)}") from exc + except ValueError as exc: + # A proxy error page answers 200 with HTML, so the body is not JSON. + raise GraphQLError(f"Invalid JSON from {status_url}: {exc}") from exc + # Reading an absent version as "" would answer "unsupported" for a 4.7 server and + # export without the relation, so demand the field rather than defaulting it. + version = payload.get("netbox-version") if isinstance(payload, dict) else None + if not isinstance(version, str) or not version.strip(): + raise GraphQLError(f"No netbox-version in the status payload from {status_url}: {payload!r}") + self.supports_module_bay_types = supports_module_bay_types(version) + return self.supports_module_bay_types def close(self): """Close the underlying HTTP session.""" @@ -477,6 +522,11 @@ def get_module_types(self, manufacturer_slugs=None): sequence of non-blank strings. """ var_decl, filter_fragment, extra_vars = self._build_manufacturer_filter(manufacturer_slugs) + module_bay_type_selection = ( + "".join(f"{relation_selection(name)}\n " for name in MODULE_TYPE_RELATIONS) + if self.supports_module_bay_types + else "" + ) query = f""" query($pagination: OffsetPaginationInput{var_decl}) {{ @@ -490,7 +540,7 @@ def get_module_types(self, manufacturer_slugs=None): weight weight_unit last_updated - manufacturer {{ + {module_bay_type_selection}manufacturer {{ id name slug @@ -698,7 +748,7 @@ def get_module_type_image_details(self): def _front_port_field_variants(fields): """Yield successive field-list tiers for the front_port_templates fallback. - Tier 1: mappings block (NetBox 4.5+) + Tier 1: mappings block and positions (NetBox 4.5+) Tier 2: rear_port_position scalar (<4.5) Tier 3: neither (field removed entirely) """ @@ -707,7 +757,8 @@ def _front_port_field_variants(fields): for f in fields: if "mappings" in f: fallback.extend(["rear_port_position", "rear_port { id name }"]) - else: + elif f != "positions": + # positions arrived with the mapping model, so no pre-4.5 tier may ask for it. fallback.append(f) yield fallback stripped = [f for f in fallback if f != "rear_port_position" and "rear_port" not in f] @@ -801,6 +852,9 @@ def get_component_templates(self, endpoint_name, manufacturer_slug=None, on_page raise ValueError("manufacturer_slug must be None or a non-empty string") fields = component.graphql_fields + if not self.supports_module_bay_types: + # Selecting a field the server's schema lacks fails the whole query. + fields = [f for f in fields if f not in component.graphql_relation_fields] list_key = component.list_key parent_fields = "device_type { id }" diff --git a/core/import_run.py b/core/import_run.py index f81fd7bf..cdac353b 100644 --- a/core/import_run.py +++ b/core/import_run.py @@ -49,8 +49,6 @@ class RunSummary: """Snapshot of the result of one completed import run.""" counter: Counter - modules: bool - rack_types: bool outcome_counts: dict failure_lines: tuple duplicate_definitions: tuple @@ -66,8 +64,6 @@ def capture(cls, netbox, repo, started_at): """ return cls( counter=Counter(netbox.counter), - modules=netbox.modules, - rack_types=netbox.rack_types, outcome_counts=netbox.outcomes.summary_by_kind(), failure_lines=tuple(netbox.outcomes.render_failure_report()), duplicate_definitions=tuple(repo.duplicate_definitions), @@ -455,10 +451,6 @@ def _process_rack_types(config, netbox, handle, progress, rack_types, vendor_nam if not rack_types: return - if not netbox.rack_types: - handle.log("Rack types require NetBox >= 4.1. Skipping rack type import.") - return - handle.verbose_log(f"{len(rack_types)} Rack-Types Found") all_rack_types = netbox.get_existing_rack_types() @@ -521,21 +513,19 @@ def _log_run_summary(handle, summary): handle.log(f"{counter['components_removed']} components removed") handle.verbose_log(f"{counter['images']} images uploaded") handle.log(f"{counter['manufacturer']} manufacturers created") - if summary.modules: - handle.log(f"{counter['module_added']} modules created") - handle.log(f"{counter['module_updated']} modules updated") - module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) - if module_failed: - handle.log(f"{module_failed} modules failed to create or update") - module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) - if module_partial: - handle.log(f"{module_partial} modules partially updated") - if summary.rack_types: - handle.log(f"{counter['rack_type_added']} rack types created") - handle.log(f"{counter['rack_type_updated']} rack types updated") - rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) - if rack_failed: - handle.log(f"{rack_failed} rack types failed") + handle.log(f"{counter['module_added']} modules created") + handle.log(f"{counter['module_updated']} modules updated") + module_failed = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.FAILED) + if module_failed: + handle.log(f"{module_failed} modules failed to create or update") + module_partial = summary.outcome_count(EntityKind.MODULE_TYPE, Outcome.PARTIAL) + if module_partial: + handle.log(f"{module_partial} modules partially updated") + handle.log(f"{counter['rack_type_added']} rack types created") + handle.log(f"{counter['rack_type_updated']} rack types updated") + rack_failed = summary.outcome_count(EntityKind.RACK_TYPE, Outcome.FAILED) + if rack_failed: + handle.log(f"{rack_failed} rack types failed") for line in summary.failure_lines: handle.log(line) @@ -673,30 +663,24 @@ def plan_vendor(self, selection, vendor): self.repo, selection.devices_path, vendor["name"], self.config.slugs or [] ) - if self.netbox.modules: - module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None - if module_hint is not None and vendor["slug"] not in module_hint: - module_types = [] - else: - module_types = _parse_vendor_files( - self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] - ) - else: + module_hint = slug_resolved["module_vendors"] if slug_resolved is not None else None + if module_hint is not None and vendor["slug"] not in module_hint: module_types = [] - - if self.netbox.rack_types: - rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None - if rack_hint is not None and vendor["slug"] not in rack_hint: - rack_types = [] - else: - rack_types = _parse_vendor_files( - self.repo, - selection.racks_path, - vendor["name"], - self.config.slugs or [], - ) else: + module_types = _parse_vendor_files( + self.repo, selection.modules_path, vendor["name"], self.config.slugs or [] + ) + + rack_hint = slug_resolved["rack_vendors"] if slug_resolved is not None else None + if rack_hint is not None and vendor["slug"] not in rack_hint: rack_types = [] + else: + rack_types = _parse_vendor_files( + self.repo, + selection.racks_path, + vendor["name"], + self.config.slugs or [], + ) return VendorPlan( vendor=vendor, @@ -738,17 +722,16 @@ def apply(self, plan): ) cache.pump() - if self.netbox.modules: - _process_module_types( - self.config, - self.netbox, - self.reporter, - self.progress, - plan.module_types, - vendor_name=plan.vendor["name"], - task_registry=self.task_registry, - ) - cache.pump() + _process_module_types( + self.config, + self.netbox, + self.reporter, + self.progress, + plan.module_types, + vendor_name=plan.vendor["name"], + task_registry=self.task_registry, + ) + cache.pump() _process_rack_types( self.config, diff --git a/core/module_bay_types.py b/core/module_bay_types.py new file mode 100644 index 00000000..76697cd6 --- /dev/null +++ b/core/module_bay_types.py @@ -0,0 +1,264 @@ +"""Resolve module-bay-type reference names to NetBox ids. + +A device type's bay says which classes of module it accepts, and a module type says +which classes it belongs to. Both sides name those classes as plain strings, so +something has to turn a name into the id of one specific NetBox object, creating it +when the target instance has never seen it. + +That is this module. Callers pass the owning manufacturer and the names; they get +ids back. The scope rule, the catalog files, the lookup, the creation, and the cache +stay in here, so the four call sites (device-type create, module-type create, change +detection, export) do not each restate them. +""" + +import os +import re + +import pynetbox +import requests +import yaml + +from core.errors import FatalError + +# Directory in the devicetype-library holding the catalog, one directory per manufacturer. +CATALOG_DIRNAME = "module-bay-types" + +# Second resolution scope, for a class one vendor defines and another fills. +FALLBACK_MANUFACTURER = "Generic" + + +def manufacturer_slug(name): + """Slugify a manufacturer name the way the library loader does. + + ``core.repo`` reduces every YAML ``manufacturer`` to this slug, because upstream data + disagrees on case (RuggedCOM vs RuggedCom). The catalog keys on the same slug so both + sides meet on one value. + """ + return re.sub(r"\W+", "-", (name or "").lower()) + + +class ModuleBayTypeError(FatalError): + """A reference could not be resolved to exactly one NetBox object. + + Raised per definition so the caller can log it and continue with the next one. + Never raised for a difference this module could paper over: an unresolved name and + a conflicting identity are both reported rather than guessed at. + """ + + +class ModuleBayCatalogError(FatalError): + """The catalog itself could not be read, so no name in it can be trusted. + + A sibling of :class:`ModuleBayTypeError`, never a subclass: every caller recovers from + that one per definition, which would turn a single unreadable catalog into one skipped + component after another instead of ending the run once. + """ + + +class ModuleBayTypeCatalog: + """Turn module-bay-type names into NetBox ids, creating what is missing. + + ``ids_for(manufacturer, names)`` resolves each name in the owning manufacturer's + scope, then in ``Generic``, and raises :class:`ModuleBayTypeError` if neither has + it. Resolution creates the NetBox object, and the manufacturer that owns it, when + they do not exist yet. Returned order is not meaningful; callers compare as sets. + + Export does not use this class: it reads the names off the records NetBox returned, + because an export run resolves nothing and so has no id-to-name mapping of its own. + """ + + def __init__(self, netbox, repo_path, handle): + """Store the NetBox client, the library checkout to read the catalog from, and the log handle.""" + self._netbox = netbox + self._catalog_dir = os.path.join(repo_path, CATALOG_DIRNAME) + self._handle = handle + self._entries = None + self._load_error = None + self._ids = {} + self._manufacturer_ids = {} + + def ids_for(self, manufacturer, names): + """Return the NetBox id for each name, resolved against *manufacturer* then Generic. + + *manufacturer* is a manufacturer slug, as produced by :func:`manufacturer_slug`. + An empty list is a valid instruction to hold no classes; anything that is not a + list of non-empty strings is refused rather than read as empty, because reading a + malformed value as empty would drop a restriction the author asked for. + """ + self._validate(names) + # Duplicates collapse: the relationship is a set. + return sorted({self._id_for(manufacturer, name) for name in names}) + + def identities_for(self, manufacturer, names): + """Return the identity each name resolves to, without touching NetBox. + + The identity is ``(manufacturer slug, slug)``: the object a reference means, not + the name it is written as. Two manufacturers may both define a class called + ``X``, so a name alone cannot say which object is intended. + + This is the query half of the module. :meth:`ids_for` is the command half and + creates what is missing; this one reads only the catalog files, so change + detection can ask what a reference means without causing anything to exist. + """ + self._validate(names) + return frozenset( + (manufacturer_slug(entry["manufacturer"]), entry["slug"]) + for entry in (self._lookup(manufacturer, name) for name in names) + ) + + @staticmethod + def _validate(names): + """Reject anything that is not a list of non-empty names.""" + if names is None or not isinstance(names, list): + raise ModuleBayTypeError(f"module_bay_types must be a list of names, got {names!r}") + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ModuleBayTypeError(f"module_bay_types entries must be non-empty names, got {name!r}") + + def _id_for(self, manufacturer, name): + entry = self._lookup(manufacturer, name) + cache_key = (manufacturer_slug(entry["manufacturer"]), entry["name"]) + if cache_key not in self._ids: + self._ids[cache_key] = self._netbox_id(entry) + return self._ids[cache_key] + + def _lookup(self, manufacturer, name): + """Find the catalog entry for *name*, owner scope first, then Generic.""" + entries = self._load_catalog() + for scope in (manufacturer, manufacturer_slug(FALLBACK_MANUFACTURER)): + entry = entries.get((scope, name)) + if entry is not None: + return entry + raise ModuleBayTypeError( + f"Module bay type {name!r} is not in the catalog for manufacturer " + f"{manufacturer!r} or {FALLBACK_MANUFACTURER!r}" + ) + + def _load_catalog(self): + """Read every catalog file once, indexed by (manufacturer, name). + + A failure is cached too: it is terminal for the run, and re-walking the tree for + every later lookup only repeats the same error more slowly. + """ + if self._entries is not None: + return self._entries + if self._load_error is not None: + raise self._load_error + try: + self._entries = self._read_catalog() + except ModuleBayCatalogError as exc: + self._load_error = exc + raise + return self._entries + + def _read_catalog(self): + """Walk the catalog directory and return every entry, indexed by (manufacturer, name).""" + + def _unreadable(exc): + # os.walk skips a directory it cannot read; a short catalog resolves to the + # wrong entry rather than failing, so make the traversal error terminal. + raise ModuleBayCatalogError( + f"Module bay type catalog directory {exc.filename!r} could not be read: {exc}" + ) from exc + + entries = {} + for root, _dirs, files in os.walk(self._catalog_dir, onerror=_unreadable): + for filename in sorted(files): + if not filename.endswith((".yaml", ".yml")): + continue + path = os.path.join(root, filename) + try: + with open(path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) + except (OSError, yaml.YAMLError) as exc: + raise ModuleBayCatalogError( + f"Module bay type catalog file {path!r} could not be read: {exc}" + ) from exc + if data is None: + continue # an empty or comment-only document is not an entry + if not isinstance(data, dict): + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is not a mapping, got {type(data).__name__}" + ) + # Reject a half-written entry here; the readers index on name and dereference slug. + invalid = [ + field + for field in ("name", "slug", "manufacturer") + if not isinstance(data.get(field), str) or not data[field].strip() + ] + if invalid: + raise ModuleBayCatalogError( + f"Module bay type in {path!r} is missing or malformed: {', '.join(invalid)}" + ) + key = (manufacturer_slug(data.get("manufacturer")), data.get("name")) + if key in entries: + raise ModuleBayCatalogError(f"Duplicate module bay type {key[1]!r} for manufacturer {key[0]!r}") + entries[key] = data + return entries + + @staticmethod + def _request(action, description): + """Run one NetBox request, reporting a rejection as a catalog error. + + Callers resolve one definition at a time and recover from ModuleBayTypeError. A + raw RequestError or a dropped connection escapes that recovery and ends the run, + which can leave a parent half created and every later definition unprocessed. + """ + try: + return action() + except pynetbox.RequestError as exc: + raise ModuleBayTypeError(f"NetBox rejected {description}: {exc}") from exc + except requests.exceptions.RequestException as exc: + raise ModuleBayTypeError(f"NetBox could not be reached for {description}: {exc}") from exc + + def _netbox_id(self, entry): + """Return the id of the NetBox object for *entry*, creating it if absent.""" + manufacturer_id = self._manufacturer_id(entry["manufacturer"]) + existing = self._request( + lambda: list( + self._netbox.dcim.module_bay_types.filter(manufacturer_id=manufacturer_id, name=entry["name"]) + ), + f"the lookup of module bay type {entry['name']!r}", + ) + for record in existing: + if record.slug != entry["slug"]: + raise ModuleBayTypeError( + f"NetBox already has module bay type {entry['name']!r} for " + f"{entry['manufacturer']!r} with slug {record.slug!r}, but the catalog " + f"says {entry['slug']!r}. Resolve the conflict in NetBox; this import " + f"will not rename it." + ) + return record.id + + payload = {"name": entry["name"], "slug": entry["slug"], "manufacturer": manufacturer_id} + if entry.get("description"): + payload["description"] = entry["description"] + created = self._request( + lambda: self._netbox.dcim.module_bay_types.create(payload), + f"creating module bay type {entry['name']!r}", + ) + self._handle.verbose_log(f"Module Bay Type Created: {entry['name']} ({entry['manufacturer']}) - {created.id}") + return created.id + + def _manufacturer_id(self, name): + """Return the id of *name*, creating the manufacturer when the catalog needs it. + + A Generic-scoped class is reachable from any vendor, so an import filtered to one + manufacturer still has to create the manufacturer that owns the class. + """ + if name in self._manufacturer_ids: + return self._manufacturer_ids[name] + found = self._request( + lambda: list(self._netbox.dcim.manufacturers.filter(slug=manufacturer_slug(name))), + f"the lookup of manufacturer {name!r}", + ) + if found: + self._manufacturer_ids[name] = found[0].id + else: + created = self._request( + lambda: self._netbox.dcim.manufacturers.create({"name": name, "slug": manufacturer_slug(name)}), + f"creating manufacturer {name!r}", + ) + self._handle.verbose_log(f"Manufacturer Created: {name} - {created.id}") + self._manufacturer_ids[name] = created.id + return self._manufacturer_ids[name] diff --git a/core/nb_serializer.py b/core/nb_serializer.py index c6535506..4977553b 100644 --- a/core/nb_serializer.py +++ b/core/nb_serializer.py @@ -4,10 +4,9 @@ comparison against existing repo YAML files. """ -import warnings from typing import Any, Sequence -from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES +from core.component_registry import BY_ENDPOINT, COMPONENT_TYPES, MODULE_TYPE_RELATIONS # Row order sets the component key order of the serialized YAML. COMPONENT_ENDPOINT_NAMES = [component.endpoint for component in COMPONENT_TYPES] @@ -26,7 +25,6 @@ "feed_leg": None, "maximum_draw": None, "allocated_draw": None, - "positions": 1, # rear port default; include only when > 1 } # Device type scalar field order for output. @@ -125,48 +123,103 @@ def _serialize_component(record: Any, fields: Sequence[str]) -> dict: return result +def _serialize_relations(record: Any, relations: Sequence[str]) -> dict: + """Return the catalog name of each related object, which is how YAML names them. + + The names come off the record the query returned, not from the import-side catalog: + an export run resolves nothing, so it has no id-to-name mapping of its own. + + An empty relation writes no key. Emitting an empty list instead would add the key to + every bay in the library and make _repo_supersedes report every existing definition as + differing, and it tells a fresh import nothing that omitting it does not. + """ + result = {} + for relation in relations: + names = sorted( + name for name in (getattr(item, "name", None) for item in getattr(record, relation, None) or []) if name + ) + if names: + result[relation] = names + return result + + def _serialize_front_port(record: Any) -> dict: - """Serialize a front port template, including rear_port mapping.""" + """Serialize a front port template's own fields. + + The rear-port linkage is no longer written here: NetBox 4.5 moved it to a through + table and the library schema follows, carrying it in a top-level ``port-mappings`` + stanza built by :func:`_port_mappings`. + """ result = _serialize_component(record, BY_ENDPOINT["front_port_templates"].fields) - mappings = getattr(record, "mappings", None) or [] - if mappings: - if len(mappings) > 1: - port_name = getattr(record, "name", "") - warnings.warn( - f"Front port '{port_name}' has {len(mappings)} mappings; " - "only the first will be exported. " - "Full multi-mapping support requires DTL schema update (see issue #78).", - UserWarning, - stacklevel=4, - ) - m = mappings[0] - rear_port = getattr(m, "rear_port", None) - if rear_port: - result["rear_port"] = rear_port.name - rear_pos = getattr(m, "rear_port_position", None) - rear_pos = _coerce_numeric(rear_pos) - if rear_pos is not None and rear_pos > 1: - result["rear_port_position"] = rear_pos - else: - # Legacy: pre-4.5 NetBox returns rear_port / rear_port_position as direct scalar fields - legacy_rp = getattr(record, "rear_port", None) - if legacy_rp: - result["rear_port"] = legacy_rp.name - legacy_pos = getattr(record, "rear_port_position", None) - legacy_pos = _coerce_numeric(legacy_pos) - if legacy_pos is not None and legacy_pos > 1: - result["rear_port_position"] = legacy_pos + # positions is schema-required but arrived in 4.5, so a pre-4.5 record has none. + result.setdefault("positions", 1) return result +def _port_mappings(records: list) -> list: + """Return the ``port-mappings`` stanza for a type's front port templates. + + Every mapping is written, not just the first: one front port may occupy several + positions across rear ports, which is what the through table exists to express. + + A server below 4.5 has no through table and answers with ``rear_port`` and + ``rear_port_position`` scalars instead. Those describe one mapping, so they are + written as one entry rather than dropped. + """ + stanza = [] + for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): + name = getattr(record, "name", None) + mappings = [] + for mapping in getattr(record, "mappings", None) or []: + rear_port = getattr(mapping, "rear_port", None) + if not rear_port: + continue + mappings.append( + { + "front_port": name, + "front_port_position": _coerce_numeric(getattr(mapping, "front_port_position", None)) or 1, + "rear_port": rear_port.name, + "rear_port_position": _coerce_numeric(getattr(mapping, "rear_port_position", None)) or 1, + } + ) + stanza.extend( + sorted(mappings, key=lambda m: (m["front_port_position"], m["rear_port_position"], m["rear_port"] or "")) + ) + if getattr(record, "mappings", None): + continue + legacy = getattr(record, "rear_port", None) + if legacy: + stanza.append( + { + "front_port": name, + "front_port_position": 1, + "rear_port": legacy.name, + "rear_port_position": _coerce_numeric(getattr(record, "rear_port_position", None)) or 1, + } + ) + return stanza + + +def module_bays_missing_position(serialized: dict) -> list: + """Return the names of module bays the library schema would reject. + + NetBox leaves ``position`` blank on a bay that names no physical slot, and a blank + string writes no key, but the schema requires one on every module bay. + """ + return [bay.get("name", "?") for bay in serialized.get("module-bays", []) if "position" not in bay] + + def _serialize_component_list(endpoint_name: str, records: list) -> list: """Serialize a list of component template records for a given endpoint.""" + component = BY_ENDPOINT[endpoint_name] out = [] for record in sorted(records, key=lambda r: str(getattr(r, "name", "") or "")): if endpoint_name == "front_port_templates": - out.append(_serialize_front_port(record)) + serialized = _serialize_front_port(record) else: - out.append(_serialize_component(record, BY_ENDPOINT[endpoint_name].fields)) + serialized = _serialize_component(record, component.fields) + serialized.update(_serialize_relations(record, component.relations)) + out.append(serialized) return out @@ -177,6 +230,9 @@ def _add_components(result: dict, type_id: int, components_by_id: dict) -> None: records = type_components.get(component.endpoint, []) if records: result[component.yaml_key] = _serialize_component_list(component.endpoint, records) + mappings = _port_mappings(type_components.get("front_port_templates", [])) + if mappings: + result["port-mappings"] = mappings def serialize_device_type(nb_record: Any, components_by_dt_id: dict) -> dict: @@ -236,6 +292,7 @@ def serialize_module_type(nb_record: Any, components_by_mt_id: dict) -> dict: if _should_include(field, val): result[field] = val + result.update(_serialize_relations(nb_record, MODULE_TYPE_RELATIONS)) _add_components(result, nb_record.id, components_by_mt_id) return result diff --git a/core/netbox_api.py b/core/netbox_api.py index 220efae1..3321d45d 100644 --- a/core/netbox_api.py +++ b/core/netbox_api.py @@ -1,11 +1,11 @@ """NetBox REST and GraphQL API client for importing device and module type libraries.""" from collections import Counter +from dataclasses import replace from contextlib import contextmanager from functools import lru_cache import hashlib import json -import re import tempfile import time import pynetbox @@ -24,7 +24,9 @@ LINK_POWER_PORT, LINK_REAR_PORTS, MODULE_TYPE_COMPONENTS, + MODULE_TYPE_RELATIONS, ) +from core.compat import parse_netbox_version, supports_module_bay_types from core.formatting import log_property_diffs from core.errors import FatalError, UnknownError from core.graphql_client import GraphQLError, NetBoxGraphQLClient @@ -50,6 +52,35 @@ def __init__(self, ignore_ssl_errors: bool, cause=None): ) +# Oldest NetBox this importer is tested against; see the CI matrix and the README. +MINIMUM_NETBOX_VERSION = (4, 3) + + +def _relation_identities_differ(catalog, manufacturer, declared, related, *, names_differ): + """Return True when the assigned objects are not the ones the references resolve to. + + Comparing names alone reports equality when NetBox holds a same-named class from + another manufacturer's scope, so the identity is compared where both sides can supply + one. Where they cannot, the name comparison the caller already made stands. + """ + from core.module_bay_types import ModuleBayTypeError, manufacturer_slug + + if not catalog or not manufacturer: + return names_differ + current = set() + for item in related: + slug = getattr(item, "slug", None) + owner = getattr(getattr(item, "manufacturer", None), "slug", None) + if not slug or not owner: + return names_differ + current.add((manufacturer_slug(owner), slug)) + try: + return catalog.identities_for(manufacturer, declared) != frozenset(current) + except ModuleBayTypeError: + # A matching name must not make an unresolvable reference look applied. + return True + + class NetBoxError(FatalError): """A fatal error reported by the NetBox integration.""" @@ -404,6 +435,42 @@ def _image_dir_for_yaml(src_file: str, src_segment: str, dst_segment: str) -> "P # from pynetbox import RequestError as APIRequestError +def _is_mapping_removal(prop_change): + """Return True when a ``_mappings`` change only takes mappings away.""" + return ( + prop_change.property_name == "_mappings" + and isinstance(prop_change.old_value, (set, frozenset)) + and isinstance(prop_change.new_value, (set, frozenset)) + and prop_change.new_value < prop_change.old_value + ) + + +def _without_gated_mapping_clears(changes, remove_components, handle=None): + """Drop mapping clears unless removal is enabled. + + Clearing a front-to-rear linkage deletes data, but it reaches NetBox as a property + change rather than a COMPONENT_REMOVED, so it would otherwise ignore the flag and + contradict the "will not remove components" notice. Filtering here rather than at the + write boundary keeps the actionable count and the applied changes in agreement. + """ + if remove_components: + return changes + kept, gated = [], [] + for change in changes: + remaining = [pc for pc in change.property_changes if not _is_mapping_removal(pc)] + if len(remaining) == len(change.property_changes): + kept.append(change) + continue + gated.append(change.component_name) + if remaining: + kept.append(replace(change, property_changes=remaining)) + if gated and handle is not None: + handle.log( + f"Kept existing port mappings on {sorted(gated)}; use --remove-components with --update to clear them." + ) + return kept + + def _count_actionable_component_changes(changes, remove_components): """Return the count of changes in *changes* that will issue an API call. @@ -462,10 +529,8 @@ def __init__(self, config, handle): self.handle = handle self.netbox: Any = None self.ignore_ssl = config.ignore_ssl_errors - self.modules = False - self.new_filters = False + self.module_bay_types = False self.m2m_front_ports = False # True for NetBox >= 4.5 (M2M port mappings) - self.rack_types = False self.force_resolve_conflicts = config.force_resolve_conflicts self.remove_unmanaged_types = config.remove_unmanaged_types self.verify_images = config.verify_images @@ -495,6 +560,7 @@ def __init__(self, config, handle): self.ignore_ssl, handle=self.handle, page_size=config.graphql_page_size, + supports_module_bay_types=self.module_bay_types, ) try: self.existing_manufacturers = self.get_manufacturers() @@ -506,9 +572,9 @@ def __init__(self, config, handle): self.handle, self.counter, self.ignore_ssl, - self.new_filters, graphql=self.graphql, m2m_front_ports=self.m2m_front_ports, + module_bay_types_supported=self.module_bay_types, repo_path=config.repo_path, max_threads=config.preload_threads, ) @@ -570,10 +636,10 @@ def connect_api(self): raise UnknownError("NetBox API Error", cause=e) from e def verify_compatibility(self): - """Check the connected NetBox version and configure feature flags accordingly. + """Refuse a server below the supported floor and set the feature flags above it. - Sets ``self.modules = True`` for NetBox >= 3.2 and ``self.new_filters = True`` - for >= 4.1. Logs the detected version when the new-filter flag is enabled. + Only the flags that still vary across supported releases are set here: the 4.3 + floor makes everything introduced at or below 4.1 unconditional. """ # nb.version should be the version in the form '3.2' # Strip non-numeric suffixes (e.g. "4.1-beta") before converting to int. @@ -601,26 +667,26 @@ def verify_compatibility(self): msg += f"\nResponse body (may be from an intermediate proxy):\n{body}" msg += f"\nHint: Verify that {self.url} is reachable and not blocked by a proxy." raise NetBoxError(msg) from e - _raw = [int(re.sub(r"\D.*", "", x.strip()) or "0") for x in nb_version.split(".")] - version_split = (_raw + [0, 0])[:2] # pad to (major, minor) to guard against single-component strings + version_split = parse_netbox_version(nb_version) - # Later than 3.2 - # Might want to check for the module-types entry as well? - if version_split[0] > 3 or (version_split[0] == 3 and version_split[1] >= 2): - self.modules = True - - # check if version >= 4.1 in order to use new filter names (https://github.com/netbox-community/netbox/issues/15410) - if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 1): - self.new_filters = True - self.rack_types = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using new filters.") + # Below the floor the run dies later naming a schema field, not the real cause. + if tuple(version_split) < MINIMUM_NETBOX_VERSION: + minimum = ".".join(str(part) for part in MINIMUM_NETBOX_VERSION) + raise NetBoxError( + f"NetBox {nb_version} is not supported: this importer requires NetBox {minimum} or later. " + f"Older releases fail part way through with a GraphQL schema error rather than here." + ) # NetBox 4.5 replaced FrontPortTemplate.rear_port (FK) + rear_port_position (int) # with a ManyToMany through table (PortMapping). The creation and read APIs differ. # https://github.com/netbox-community/netbox/issues/20564 if version_split[0] > 4 or (version_split[0] == 4 and version_split[1] >= 5): self.m2m_front_ports = True - self.handle.log(f"Netbox version {self.netbox.version} found. Using M2M front/rear port mappings.") + self.handle.log(f"Netbox version {nb_version} found. Using M2M front/rear port mappings.") + + if supports_module_bay_types(nb_version): + self.module_bay_types = True + self.handle.log(f"Netbox version {nb_version} found. Module bay types are supported.") def get_manufacturers(self): """Fetch all manufacturers from NetBox via GraphQL and return them indexed by name.""" @@ -733,7 +799,6 @@ def _try_resolve_and_retry_device_type_update(self, dt, device_type, updates, er netbox=self.netbox, device_type_id=dt.id, device_type_yaml=device_type, - new_filters=self.new_filters, ) except Exception as exc: # defensive: classifier must never break the run self.handle.verbose_log(f"Failure classifier raised {type(exc).__name__}: {exc}") @@ -1043,7 +1108,10 @@ def _handle_existing_device_type( # Apply component changes component_errors = [] if dt_change.component_changes: - actionable_count = _count_actionable_component_changes(dt_change.component_changes, remove_components) + component_changes = _without_gated_mapping_clears( + dt_change.component_changes, remove_components, self.handle + ) + actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1053,11 +1121,11 @@ def _handle_existing_device_type( self.device_types.update_components( device_type, dt.id, - dt_change.component_changes, + component_changes, parent_type="device", ) if remove_components: - self.device_types.remove_components(dt.id, dt_change.component_changes, parent_type="device") + self.device_types.remove_components(dt.id, component_changes, parent_type="device") after_components = ( self.counter["components_updated"], self.counter["components_added"], @@ -1160,9 +1228,13 @@ def _create_device_type_components(self, device_type, dt_id, src_file, saved_ima yaml_key = component.yaml_key if yaml_key not in device_type: continue - if yaml_key == "module-bays" and not self.modules: - continue - self.device_types.create_components(yaml_key, device_type[yaml_key], dt_id, context=src_file) + self.device_types.create_components( + yaml_key, + device_type[yaml_key], + dt_id, + context=src_file, + manufacturer=device_type.get("manufacturer"), + ) if component_errors: # The type exists but not all of its components do. self.outcomes.record( @@ -1522,6 +1594,8 @@ def filter_actionable_module_types(self, module_types, all_module_types, only_ne if not values_equal(module_type[f], nb_val): changed_fields_info.append((f, nb_val, module_type[f])) + changed_fields_info += self._type_relation_changes(module_type, existing_module) + component_changes = detector._compare_components(module_type, existing_module.id, parent_type="module") if changed_fields_info or component_changes: @@ -1574,6 +1648,58 @@ def _fetch_module_type_existing_images(self): ) return module_type_existing_images + def _type_relation_changes(self, module_type, existing_module): + """Return (field, current, wanted) for each of the type's own relations that differs. + + A relation is a list, so the scalar comparison loop never sees it. A field the + query did not return is skipped rather than read as empty, which would otherwise + report a change on every run. + """ + changes: list[tuple[str, list[str], list[str]]] = [] + if not self.module_bay_types: + return changes + for field in MODULE_TYPE_RELATIONS: + if field not in module_type: + continue + netbox_value = getattr(existing_module, field, _MISSING) + if netbox_value is _MISSING: + continue + related = netbox_value if isinstance(netbox_value, (list, tuple)) else [] + declared = module_type[field] + if not isinstance(declared, list) or any(not isinstance(x, str) or not x.strip() for x in declared): + # Malformed or bare key: leave the relation unmanaged rather than clear it. + continue + wanted = sorted(set(declared)) + current = sorted({name for name in (getattr(item, "name", None) for item in related) if name}) + if _relation_identities_differ( + self.device_types.module_bay_type_catalog, + self.device_types._manufacturer_slug(module_type.get("manufacturer")), + declared, + related, + names_differ=wanted != current, + ): + changes.append((field, current, wanted)) + return changes + + def _resolve_type_relations(self, payload): + """Return *payload* with its own relation fields resolved from names to ids. + + Raises ModuleBayTypeError if a name does not resolve, so the caller can report the + module type rather than write it with the restriction silently dropped. + """ + names = {field: payload[field] for field in MODULE_TYPE_RELATIONS if field in payload} + if not names: + return payload + if not self.module_bay_types: + # The server predates ModuleBayType; sending the field would be rejected. + return {k: v for k, v in payload.items() if k not in names} + manufacturer = self.device_types._manufacturer_slug(payload.get("manufacturer")) + resolved = { + field: self.device_types.module_bay_type_catalog.ids_for(manufacturer, value) + for field, value in names.items() + } + return {**payload, **resolved} + def _try_update_module_type(self, curr_mt, module_type_res, src_file): """Apply pending field updates to an existing module type in NetBox. @@ -1590,6 +1716,18 @@ def _try_update_module_type(self, curr_mt, module_type_res, src_file): continue if not values_equal(curr_mt[field], current_value): updates[field] = curr_mt[field] + if self.module_bay_types: + from core.module_bay_types import ModuleBayTypeError + + for field, _current, wanted in self._type_relation_changes(curr_mt, module_type_res): + try: + updates[field] = self.device_types.module_bay_type_catalog.ids_for( + self.device_types._manufacturer_slug(curr_mt.get("manufacturer")), wanted + ) + except ModuleBayTypeError as exc: + # Not fatal to the run, but not a success either; the caller records it. + self.handle.log(f"Skipped {field} on {curr_mt.get('model')}: {exc} (Context: {src_file})") + return False, False if not updates: return True, False try: @@ -1622,7 +1760,12 @@ def _create_module_type_components(self, curr_mt, module_type_id, src_file): yaml_key = component.yaml_key if yaml_key in curr_mt: self.device_types.create_components( - yaml_key, curr_mt[yaml_key], module_type_id, parent_type="module", context=src_file + yaml_key, + curr_mt[yaml_key], + module_type_id, + parent_type="module", + context=src_file, + manufacturer=curr_mt.get("manufacturer"), ) if component_errors: # The module type exists but not all of its components do. @@ -1651,6 +1794,7 @@ def _apply_module_type_component_updates( self.device_types.ensure_components_ready(manufacturer_slug=curr_mt["manufacturer"]["slug"]) identity = f"{module_type_res.manufacturer.name}/{module_type_res.model}" component_changes = self.change_detector._compare_components(curr_mt, module_type_res.id, parent_type="module") + component_changes = _without_gated_mapping_clears(component_changes, remove_components, self.handle) if component_changes: actionable_count = _count_actionable_component_changes(component_changes, remove_components) before_updated = self.counter["components_updated"] @@ -1776,7 +1920,20 @@ def _process_single_module_type( ) else: try: - module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, curr_mt) + from core.module_bay_types import ModuleBayTypeError + + try: + payload = self._resolve_type_relations(curr_mt) + except ModuleBayTypeError as exc: + self.handle.log(f"Error creating Module Type: {exc} (Context: {src_file})") + self._record_failure( + EntityKind.MODULE_TYPE, + self._yaml_identity(curr_mt), + str(exc), + src_file, + ) + return False + module_type_res = _retry_on_connection_error(self.netbox.dcim.module_types.create, payload) self.counter["module_added"] += 1 is_new = True manufacturer_slug = curr_mt["manufacturer"]["slug"] @@ -2126,7 +2283,7 @@ class _FrontPortRecordWithMappings: All other attribute accesses are forwarded to the underlying record. """ - __slots__ = ("_record", "_mappings_canonical") + __slots__ = ("_record", "_mappings_canonical", "_mappings_m2m") def __init__(self, record): """Wrap *record* and pre-compute a canonical mappings list for ChangeDetector compatibility. @@ -2160,10 +2317,18 @@ def __init__(self, record): else: # NetBox < 4.5: rear_port_position is a direct scalar field rp_pos = getattr(record, "rear_port_position", None) + # The pre-4.5 query asks for rear_port { id name }; keeping the name is what lets + # a move to a different rear port be seen at all. + legacy_rp = getattr(record, "rear_port", None) + legacy_name = ( + (legacy_rp.get("name") if isinstance(legacy_rp, dict) else getattr(legacy_rp, "name", None)) + if legacy_rp is not None + else None + ) canonical = ( [ { - "rear_port_name": None, + "rear_port_name": legacy_name, "front_port_position": 1, "rear_port_position": rp_pos, } @@ -2172,6 +2337,10 @@ def __init__(self, record): else None # Both mappings and rear_port_position absent; skip comparison. ) object.__setattr__(self, "_mappings_canonical", canonical) + # Which model the record came from, recorded rather than inferred: an empty M2M list + # carries no names to infer from, and reading it as pre-4.5 drops the rear port name + # the patch needs. + object.__setattr__(self, "_mappings_m2m", mappings_raw is not None) def __getattr__(self, name): """Delegate attribute access to the wrapped record.""" @@ -2187,11 +2356,11 @@ def __init__( handle, counter, ignore_ssl, - new_filters, *, graphql, repo_path, m2m_front_ports=False, + module_bay_types_supported=False, max_threads=8, ): """Initialize empty DeviceTypes cache structures; no data is fetched at construction time. @@ -2204,8 +2373,8 @@ def __init__( handle (LogHandler): Sink for creation and error messages. counter (Counter): Shared operation counter updated during creation. ignore_ssl (bool): Whether SSL certificate verification is disabled. - new_filters (bool): Whether to use updated filter parameter names (NetBox >= 4.1). graphql (NetBoxGraphQLClient): GraphQL client for read queries. + module_bay_types_supported (bool): True when NetBox supports ModuleBayType (>= 4.7). repo_path (str): Local library checkout, used to read the module-type schema. m2m_front_ports (bool): Whether NetBox uses the 4.5+ M2M port mapping model. max_threads (int): Maximum number of concurrent threads for component preloading. @@ -2214,20 +2383,20 @@ def __init__( self.handle = handle self.counter = counter self.ignore_ssl = ignore_ssl - self.new_filters = new_filters self.graphql = graphql self.repo_path = repo_path self.m2m_front_ports = m2m_front_ports + self.module_bay_types_supported = module_bay_types_supported self.max_threads = max_threads self.components = ComponentCache( netbox, graphql, handle, - new_filters, max_threads, wrap_record=_FrontPortRecordWithMappings, ) self._image_progress = None + self._module_bay_type_catalog = None # Component failures for the entity currently inside collect_component_errors(). self._component_errors: list[str] = [] self.existing_device_types = {} @@ -2428,7 +2597,11 @@ def _apply_mappings_change(self, comp_name, new_mappings, yaml_mappings, update_ update_data["rear_port"] = None update_data["rear_port_position"] = None return - first = next(iter(new_mappings)) + if len(new_mappings) > 1: + self._log_component_error( + f'Multiple mappings for front port "{comp_name}" on NetBox < 4.5: only first mapping applied' + ) + first = sorted(new_mappings)[0] if len(first) != 3: # Legacy NetBox (<4.5): ChangeDetector emits 2-tuples (fp_pos, rp_pos) # because rear port names are unavailable via the GraphQL API. @@ -2486,6 +2659,7 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, if change.component_name in existing: comp = existing[change.component_name] update_data = {"id": comp.id} + unresolved = False for pc in change.property_changes: if comp_type == "front-ports" and pc.property_name == "_mappings": yaml_front_port = next( @@ -2501,8 +2675,21 @@ def _apply_updates_for_type(self, comp_type, changes, yaml_data, device_type_id, parent_type, ) continue + if pc.property_name in component.relations: + # The comparison works in names; NetBox wants ids. + from core.module_bay_types import ModuleBayTypeError + + try: + update_data[pc.property_name] = self.module_bay_type_catalog.ids_for( + self._manufacturer_slug(yaml_data.get("manufacturer")), pc.new_value + ) + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{change.component_name}': {exc}") + unresolved = True + break + continue update_data[pc.property_name] = pc.new_value - if len(update_data) > 1: # has fields beyond just "id" + if not unresolved and len(update_data) > 1: # has fields beyond just "id" updates.append(update_data) success_count = 0 @@ -2547,7 +2734,13 @@ def _apply_additions_for_type(self, comp_type, changes, yaml_data, device_type_i if not components_to_add: return - self.create_components(comp_type, components_to_add, device_type_id, parent_type=parent_type) + self.create_components( + comp_type, + components_to_add, + device_type_id, + parent_type=parent_type, + manufacturer=yaml_data.get("manufacturer"), + ) def update_components(self, yaml_data, device_type_id, component_changes, parent_type="device"): """Update existing components and add new components based on detected changes. @@ -2778,7 +2971,7 @@ def link_rear_ports(items, pid): else: if len(resolved) > 1: ctx = f" (Context: {context})" if context else "" - self.handle.log( + self._log_component_error( f'Multiple mappings for {label} "{port["name"]}" on NetBox < 4.5: ' f"only first mapping applied{ctx}" ) @@ -2837,7 +3030,66 @@ def _link_bridges(self, bridged, parent_id, parent_type, context=None): f"Connection error bridging interfaces after {_MAX_RETRIES} retries: {e} (Context: {context})" ) - def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None): + @property + def module_bay_type_catalog(self): + """The module-bay-type catalog, built once per run from the library checkout.""" + if self._module_bay_type_catalog is None: + from core.module_bay_types import ModuleBayTypeCatalog + + self._module_bay_type_catalog = ModuleBayTypeCatalog(self.netbox, self.repo_path, self.handle) + return self._module_bay_type_catalog + + @staticmethod + def _manufacturer_slug(manufacturer): + """Return the manufacturer slug, whatever shape the caller happens to hold. + + ``core.repo`` rewrites every YAML ``manufacturer`` to ``{"slug": ...}`` before the + importer sees it, so that is the usual shape; a plain name is slugified here. + """ + from core.module_bay_types import manufacturer_slug + + if isinstance(manufacturer, dict): + return manufacturer.get("slug") or manufacturer_slug(manufacturer.get("name")) + return manufacturer_slug(getattr(manufacturer, "name", manufacturer)) + + def _resolve_relations(self, component, items, manufacturer): + """Turn each relation field's names into NetBox ids, dropping items that cannot resolve. + + A component whose restriction cannot be resolved is skipped and logged rather than + created without it: creating the bay anyway would silently discard the restriction. + """ + if not component.relations: + return items + if not self.module_bay_types_supported: + # The server predates ModuleBayType; drop the field rather than have it rejected. + return [{k: v for k, v in item.items() if k not in component.relations} for item in items] + from core.module_bay_types import ModuleBayTypeError + + manufacturer = self._manufacturer_slug(manufacturer) + resolved = [] + for item in items: + names = {field: item[field] for field in component.relations if field in item} + if not names: + resolved.append(item) + continue + if not manufacturer: + # No scope to resolve in, and NetBox wants ids: sending the names would fail. + self._log_component_error( + f"Skipped {component.label} '{item.get('name', 'Unknown')}': no manufacturer to " + f"resolve {', '.join(sorted(names))} in" + ) + continue + try: + replacements = { + field: self.module_bay_type_catalog.ids_for(manufacturer, value) for field, value in names.items() + } + except ModuleBayTypeError as exc: + self._log_component_error(f"Skipped {component.label} '{item.get('name', 'Unknown')}': {exc}") + continue + resolved.append({**item, **replacements}) + return resolved + + def create_components(self, yaml_key, items, parent_id, parent_type="device", context=None, manufacturer=None): """Create component templates of one kind for one parent, skipping those that exist. The registry row for *yaml_key* supplies the endpoint, the cache name, the log label @@ -2850,6 +3102,8 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co parent_id (int): NetBox ID of the parent device or module type. parent_type (str): ``"device"`` or ``"module"``. context (str | None): Optional context string appended to log messages. + manufacturer (dict | str | None): Owning manufacturer, used to resolve any + relation fields the registry row declares. """ component = BY_YAML_KEY[yaml_key] label = component.create_label(parent_type) @@ -2869,7 +3123,7 @@ def create_components(self, yaml_key, items, parent_id, parent_type="device", co self._create_generic( component, - items, + self._resolve_relations(component, items, manufacturer), parent_id, parent_type=parent_type, post_process=post_process, diff --git a/core/normalization.py b/core/normalization.py index c8846ec9..d807f38a 100644 --- a/core/normalization.py +++ b/core/normalization.py @@ -1,6 +1,11 @@ """Shared value-normalization helpers for YAML-vs-NetBox comparisons.""" +def is_explicit_list(value): + """Only an explicit YAML list manages a relation; null leaves it unmanaged.""" + return isinstance(value, list) + + def normalize_values(yaml_val, nb_val): """Normalize a YAML/NetBox value pair for comparison. diff --git a/core/repo.py b/core/repo.py index 2022cb57..62472f7a 100644 --- a/core/repo.py +++ b/core/repo.py @@ -13,6 +13,7 @@ from core.config import LOCAL_REPO_URL, is_local_repo_url from core.errors import FatalError, UnknownError +from core.normalization import is_explicit_list # Top-level directories that make a checkout a device-type library. LIBRARY_TYPE_DIRS = ("device-types", "module-types", "rack-types") @@ -272,6 +273,59 @@ def validate_repo_path(repo_path): return True, "" +def _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared): + """Return ``({front_port_name: [mapping, ...]}, error)`` for the pre-4.5 inline format. + + The inline keys are removed from each entry as they are read, so the caller is left with + one representation to reason about. + """ + inline_mappings: dict = {} + for fp in front_ports: + rp_name = fp.get("rear_port") + if rp_name is None: + continue + fp_name = fp.get("name") + if rear_ports_declared and rp_name not in rear_by_name: + return {}, f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" + rp_pos = fp.pop("rear_port_position", 1) + fp.pop("rear_port") + inline_mappings.setdefault(fp_name, []).append( + {"rear_port": rp_name, "front_port_position": 1, "rear_port_position": rp_pos} + ) + return inline_mappings, None + + +def _conflicting_mapping(inline_mappings, stanza_mappings): + """Return an error when the two formats cannot both be honoured. + + Carrying both is allowed only while they agree, which is what a half-finished migration + looks like; disagreeing is the case where guessing a winner would silently pick one. + A stanza speaks for the whole file, so an inline linkage it omits cannot be honoured + either. A port the stanza alone names is not in question: it had no inline linkage. + """ + if not (inline_mappings and stanza_mappings): + return None + + def _shape(mappings): + return sorted((m["rear_port"], m["front_port_position"], m["rear_port_position"]) for m in mappings) + + for name in sorted(inline_mappings): + if name not in stanza_mappings: + return ( + f"Error: front port '{name}' declares an inline rear_port but the port-mappings " + f"stanza does not list it; the stanza is authoritative, so add '{name}' to it " + f"or remove the inline rear_port keys" + ) + inline = _shape(inline_mappings[name]) + stanza = _shape(stanza_mappings[name]) + if inline != stanza: + return ( + f"Error: front port '{name}' has conflicting mapping definitions " + f"(inline: {inline}, port-mappings stanza: {stanza})" + ) + return None + + def normalize_port_mappings(data): """Normalize port mapping definitions in a parsed YAML device/module type dict. @@ -309,9 +363,9 @@ def normalize_port_mappings(data): """ front_ports = data.get("front-ports") or [] port_mappings_stanza = data.get("port-mappings") - - if not front_ports and "port-mappings" not in data: - return None + stanza_authoritative = is_explicit_list(port_mappings_stanza) + if port_mappings_stanza is not None and not stanza_authoritative: + return f"Error: port-mappings must be a list: {port_mappings_stanza!r}" front_by_name = {fp["name"]: fp for fp in front_ports if fp.get("name")} rear_ports_declared = "rear-ports" in data @@ -320,28 +374,16 @@ def normalize_port_mappings(data): # --- Old inline format --- # Collect rear_port references declared directly on front-port entries. - inline_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - for fp in front_ports: - rp_name = fp.get("rear_port") - if rp_name is None: - continue - fp_name = fp.get("name") - if rear_ports_declared and rp_name not in rear_by_name: - return f"Error: front-port '{fp_name}' references unknown rear_port '{rp_name}'" - rp_pos = fp.pop("rear_port_position", 1) - fp.pop("rear_port") - inline_mappings.setdefault(fp_name, []).append( - { - "rear_port": rp_name, - "front_port_position": 1, - "rear_port_position": rp_pos, - } - ) + inline_mappings, error = _collect_inline_mappings(front_ports, rear_by_name, rear_ports_declared) + if error: + return error # --- New port-mappings stanza --- stanza_mappings: dict = {} # {front_port_name: [mapping_dict, ...]} - if "port-mappings" in data: + if stanza_authoritative: for entry in port_mappings_stanza or []: + if not isinstance(entry, dict): + return f"Error: port-mappings entry must be a mapping: {entry!r}" fp_name = entry.get("front_port") rp_name = entry.get("rear_port") if not fp_name or not rp_name: @@ -357,29 +399,23 @@ def normalize_port_mappings(data): "rear_port_position": entry.get("rear_port_position", 1), } ) - del data["port-mappings"] - - # --- Conflict detection --- - # Accept both formats simultaneously only when they describe identical mappings. - if inline_mappings and stanza_mappings: - all_names = set(inline_mappings) | set(stanza_mappings) - for name in all_names: - inline = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in inline_mappings.get(name, []) - ) - stanza = sorted( - (m["rear_port"], m["front_port_position"], m["rear_port_position"]) - for m in stanza_mappings.get(name, []) + data.pop("port-mappings", None) + + conflict = _conflicting_mapping(inline_mappings, stanza_mappings) + if conflict: + return conflict + + if stanza_authoritative: + if not stanza_mappings and inline_mappings: + return ( + "Error: port-mappings is empty but front port(s) " + f"{sorted(inline_mappings)} still declare an inline rear_port" ) - if inline != stanza: - return ( - f"Error: front port '{name}' has conflicting mapping definitions " - f"(inline: {inline}, port-mappings stanza: {stanza})" - ) + for fp in front_ports: + fp["_mappings"] = stanza_mappings.get(fp.get("name"), []) + return None - effective = stanza_mappings if stanza_mappings else inline_mappings - for fp_name, mappings in effective.items(): + for fp_name, mappings in inline_mappings.items(): if fp_name in front_by_name: front_by_name[fp_name]["_mappings"] = mappings diff --git a/core/update_failure_resolver.py b/core/update_failure_resolver.py index 7e2eb1f6..82543a5d 100644 --- a/core/update_failure_resolver.py +++ b/core/update_failure_resolver.py @@ -24,8 +24,6 @@ from enum import Enum from typing import Any, Callable, List, Optional -from core.compat import device_type_filter_kwargs - class FailureKind(str, Enum): """High-level classification of a NetBox update failure.""" @@ -122,7 +120,7 @@ def _matches_subdevice_role_constraint(payload: Any) -> bool: return False -def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> tuple[int, List[str]]: +def _count_dependent_devices(netbox: Any, device_type_id: int) -> tuple[int, List[str]]: """Query NetBox for devices using *device_type_id*. Returns ``(count, sample_names)`` where ``sample_names`` is up to 5 names @@ -133,10 +131,8 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ - filter_kwargs = device_type_filter_kwargs(device_type_id, new_filters=new_filters) + filter_kwargs = {"device_type_id": device_type_id} try: devices = list(netbox.dcim.devices.filter(**filter_kwargs, limit=5)) except Exception: @@ -152,7 +148,7 @@ def _count_dependent_devices(netbox: Any, device_type_id: int, *, new_filters: b return total, sample -def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: bool = False) -> Optional[List[Any]]: +def _list_device_bay_templates(netbox: Any, device_type_id: int) -> Optional[List[Any]]: """Return all ``DeviceBayTemplate`` records attached to *device_type_id*. Returns ``None`` when the NetBox query itself fails (network error, 5xx, etc.) @@ -161,15 +157,9 @@ def _list_device_bay_templates(netbox: Any, device_type_id: int, *, new_filters: Args: netbox: pynetbox API client. device_type_id: ID of the device type to query. - new_filters: When True, use ``device_type_id`` filter name (NetBox ≥ 4.1); - otherwise use the legacy ``devicetype_id`` name. """ try: - return list( - netbox.dcim.device_bay_templates.filter( - **device_type_filter_kwargs(device_type_id, new_filters=new_filters) - ) - ) + return list(netbox.dcim.device_bay_templates.filter(device_type_id=device_type_id)) except Exception: return None @@ -180,7 +170,6 @@ def classify_device_type_update_failure( netbox: Any, device_type_id: int, device_type_yaml: dict, - new_filters: bool = False, ) -> FailureResolution: """Classify a ``pynetbox.RequestError`` raised while updating a device type. @@ -192,7 +181,6 @@ def classify_device_type_update_failure( device_type_yaml: Parsed YAML dict for this device-type (used to detect whether the YAML *also* lists device bays — in which case we cannot blindly delete them). - new_filters: When True, use updated filter parameter names (NetBox ≥ 4.1). Returns: A :class:`FailureResolution` describing the constraint and (when safe) @@ -207,7 +195,7 @@ def classify_device_type_update_failure( ) # SUBDEVICE_ROLE_FLIP path ------------------------------------------------- - blocking_templates = _list_device_bay_templates(netbox, device_type_id, new_filters=new_filters) + blocking_templates = _list_device_bay_templates(netbox, device_type_id) if blocking_templates is None: return FailureResolution( kind=FailureKind.MANUAL_REQUIRED, @@ -216,7 +204,7 @@ def classify_device_type_update_failure( ) blocking_names = [getattr(t, "name", str(getattr(t, "id", "?"))) for t in blocking_templates] - dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id, new_filters=new_filters) + dep_count, dep_sample = _count_dependent_devices(netbox, device_type_id) # YAML must NOT redefine device-bays — otherwise deleting them would just # cause our own component-creation step to fail or thrash. This catches diff --git a/tests/conftest.py b/tests/conftest.py index 364bd128..a2d55c4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,14 @@ def mock_git_repo(request): @pytest.fixture def mock_pynetbox(): - """Mock pynetbox to prevent API calls.""" + """Mock pynetbox to prevent API calls. + + Defaults the reported version to the oldest supported release, so a test that does not + care about version gating still constructs a NetBox; the importer refuses anything + older. Individual tests override it to exercise a specific release. + """ with patch("core.netbox_api.pynetbox") as mock_nb: + mock_nb.api.return_value.version = "4.3" yield mock_nb @@ -108,6 +114,13 @@ def mock_graphql_requests(request): } } mock_session.post.return_value = response + # /api/status/ is a GET; without this the probe reads a synthesized MagicMock and + # any shape check on the payload sees something no NetBox would ever return. + status = MagicMock() + status.status_code = 200 + status.raise_for_status = MagicMock() + status.json.return_value = {"netbox-version": "4.7.0"} + mock_session.get.return_value = status yield mock_session.post diff --git a/tests/helpers.py b/tests/helpers.py index 9bbad389..c6f0f381 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,13 @@ """Shared test utilities for the NetBox device-type importer test suite.""" +import json +import re +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlparse + +import pynetbox def paginate_dispatch(data_dict): @@ -49,3 +56,195 @@ def recording_handle(): console = RecordingConsole() handle.set_console(console) return handle, console + + +class FakeNetBox: + """A local HTTP server answering the slice of the NetBox REST API the importer uses. + + Collections are addressed by their URL segment with dashes turned into underscores, + so ``module-bay-types`` is ``module_bay_types``. GET filters, POST creates and PATCH + bulk-updates behave the way pynetbox expects, so a real client drives it and the + serialising, filtering and paginating under test are the production ones. + + ``/graphql/`` answers every query with an empty list, which is enough to bring a real + :class:`~core.netbox_api.NetBox` up; the REST half is where the assertions live. + + A test using this must carry the ``real_http`` marker. Without it the suite patches + ``requests.Session`` and no request ever leaves the client. + """ + + _IGNORED_FILTERS = ("limit", "offset", "brief", "exclude") + + # GraphQL collections are named "_list"; the client asks for one per query. + _LIST_KEY = re.compile(r"\b(\w+_list)\b") + + def __init__(self, errors=None, netbox_version="4.7.0", **collections): + """Start the server with *collections* seeded, keyed by collection name. + + *errors* maps a collection name to an HTTP status the server answers it with, so a + test can drive a rejection the importer has to survive. *netbox_version* is what + ``/api/status/`` reports, which is how the export side decides what it may select. + """ + self.collections = {name: [dict(r) for r in records] for name, records in collections.items()} + self.errors = dict(errors or {}) + self.netbox_version = netbox_version + self.requests = [] + self._server = HTTPServer(("127.0.0.1", 0), self._handler()) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def url(self): + """Return the base URL a client should talk to.""" + return f"http://127.0.0.1:{self._server.server_port}" + + def api(self, token="test-token"): + """Return a real pynetbox client pointed at this server.""" + return pynetbox.api(self.url, token=token) + + def close(self): + """Stop serving and release the listening socket.""" + self._server.shutdown() + self._server.server_close() + + def collection(self, name): + """Return the stored records for a collection, creating it empty when unseen.""" + return self.collections.setdefault(name, []) + + def sent(self, verb, name): + """Return the payload of each *verb* request made to collection *name*. + + A bulk create or update sends a list; its entries are returned individually, so a + caller asserting on what was written does not have to care which shape was used. + """ + out = [] + for method, collection, payload in self.requests: + if method != verb or collection != name: + continue + out.extend(payload) if isinstance(payload, list) else out.append(payload) + return out + + def matches(self, name, query): + """Filter a collection the way the NetBox REST API filters it.""" + out = [] + for record in self.collection(name): + ok = True + for key, values in query.items(): + if key in self._IGNORED_FILTERS: + continue + field = key[:-3] if key.endswith("_id") else key + actual = record.get(field) + if isinstance(actual, dict): + actual = actual.get("id") + if str(actual) not in values: + ok = False + break + if ok: + out.append(record) + return out + + def _create(self, name, payload): + """Store the new record(s) and echo them back the way NetBox does. + + A list payload is a bulk create, which is how the importer adds components, and it + answers with a list. + """ + if isinstance(payload, list): + return [self._create_one(name, item) for item in payload] + return self._create_one(name, payload) + + def _create_one(self, name, payload): + """Store one new record and return it as NetBox would echo it back.""" + collection = self.collection(name) + record = {"id": 1000 + len(collection), **payload} + # NetBox echoes a foreign key back as a nested object, not the id it was given. + if isinstance(record.get("manufacturer"), int): + owner = next((m for m in self.collection("manufacturers") if m["id"] == record["manufacturer"]), {}) + record["manufacturer"] = {"id": record["manufacturer"], "name": owner.get("name")} + collection.append(record) + return record + + def _patch(self, name, payload): + """Merge a bulk update into the stored records and return the updated ones.""" + updated = [] + for entry in payload if isinstance(payload, list) else [payload]: + record = next((r for r in self.collection(name) if r["id"] == entry.get("id")), None) + if record is None: + continue + record.update(entry) + updated.append(record) + return updated + + def _handler(self): + """Build the request handler class bound to this server's state.""" + state = self + + class Handler(BaseHTTPRequestHandler): + def _collection_name(self): + return urlparse(self.path).path.rstrip("/").rsplit("/", 1)[-1].replace("-", "_") + + def _body(self): + return json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))) or b"{}") + + def _reply(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _refused(self, name): + """Answer with the configured status for *name*, if the test set one.""" + status = state.errors.get(name) + if status is None: + return False + self._reply(status, {"detail": f"{name} refused with {status}"}) + return True + + def do_GET(self): + parsed = urlparse(self.path) + name = self._collection_name() + state.requests.append(("GET", name, parsed.query)) + if name == "status": + self._reply(200, {"netbox-version": state.netbox_version}) + return + if self._refused(name): + return + results = state.matches(name, parse_qs(parsed.query)) + self._reply(200, {"count": len(results), "next": None, "previous": None, "results": results}) + + def do_POST(self): + name = self._collection_name() + payload = self._body() + if name == "graphql": + state.requests.append(("POST", name, payload)) + key = state._LIST_KEY.search(payload.get("query", "")) + self._reply(200, {"data": {key.group(1) if key else "unknown_list": []}}) + return + state.requests.append(("POST", name, payload)) + if self._refused(name): + return + self._reply(201, state._create(name, payload)) + + def do_PATCH(self): + name = self._collection_name() + payload = self._body() + state.requests.append(("PATCH", name, payload)) + if self._refused(name): + return + self._reply(200, state._patch(name, payload)) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + return Handler + + +def write_module_bay_type(root, manufacturer, slug, name, description=None): + """Write one module-bay-type catalog file where the devicetype-library puts it.""" + directory = root / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + body = f"name: {name}\nslug: {slug}\nmanufacturer: {manufacturer}\n" + if description: + body += f"description: {description}\n" + (directory / f"{slug}.yaml").write_text(body, encoding="utf-8") diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index 3a9b6434..f56450d3 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -19,7 +19,7 @@ def _cache(**records): Populating through the real object means these tests read the same index the importer builds, rather than a dict that merely looks like it. """ - cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), new_filters=True, max_threads=1) + cache = ComponentCache(MagicMock(), MagicMock(), MagicMock(), max_threads=1) for endpoint_name, items in records.items(): cache.populate(endpoint_name, items) return cache @@ -428,6 +428,54 @@ def _make_netbox_comp(self, canonical, **attrs): """Build a netbox component with _mappings_canonical and explicit attributes.""" return SimpleNamespace(_mappings_canonical=canonical, **attrs) + def test_adding_a_mapping_to_an_unmapped_45_port_keeps_the_rear_port_name(self): + """mappings=[] is 4.5+ saying "none", not a pre-4.5 record without the field. + + Inferring the model from the data cannot tell those apart when the list is empty, and + a positions-only tuple makes _build_mappings_patch return None, so the mapping is + never added and nothing is logged. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + netbox_comp = _FrontPortRecordWithMappings(SimpleNamespace(name="FP1", type="8p8c", mappings=[])) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "adding a mapping is a change" + tup = next(iter(changes[0].new_value)) + assert len(tup) == 3, "a positions-only tuple cannot rebuild the M2M mapping" + assert tup[0] == "RP1" + + def test_a_legacy_rear_port_change_is_detected_by_name(self): + """The pre-4.5 query asks for rear_port { id name }, so the name is available. + + Discarding it left only positions to compare, and RP1/1 -> RP2/1 produced no change, + no PATCH and no warning: the definition silently never synced. + """ + from core.netbox_api import _FrontPortRecordWithMappings + + legacy = SimpleNamespace(name="FP1", type="8p8c", rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) + netbox_comp = _FrontPortRecordWithMappings(legacy) + yaml_comp = { + "name": "FP1", + "type": "8p8c", + "_mappings": [{"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1}], + } + + changes = self._cd()._compare_component_properties( + yaml_comp, netbox_comp, ["name", "type", "_mappings"], comp_type="front-ports" + ) + + assert len(changes) == 1, "moving the front port to a different rear port is a change" + assert next(iter(changes[0].new_value))[0] == "RP2" + def test_identical_mappings_no_change(self): """Same mapping on both sides → no property change.""" yaml_comp = { diff --git a/tests/test_component_cache.py b/tests/test_component_cache.py index a6d9e330..a915936f 100644 --- a/tests/test_component_cache.py +++ b/tests/test_component_cache.py @@ -187,7 +187,6 @@ def make_cache(netbox=None, graphql=None, handle=None, **kwargs): netbox or FakeNetBox(), graphql or FakeGraphQL(), handle or FakeHandle(), - kwargs.pop("new_filters", True), kwargs.pop("max_threads", 4), **kwargs, ) @@ -279,14 +278,6 @@ def test_a_miss_filters_by_module_type_for_a_module_parent(self): assert endpoint.filter_calls == [{"module_type_id": 5}] - def test_old_netbox_filter_names_are_used_when_asked(self): - cache = make_cache(new_filters=False) - endpoint = FakeEndpoint() - - cache.get("interface_templates", "device", 1, endpoint) - - assert endpoint.filter_calls == [{"devicetype_id": 1}] - def test_an_empty_result_still_becomes_a_hit(self): """Otherwise every parent with no components is re-read on each lookup.""" cache = make_cache() diff --git a/tests/test_component_registry.py b/tests/test_component_registry.py index 2b19c784..f1d05dbe 100644 --- a/tests/test_component_registry.py +++ b/tests/test_component_registry.py @@ -128,10 +128,18 @@ def test_the_query_selects_exactly_these_fields(self): "label", "description", "color", + "positions", _FRONT_PORT_MAPPINGS, ], "device_bay_templates": ["id", "name", "label", "description"], - "module_bay_templates": ["id", "name", "position", "label", "description"], + "module_bay_templates": [ + "id", + "name", + "position", + "label", + "description", + "module_bay_types { id name slug manufacturer { slug } }", + ], } def test_only_the_front_port_query_selects_a_nested_block(self): @@ -146,13 +154,17 @@ class TestDerivedComparisonAndExport: @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_every_compared_property_is_fetched(self, component): """A property compared but never queried reads as missing and is skipped in silence.""" - queried = set(component.graphql_fields) | {"_mappings"} + # A relation is selected as a nested block, so take the field name each selection + # opens with. Adding component.relations here instead would make the assertion + # hold even if the query stopped selecting them. + queried = {selection.split(None, 1)[0] for selection in component.graphql_fields} | {"_mappings"} assert set(component.compare_properties) <= queried @pytest.mark.parametrize("component", COMPONENT_TYPES, ids=lambda c: c.yaml_key) def test_the_export_writes_every_scalar_the_query_reads(self, component): """Export fields are the query's scalars: an unexported scalar drops out of a round trip.""" - scalars = [name for name in component.graphql_fields if name != "id" and name not in component.graphql_extra] + non_scalar = set(component.graphql_extra) | set(component.graphql_relation_fields) + scalars = [name for name in component.graphql_fields if name != "id" and name not in non_scalar] assert list(component.fields) == scalars def test_front_ports_compare_the_mapping_the_query_selects(self): diff --git a/tests/test_config.py b/tests/test_config.py index 1aae1838..e289960c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -139,3 +139,55 @@ def test_a_branch_set_beside_a_real_url_needs_no_notice(self): config = _resolve(REPO_URL="https://example.com/repo.git", REPO_BRANCH="feature") assert not any("REPO_BRANCH" in notice for notice in config.notices), config.notices + + +class TestInsecureNetboxUrlIsReported: + """The API token travels in an Authorization header, so cleartext transport leaks it.""" + + @pytest.mark.parametrize( + "url", + [ + "http://netbox.example.com", + "http://netbox.example.com:8000/", + "http://10.0.0.5:8000", + ], + ) + def test_http_to_a_remote_host_is_reported(self, url): + config = _resolve(NETBOX_URL=url) + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + @pytest.mark.parametrize( + "url", + [ + "http://localhost:8000", + "http://127.0.0.1:8000/", + "http://[::1]:8000", + ], + ) + def test_http_to_loopback_needs_no_notice(self, url): + """A loopback URL never leaves the host, and the integration tests rely on it.""" + config = _resolve(NETBOX_URL=url) + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_https_needs_no_notice(self): + config = _resolve(NETBOX_URL="https://netbox.example.com") + + assert not any("NETBOX_URL" in notice for notice in config.notices), config.notices + + +class TestCleartextCheckMatchesWhatRequestsWillDo: + """The notice is worthless if the URL it parses is not the URL the token is sent to.""" + + def test_a_backslash_authority_is_not_treated_as_loopback(self): + """The host reads as localhost here, but requests targets the address before the backslash.""" + config = _resolve(NETBOX_URL="http://198.18.0.1\\@localhost") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices + + def test_an_unparseable_url_is_reported_not_raised(self): + """Urlparse raises on some authorities; an advisory notice must not abort the run.""" + config = _resolve(NETBOX_URL="http://user[foo]@198.18.0.1") + + assert any("NETBOX_URL" in notice for notice in config.notices), config.notices diff --git a/tests/test_export_manifest.py b/tests/test_export_manifest.py index 8ffeb796..c646a241 100644 --- a/tests/test_export_manifest.py +++ b/tests/test_export_manifest.py @@ -93,11 +93,10 @@ class TestIsEntryFresh: """Tests for is_entry_fresh function.""" def test_fresh_when_last_updated_matches(self): - manifest = { - "device-types": {"Nokia/acme-x": {"last_updated": "2024-01-01T00:00:00Z"}}, - "module-types": {}, - "rack-types": {}, - } + """Built through update_entry: a hand-written entry cannot carry the schema revision.""" + manifest = {"device-types": {}, "module-types": {}, "rack-types": {}} + update_entry(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") + assert is_entry_fresh(manifest, "device-types", "Nokia/acme-x", "2024-01-01T00:00:00Z") is True def test_stale_when_last_updated_differs(self): @@ -125,3 +124,32 @@ def test_updates_existing_entry(self): manifest = {"device-types": {"Nokia/acme-x": {"last_updated": "old"}}, "module-types": {}, "rack-types": {}} update_entry(manifest, "device-types", "Nokia/acme-x", "2024-02-01T00:00:00Z") assert manifest["device-types"]["Nokia/acme-x"]["last_updated"] == "2024-02-01T00:00:00Z" + + +class TestExportSchemaRevision: + """last_updated alone cannot see a change in what the exporter writes.""" + + def test_an_entry_from_an_older_exporter_is_not_fresh(self): + """The record did not change, but the serialized shape did, so it must be rewritten.""" + from core.export_manifest import is_entry_fresh + + manifest = {"device-types": {"Acme/x": {"last_updated": "2026-01-01T00:00:00Z"}}} + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is False + + def test_an_entry_this_exporter_wrote_is_fresh(self): + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") is True + + def test_a_changed_record_is_still_not_fresh(self): + """The revision must not paper over the timestamp check it sits beside.""" + from core.export_manifest import is_entry_fresh, update_entry + + manifest = {"device-types": {}} + update_entry(manifest, "device-types", "Acme/x", "2026-01-01T00:00:00Z") + + assert is_entry_fresh(manifest, "device-types", "Acme/x", "2026-02-02T00:00:00Z") is False diff --git a/tests/test_exporter.py b/tests/test_exporter.py index 18c06bf2..8e063884 100644 --- a/tests/test_exporter.py +++ b/tests/test_exporter.py @@ -239,6 +239,119 @@ def test_yaml_equal_normalizes_component_order_and_numbers(self): class TestRepoSupersedes: """Tests for _repo_supersedes / _is_subset (asymmetric containment).""" + def test_an_empty_relation_does_not_make_every_definition_differ(self): + """The serializer omits an empty relation, and this is why it has to. + + _is_subset requires every NetBox leaf to be present in the repo YAML, and + _normalize_for_compare does not drop empty lists. A serialized + "module_bay_types: []" would therefore be absent from every library definition + and re-export the whole library on a NetBox 4.7 server. + """ + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +module-bays: + - {name: Slot 0, position: '0'} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + components = {1: {"module_bay_templates": [bay]}} + serialized = serialize_device_type(record, components) + + assert serialized["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert _repo_supersedes(repo, serialized), "an unchanged definition must not re-export" + bay.module_bay_types = [DotDict(name="Control")] + populated = serialize_device_type(record, components) + assert populated["module-bays"][0]["module_bay_types"] == ["Control"] + assert not _repo_supersedes(repo, populated), "a new restriction must trigger export" + + def test_a_default_positions_does_not_make_every_front_port_differ(self): + """The export writes the schema-required positions; it must not re-export the library. + + A library entry that omits positions means the default, 1. _is_subset requires every + NetBox leaf to be present in the repo YAML, so a serialized positions: 1 compared + against an entry that omits it would report every such definition as differing. + """ + from types import SimpleNamespace + from core.nb_serializer import _serialize_front_port + + legacy = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + serialized = _serialize_front_port(legacy) + assert serialized == {"name": "FP1", "type": "8p8c", "positions": 1} + + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [serialized]} + + assert _repo_supersedes(repo, nb), "an omitted positions is the default, not a difference" + + def test_a_non_default_positions_still_differs(self): + """Only the default may be treated as absent, or a real change would be suppressed.""" + repo = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c"}]} + nb = {"model": "PP", "front-ports": [{"name": "FP1", "type": "8p8c", "positions": 4}]} + + assert _repo_supersedes(repo, nb) is False + + @pytest.mark.parametrize("reverse_repo", [False, True]) + @pytest.mark.parametrize("reverse_netbox", [False, True]) + def test_reordered_netbox_mappings_do_not_trigger_export(self, reverse_repo, reverse_netbox): + from core.graphql_client import DotDict + from core.nb_serializer import serialize_device_type + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Panel +slug: acme-panel +u_height: 1 +is_full_depth: false +front-ports: + - {name: FP1, type: lc-upc, positions: 2} +port-mappings: + - {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} + - {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 3} +""") + record = DotDict( + id=1, manufacturer=DotDict(name="Acme"), model="Panel", slug="acme-panel", u_height=1, is_full_depth=False + ) + port = DotDict( + name="FP1", + type="lc-upc", + positions=2, + mappings=[ + DotDict(rear_port=DotDict(name="RP2"), front_port_position=2, rear_port_position=3), + DotDict(rear_port=DotDict(name="RP1"), front_port_position=1, rear_port_position=1), + ], + ) + if reverse_repo: + repo["port-mappings"].reverse() + if reverse_netbox: + port.mappings.reverse() + serialized = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + + assert _repo_supersedes(repo, serialized), "unchanged mappings must not trigger export" + repo["port-mappings"].append(dict(repo["port-mappings"][0])) + assert not _repo_supersedes(repo, serialized), "duplicate mappings must keep their multiplicity" + repo["port-mappings"].pop() + port.mappings.append(port.mappings[0]) + duplicated = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, duplicated), "NetBox duplicates must keep their multiplicity" + port.mappings.pop() + port.mappings[0].rear_port_position = 4 + changed = serialize_device_type(record, {1: {"front_port_templates": [port]}}) + assert not _repo_supersedes(repo, changed) + def test_equal_dicts(self): repo = {"manufacturer": "Nokia", "model": "X", "u_height": 1} nb = {"manufacturer": "Nokia", "model": "X", "u_height": 1} @@ -853,11 +966,17 @@ def test_fetch_vendor_components_groups_device_and_module_records(self, tmp_path def _side_effect(endpoint_name, manufacturer_slug=None): return [dt_rec, mt_rec] if endpoint_name == "interface_templates" else [] + # A worker must fetch through its own clone, so only the clone answers. + worker = MagicMock() + worker.get_component_templates.side_effect = _side_effect mock_client = MagicMock() - mock_client.get_component_templates.side_effect = _side_effect - with patch("core.export.NetBoxGraphQLClient", return_value=mock_client): - dt_result, mt_result = exporter._fetch_vendor_components("nokia") + mock_client.get_component_templates.side_effect = AssertionError("worker must use clone()") + mock_client.clone.return_value = worker + exporter.graphql = mock_client + dt_result, mt_result = exporter._fetch_vendor_components("nokia") + + assert mock_client.clone.called assert dt_result[11]["interface_templates"] == [dt_rec] assert mt_result[22]["interface_templates"] == [mt_rec] @@ -1539,3 +1658,146 @@ def test_a_stray_file_named_like_a_library_directory_stops_the_export(self, tmp_ with pytest.raises(FileNotFoundError, match="No device-type library found"): self._exporter(tmp_path, repo)._verify_repo_available() + + +class TestModuleBayPositionWarning: + """NetBox allows a blank module-bay position; the DTL schema requires one.""" + + @staticmethod + def _item(module_bays, kind="device-type"): + return ExportItem( + kind=kind, + nb_record=_make_dt(), + repo_yaml=None, + serialized={"model": "7750-SR-7s", "module-bays": module_bays}, + reason="absent", + mfr_name="Nokia", + filename="nokia-7750-sr-7s.yaml", + manifest_key="Nokia/nokia-7750-sr-7s", + ) + + @staticmethod + def _write(tmp_path, item): + """Drive the real write path with a real LogHandler, which prints to stdout.""" + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + exporter._get_module_image_details = dict + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + def test_a_bay_without_a_position_is_named_in_the_log(self, tmp_path, capsys): + item = self._item([{"name": "Slot 0"}, {"name": "Slot 1", "position": "1"}]) + + self._write(tmp_path, item) + + out = capsys.readouterr().out + assert "Slot 0" in out + assert "position" in out + assert "nokia-7750-sr-7s.yaml" in out + assert "Slot 1" not in out, "a bay that has a position is not a problem" + + def test_a_bay_positioned_at_zero_is_not_reported(self, tmp_path, capsys): + """'0' is a real position: the MX304 PSU bays use '0' and '1'.""" + item = self._item([{"name": "Slot 0", "position": "0"}]) + + self._write(tmp_path, item) + + # Assert the file was written too: silence alone would also hold if nothing ran. + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert "no position" not in capsys.readouterr().out + + def test_a_type_with_no_module_bays_reports_nothing(self, tmp_path, capsys): + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), False, None) + item = replace(self._item([]), serialized={"model": "7750-SR-7s"}) + + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Nokia" / item.filename).read_text()) + assert written == {"model": "7750-SR-7s"}, "the write path ran, it simply had nothing to warn about" + assert "no position" not in capsys.readouterr().out + + def test_a_module_type_bay_is_checked_too(self, tmp_path, capsys): + """Device types and module types share the module-bay schema definition.""" + item = self._item([{"name": "Sub 0"}], kind="module-type") + + self._write(tmp_path, item) + + assert "Sub 0" in capsys.readouterr().out + + +class TestUnqueriedRelationsSurviveTheExport: + """A pre-4.7 server never returns module_bay_types, so the export must not strip it.""" + + @staticmethod + def _item(repo_yaml, serialized): + return ExportItem( + kind="device-type", + nb_record=_make_dt(), + repo_yaml=repo_yaml, + serialized=serialized, + reason="differs", + mfr_name="Juniper", + filename="mx304.yaml", + manifest_key="Juniper/mx304", + ) + + def _write(self, tmp_path, item, supported): + exporter = Exporter(_make_settings(tmp_path), _make_handle(), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = supported + exporter._get_module_image_details = dict + exporter._write_export_items([item], {}, tmp_path / "manifest.json", None) + return yaml.safe_load((tmp_path / "extra" / "device-types" / "Juniper" / "mx304.yaml").read_text()) + + def test_a_bay_relation_survives_when_the_server_cannot_return_it(self, tmp_path): + """Only the description changed; the relation must not be collateral damage.""" + repo = { + "model": "MX304", + "description": "old", + "module-bays": [{"name": "RE0", "position": "0", "module_bay_types": ["MX304-RE"]}], + } + serialized = {"model": "MX304", "description": "new", "module-bays": [{"name": "RE0", "position": "0"}]} + + written = self._write(tmp_path, self._item(repo, serialized), supported=False) + + assert written["description"] == "new", "the real change still lands" + assert written["module-bays"][0]["module_bay_types"] == ["MX304-RE"] + + def test_a_server_that_can_return_it_stays_authoritative(self, tmp_path): + """An export selected for a changed description must retain the supported server's answer.""" + from core.graphql_client import DotDict + + repo = yaml.safe_load(""" +manufacturer: Acme +model: Chassis +slug: acme-chassis +u_height: 1 +is_full_depth: false +description: old +module-bays: + - {name: Slot 0, position: '0', module_bay_types: [Control]} +""") + record = DotDict( + id=1, + manufacturer=DotDict(name="Acme", slug="acme"), + model="Chassis", + slug="acme-chassis", + u_height=1, + is_full_depth=False, + description="new", + front_image=None, + rear_image=None, + last_updated="2026-01-01T00:00:00Z", + ) + bay = DotDict(name="Slot 0", position="0", module_bay_types=[]) + exporter = Exporter(_make_settings(tmp_path), LogHandler(False), str(tmp_path / "extra"), True, None) + exporter.graphql.supports_module_bay_types = True + items = exporter._determine_export_set_for_device_types( + [record], {("acme", "acme-chassis"): repo}, {1: {"module_bay_templates": [bay]}} + ) + + assert len(items) == 1 + assert items[0].reason == "differs" + exporter._write_export_items(items, {}, tmp_path / "manifest.json", None) + written = yaml.safe_load((tmp_path / "extra" / "device-types" / "Acme" / "Chassis.yaml").read_text()) + assert written["description"] == "new" + assert written["module-bays"] == [{"name": "Slot 0", "position": "0"}] + assert "module_bay_types" not in written["module-bays"][0] diff --git a/tests/test_graphql_client.py b/tests/test_graphql_client.py index 6c5b22b5..378a83e1 100644 --- a/tests/test_graphql_client.py +++ b/tests/test_graphql_client.py @@ -118,22 +118,153 @@ def test_logging_dependency_has_a_public_read_only_handle(self): client.handle = LogHandler(True) def test_clone_uses_an_independent_session_with_the_same_settings(self): + """Every stored setting is compared, so a new one cannot be dropped unnoticed. + + clone() re-lists constructor arguments by hand. Listing the settings here by hand + too let supports_module_bay_types default back to False in every cloned worker. + """ handle = MagicMock() sessions = [MagicMock(), MagicMock()] with patch("core.graphql_client.requests.Session", side_effect=sessions): - client = NetBoxGraphQLClient("http://netbox.local", "token", True, handle, 250) + client = NetBoxGraphQLClient( + "http://netbox.local", "token", True, handle, 250, supports_module_bay_types=True + ) clone = client.clone() assert clone is not client assert clone._session is sessions[1] - assert (clone.url, clone.token, clone.ignore_ssl, clone.handle, clone.DEFAULT_PAGE_SIZE) == ( - client.url, - client.token, - client.ignore_ssl, - client.handle, - client.DEFAULT_PAGE_SIZE, - ) + settings = lambda c: {k: v for k, v in vars(c).items() if k != "_session"} # noqa: E731 + assert settings(clone) == settings(client) + + @pytest.mark.real_http + def test_the_cloned_session_is_configured_not_merely_new(self): + """A bare requests.Session() would be independent and completely unauthenticated. + + Marked real_http only to get real Session objects; nothing here sends a request. + """ + client = NetBoxGraphQLClient("http://netbox.local", "nbt_key.secret", ignore_ssl=True) + + clone = client.clone() + + assert clone._session is not client._session + assert clone._session.headers["Authorization"] == "Bearer nbt_key.secret" + assert clone._session.headers["Content-Type"] == "application/json" + assert clone._session.verify is False + + @pytest.mark.real_http + def test_a_v1_token_clone_keeps_the_legacy_auth_scheme(self): + client = NetBoxGraphQLClient("http://netbox.local", "0123456789abcdef") + + assert client.clone()._session.headers["Authorization"] == "Token 0123456789abcdef" + + @pytest.mark.real_http + def test_an_unreachable_server_fails_the_probe_as_a_graphql_error(self): + """It is the export's first request, so a raw transport error escapes as a traceback.""" + from core.graphql_client import GraphQLError + from helpers import FakeNetBox + + server = FakeNetBox() + url = server.url + server.close() # nothing is listening on that port any more + + with pytest.raises(GraphQLError): + NetBoxGraphQLClient(url, "tok").detect_module_bay_type_support() + + @pytest.mark.real_http + def test_a_non_json_status_body_fails_the_probe_as_a_graphql_error(self): + """A proxy error page answers 200 with HTML; json() then raises ValueError.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"gateway" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + @pytest.mark.parametrize( + ("body", "why"), + [ + (b"[]", "a JSON list has no .get, so the probe raised a bare AttributeError"), + (b"{}", "no netbox-version read as '' and silently disabled the relation"), + (b'{"netbox-version": ""}', "an empty version is not a version"), + (b'{"netbox-version": null}', "a null version is not a version"), + ], + ) + def test_a_wrong_shaped_status_body_fails_the_probe(self, body, why): + """Silently deciding 'unsupported' on a 4.7 server exports without the relation.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + from core.graphql_client import GraphQLError + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + """Silence the default stderr access log.""" + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + client = NetBoxGraphQLClient(f"http://127.0.0.1:{server.server_port}", "tok") + with pytest.raises(GraphQLError): + client.detect_module_bay_type_support() + finally: + server.shutdown() + server.server_close() + + @pytest.mark.real_http + def test_the_version_probe_decides_whether_the_relation_may_be_selected(self): + """Export has no pynetbox client, so it asks NetBox here and both sides use compat.""" + from helpers import FakeNetBox + + for version, expected in (("4.7.0", True), ("4.7.0-beta2", True), ("4.6.9", False), ("4.3.7", False)): + server = FakeNetBox(netbox_version=version) + try: + client = NetBoxGraphQLClient(server.url, "tok") + assert client.detect_module_bay_type_support() is expected, version + assert client.supports_module_bay_types is expected + finally: + server.close() + + def test_clone_carries_a_setting_it_does_not_name(self): + """clone() must not enumerate settings: the one it forgets is the one that breaks. + + supports_module_bay_types was lost exactly that way, and adding it to the argument + list only fixes the setting that was already missed. + """ + with patch("core.graphql_client.requests.Session"): + client = NetBoxGraphQLClient("http://netbox.local", "token") + client.added_after_this_test_was_written = "carried" + clone = client.clone() + + assert clone.added_after_this_test_was_written == "carried" def test_init_stores_config(self): from core.graphql_client import NetBoxGraphQLClient @@ -1025,6 +1156,104 @@ def _make_client(self): return NetBoxGraphQLClient("http://netbox.local", "tok") + @pytest.mark.real_http + @pytest.mark.parametrize("caller", ["graphql_relation_fields", "get_module_types"]) + def test_relation_selections_use_the_shared_helper(self, caller): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + expected = "module_bay_types { id name slug manufacturer { slug } }" + if caller == "graphql_relation_fields": + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [expected] + else: + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + assert expected in " ".join(query.split()) + finally: + server.close() + + @pytest.mark.real_http + def test_module_type_query_contains_the_complete_relation_selection(self): + from core.component_registry import BY_ENDPOINT + from helpers import FakeNetBox + + server = FakeNetBox() + try: + client = NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True) + assert client.get_module_types() == {} + query = server.sent("POST", "graphql")[0]["query"] + finally: + server.close() + + selection = "module_bay_types { id name slug manufacturer { slug } }" + assert BY_ENDPOINT["module_bay_templates"].graphql_relation_fields == [selection] + assert " ".join(query.split()) == " ".join( + """ +query($pagination: OffsetPaginationInput) { + module_type_list(pagination: $pagination) { + id model part_number airflow description comments weight weight_unit last_updated + module_bay_types { id name slug manufacturer { slug } } + manufacturer { id name slug } + } +} +""".split() + ) + + def test_a_clone_still_selects_the_module_bay_type_relation(self, mock_post): + """The prefetch runs on clones, not on the client it was cloned from. + + A clone that drops the relation reads every bay without it, and the comparison + then skips a field it never received. + """ + from core.graphql_client import NetBoxGraphQLClient + + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=True) + mock_post.side_effect = _make_paged_responses({"module_bay_template_list": []}, "module_bay_template_list") + + client.clone().get_component_templates("module_bay_templates") + + queries = [call.kwargs["json"]["query"] for call in mock_post.call_args_list] + assert queries and all("module_bay_types" in q for q in queries) + + def test_the_module_type_query_selects_the_relation_only_when_supported(self, mock_post): + """The selection has to track the server, in both directions. + + An unselected field reads as missing and is skipped by the comparison, and + selecting it on a server below 4.7 fails the whole query. + """ + from core.graphql_client import NetBoxGraphQLClient + + def _query_for(supported): + client = NetBoxGraphQLClient("http://netbox.local", "tok", supports_module_bay_types=supported) + mock_post.reset_mock() + mock_post.side_effect = _make_paged_responses({"module_type_list": []}, "module_type_list") + client.get_module_types() + return mock_post.call_args_list[0].kwargs["json"]["query"] + + assert "module_bay_types" in _query_for(True) + assert "module_bay_types" not in _query_for(False) + + def test_the_front_port_fallback_drops_every_45_only_field(self, mock_post): + """Positions arrived with the mapping model, so a pre-4.5 tier must not ask for it. + + Leaving it in a fallback tier makes every tier fail the same way, and the whole + front-port preload dies on a server that would answer the older shape. + """ + from core.component_registry import BY_ENDPOINT + from core.graphql_client import NetBoxGraphQLClient + + fields = list(BY_ENDPOINT["front_port_templates"].graphql_fields) + assert "positions" in fields, "guard: the registry is expected to select positions" + + tiers = list(NetBoxGraphQLClient._front_port_field_variants(fields)) + + assert any("positions" in f for f in tiers[0]), "the 4.5 tier keeps it" + for tier in tiers[1:]: + assert not any("positions" in f for f in tier), f"pre-4.5 tier still asks for it: {tier}" + def test_returns_dotdict_records_with_parent_info(self, mock_post): """Records should be DotDicts with device_type/module_type and correct id types.""" data = { diff --git a/tests/test_import_run.py b/tests/test_import_run.py index 1aa14152..9c9d5694 100644 --- a/tests/test_import_run.py +++ b/tests/test_import_run.py @@ -59,7 +59,6 @@ class _NetBoxBoundary: """Expose run state and fail if planning starts an import.""" def __init__(self): - self.modules = True self.rack_types = True self.device_types = _DeviceTypes() self.outcomes = _Outcomes() @@ -271,8 +270,6 @@ def test_execute_returns_snapshot_and_owns_console_lifecycle(make_config, tmp_pa assert isinstance(summary, RunSummary) assert summary.counter["added"] == 2 - assert summary.modules is True - assert summary.rack_types is True assert progress_factory.entered is True assert progress_factory.exited is True assert handle.console is None @@ -434,11 +431,9 @@ def test_banners_omit_the_separator_when_no_vendor_is_given(make_config): class _OutcomeNetBox: """Run-state stand-in carrying a real OutcomeRegistry.""" - def __init__(self, *, modules=True, rack_types=True, counter=None): + def __init__(self, *, counter=None): from core.outcomes import OutcomeRegistry - self.modules = modules - self.rack_types = rack_types self.outcomes = OutcomeRegistry() base = dict( added=0, diff --git a/tests/test_module_bay_type_sync.py b/tests/test_module_bay_type_sync.py new file mode 100644 index 00000000..afe27626 --- /dev/null +++ b/tests/test_module_bay_type_sync.py @@ -0,0 +1,577 @@ +"""Tests for keeping module bay type assignments in sync with NetBox. + +Resolving a reference name to an id is :mod:`core.module_bay_types` and is tested there. +This file covers the wiring around it: the relation a module type carries itself, the +component update path, and the gate that keeps the field off a server that predates it. + +Both sides are real. The catalog is written to disk and read by a real +``ModuleBayTypeCatalog``; a local HTTP server answers a real ``pynetbox`` client and the +real GraphQL client, so the payloads asserted on here are the ones that would go over the +wire. Only the version handshake is stood in for, to fix the server release under test. +""" + +import pynetbox +import pytest + +from core.change_detector import ChangeType, ComponentChange, PropertyChange +from core.component_registry import BY_YAML_KEY +from core.graphql_client import NetBoxGraphQLClient +from core.module_bay_types import ModuleBayTypeCatalog +from core.netbox_api import DeviceTypes, NetBox +from core.outcomes import EntityKind, Outcome +from helpers import FakeNetBox, recording_handle, write_module_bay_type + +# The suite patches requests.Session by default, which would stop every request below. +pytestmark = pytest.mark.real_http + +JUNIPER = {"id": 1, "name": "Juniper", "slug": "juniper"} + + +def stub(**attrs): + """Return an object carrying exactly *attrs*, the way NetBox nests a related object.""" + return type("Stub", (), attrs)() + + +def created_id(server, slug): + """Return the id the server gave the module bay type it created for *slug*.""" + return next(record["id"] for record in server.collection("module_bay_types") if record["slug"] == slug) + + +def related(name, slug=None, manufacturer_slug=None): + """Return a module bay type as NetBox returns it nested inside a relation.""" + return stub(name=name, slug=slug, manufacturer=stub(slug=manufacturer_slug)) + + +class NetBoxRecord: + """An existing NetBox object as the importer holds it: an id, plus what was fetched. + + A field the query did not ask for is genuinely absent, which is the difference the + comparison turns on, so this carries only what a test hands it. + """ + + def __init__(self, record_id, **fields): + """Store the id and the fields this record is meant to have.""" + self.id = record_id + for name, value in fields.items(): + setattr(self, name, value) + + +@pytest.fixture +def catalog_root(tmp_path): + """Write the catalog these tests resolve against and return its root.""" + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot") + return root + + +@pytest.fixture +def server(): + """Run a local NetBox-shaped server seeded with the manufacturers the catalog needs.""" + fake = FakeNetBox(manufacturers=[JUNIPER]) + yield fake + fake.close() + + +@pytest.fixture +def make_device_types(server, catalog_root): + """Build a real DeviceTypes talking to the local server, with its cache already primed.""" + + def _make(module_bay_types_supported=True): + handle, console = recording_handle() + device_types = DeviceTypes( + server.api(), + handle, + {}, + False, + graphql=NetBoxGraphQLClient(server.url, "test-token", supports_module_bay_types=True), + repo_path=str(catalog_root), + module_bay_types_supported=module_bay_types_supported, + ) + device_types.components.ensure_ready() + return device_types, console + + return _make + + +@pytest.fixture +def netbox(make_config, mock_pynetbox, server, catalog_root): + """Build a real NetBox against the local server, reporting the release that has the feature.""" + mock_pynetbox.api.return_value.version = "4.7" + mock_pynetbox.RequestError = pynetbox.RequestError + handle, console = recording_handle() + config = make_config(netbox_url=server.url, repo_path=str(catalog_root)) + nb = NetBox(config, handle) + assert nb.module_bay_types, "a 4.7 server supports module bay types; the rest of this file assumes it" + + # connect_api ran against the patched pynetbox; every call under test uses a real client. + api = server.api() + nb.netbox = api + nb.device_types.netbox = api + nb.device_types.components.netbox = api + nb.device_types.components.ensure_ready() + return nb, console + + +class TestModuleTypeOwnRelation: + """A module type says which classes it belongs to, and that has to stay in sync.""" + + def test_a_missing_class_is_reported_as_a_change(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", [], ["MX304-RE"])] + + def test_the_right_class_is_left_alone(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_the_same_name_from_the_wrong_scope_is_corrected(self, netbox): + """Juniper owns MX304-RE. A Generic object of that name is not the one referenced.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "generic")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-RE"], ["MX304-RE"])] + + def test_names_are_compared_when_netbox_returned_no_identity(self, netbox): + """A record carrying only a name cannot answer the scope question; names still can.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_unresolvable_reference_is_reported_even_when_the_name_matches(self, netbox): + """A name that happens to match must not make an unresolvable reference look applied. + + Comparison cannot say which object the name means, so the write path has to try, + fail, and refuse. Reporting no change here skips that entirely. + """ + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["NO-SUCH-CLASS"]} + + assert nb._type_relation_changes(module_type, existing) == [ + ("module_bay_types", ["NO-SUCH-CLASS"], ["NO-SUCH-CLASS"]) + ] + + def test_a_matching_but_unresolvable_name_does_not_read_as_updated(self, netbox, server): + """The end of that path: the module type is refused, not patched with its scalars.""" + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[related("NO-SUCH-CLASS", "no-such", "juniper")], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + assert nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") == (False, False) + assert not server.sent("PATCH", "module_types") + + def test_a_field_the_query_did_not_return_is_skipped(self, netbox): + """Reading an absent field as empty would report a change on every run.""" + nb, _ = netbox + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, NetBoxRecord(7)) == [] + + def test_an_omitted_key_leaves_the_relation_unmanaged(self, netbox): + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + + assert nb._type_relation_changes({"model": "JNP304-RE"}, existing) == [] + + def test_a_malformed_reference_leaves_the_relation_unmanaged(self, netbox): + """A bare key parses as None; clearing on it would drop a restriction nobody removed.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-RE", "mx304-re", "juniper")]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": None} + + assert nb._type_relation_changes(module_type, existing) == [] + + def test_names_are_compared_when_the_definition_names_no_manufacturer(self, netbox): + """Without an owning manufacturer there is no scope, so the name comparison stands.""" + nb, _ = netbox + existing = NetBoxRecord(7, module_bay_types=[related("MX304-LMIC", "mx304-lmic", "juniper")]) + module_type = {"model": "JNP304-RE", "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [("module_bay_types", ["MX304-LMIC"], ["MX304-RE"])] + + def test_an_older_server_reports_no_relation_changes(self, netbox): + """Below 4.7 the relation does not exist, so there is nothing to compare.""" + nb, _ = netbox + nb.module_bay_types = False + existing = NetBoxRecord(7, module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._type_relation_changes(module_type, existing) == [] + + +class TestModuleTypeCreatePayload: + """What the create path sends for a module type's own relation.""" + + def test_names_are_replaced_by_ids(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + resolved = nb._resolve_type_relations(payload) + + created = server.sent("POST", "module_bay_types") + assert [p["slug"] for p in created] == ["mx304-re"] + assert resolved["module_bay_types"] == [created_id(server, "mx304-re")] + assert resolved["model"] == "JNP304-RE" + + def test_a_payload_without_the_relation_is_untouched(self, netbox, server): + nb, _ = netbox + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + + assert nb._resolve_type_relations(payload) == payload + assert not server.sent("POST", "module_bay_types") + + def test_an_older_server_never_sees_the_field(self, netbox, server): + """Sending a field the server does not have would fail the whole create.""" + nb, _ = netbox + nb.module_bay_types = False + payload = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + assert nb._resolve_type_relations(payload) == {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}} + assert not server.sent("POST", "module_bay_types") + + +class TestModuleTypeUpdate: + """The update path patches the relation, and survives one it cannot resolve.""" + + def test_a_changed_relation_is_patched_as_ids(self, netbox, server): + nb, _ = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = {"model": "JNP304-RE", "manufacturer": {"slug": "juniper"}, "module_bay_types": ["MX304-RE"]} + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (True, True) + assert server.sent("PATCH", "module_types") == [{"id": 7, "module_bay_types": [created_id(server, "mx304-re")]}] + + def test_an_unresolvable_reference_is_logged_and_the_run_goes_on(self, netbox, server): + """One bad module type must not end the run, and must not be written without it.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord(7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[]) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False), "an unapplied relation must not read as success" + assert not server.sent("PATCH", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + def test_a_scalar_change_is_held_back_when_the_relation_cannot_resolve(self, netbox, server): + """Patching the description while dropping the restriction is a half-applied write.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, + model="JNP304-RE", + manufacturer=stub(name="Juniper"), + module_bay_types=[], + description="old", + ) + module_type = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + ok, updated = nb._try_update_module_type(module_type, existing, "juniper/jnp304-re.yaml") + + assert (ok, updated) == (False, False) + assert not server.sent("PATCH", "module_types"), "the scalar PATCH must not go out alone" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestModuleTypeCreateRefusal: + """A module type whose relation cannot be resolved is reported, not written without it.""" + + def test_an_unresolvable_reference_skips_the_module_type(self, netbox, server): + nb, console = netbox + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "module_bay_types": ["NO-SUCH-CLASS"], + } + + created = nb._process_single_module_type(curr_mt, "juniper/jnp304-re.yaml", {}, {}, only_new=False) + + assert created is False + assert not server.sent("POST", "module_types") + assert any("NO-SUCH-CLASS" in line for line in console.lines) + # The registry is the only tally of failures; a path that merely logs is absent + # from the run summary and from the itemised report. + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [(r.kind, r.identity) for r in failed] == [(EntityKind.MODULE_TYPE, "juniper/JNP304-RE")] + assert "NO-SUCH-CLASS" in failed[0].reason + + +class TestModuleTypeUpdateOutcome: + """The run summary must show the module type whose relation could not be applied.""" + + def test_an_unresolvable_relation_is_recorded_as_a_failure(self, netbox, server): + """Driven through the parent operation, because that is where the outcome is recorded.""" + nb, console = netbox + server.collection("module_types").append({"id": 7, "model": "JNP304-RE"}) + existing = NetBoxRecord( + 7, model="JNP304-RE", manufacturer=stub(name="Juniper"), module_bay_types=[], description="old" + ) + curr_mt = { + "model": "JNP304-RE", + "manufacturer": {"slug": "juniper"}, + "description": "new", + "module_bay_types": ["NO-SUCH-CLASS"], + } + + nb._process_single_module_type( + curr_mt, "juniper/jnp304-re.yaml", {"juniper": {"JNP304-RE": existing}}, {}, only_new=False + ) + + failed = [r for r in nb.outcomes.records if r.outcome is Outcome.FAILED] + assert [r.kind for r in failed] == [EntityKind.MODULE_TYPE], "the failure must reach the run summary" + assert not server.sent("PATCH", "module_types"), "nothing may be written for a type it could not apply" + assert any("NO-SUCH-CLASS" in line for line in console.lines) + + +class TestComponentCreatePayload: + """A module bay is created with the restriction its definition asked for, or not at all.""" + + def test_names_are_replaced_by_ids(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert resolved == [{"name": "FPC 0", "module_bay_types": [created_id(server, "mx304-lmic")]}] + + def test_an_item_without_the_field_passes_through(self, make_device_types, server): + device_types, _ = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == items + assert not server.sent("POST", "module_bay_types") + + def test_an_unresolvable_restriction_drops_the_bay_rather_than_relaxing_it(self, make_device_types, server): + """Creating the bay without its restriction would silently accept any module.""" + device_types, console = make_device_types() + component = BY_YAML_KEY["module-bays"] + items = [ + {"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}, + {"name": "FPC 1", "module_bay_types": ["MX304-LMIC"]}, + ] + + with device_types.collect_component_errors() as collected: + resolved = device_types._resolve_relations(component, items, {"slug": "juniper"}) + + assert [item["name"] for item in resolved] == ["FPC 1"] + # Collected, not just printed: the parent's outcome reason is built from these. + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + assert any("FPC 0" in line for line in console.lines) + + def test_an_older_server_never_sees_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + component = BY_YAML_KEY["module-bays"] + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(component, items, {"slug": "juniper"}) == [{"name": "FPC 0"}] + assert not server.sent("POST", "module_bay_types") + + def test_a_netbox_rejection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A 403 while resolving must not escape the per-component recovery. + + The callers recover from ModuleBayTypeError only, so a raw RequestError ends the + run with a traceback and leaves a partly created parent behind. + """ + device_types, _ = make_device_types() + server.errors["module_bay_types"] = 403 + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e and "403" in e] + + def test_a_lost_connection_skips_the_bay_instead_of_ending_the_run(self, make_device_types, server): + """A dropped connection is not a RequestError, so it escaped the same recovery.""" + device_types, _ = make_device_types() + server.close() # the port stops answering; resolution now hits a refused connection + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + with device_types.collect_component_errors() as collected: + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, {"slug": "juniper"}) == [] + assert [e for e in collected if "FPC 0" in e] + + def test_a_component_kind_with_no_relations_is_untouched(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "xe-0/0/0", "type": "100gbase-x-qsfp28"}] + + assert device_types._resolve_relations(BY_YAML_KEY["interfaces"], items, {"slug": "juniper"}) == items + + def test_an_unknown_manufacturer_drops_the_bay_rather_than_sending_names(self, make_device_types): + """Without a scope the names cannot become ids, and NetBox rejects raw names.""" + device_types, console = make_device_types() + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [] + assert any("FPC 0" in line for line in console.lines) + + def test_an_unknown_manufacturer_still_passes_through_a_bay_with_no_restriction(self, make_device_types): + device_types, _ = make_device_types() + items = [{"name": "FPC 0"}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == items + + def test_an_older_server_strips_the_field_even_without_a_manufacturer(self, make_device_types): + """The manufacturer question must not decide whether an unsupported field is sent.""" + device_types, _ = make_device_types(module_bay_types_supported=False) + items = [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}] + + assert device_types._resolve_relations(BY_YAML_KEY["module-bays"], items, None) == [{"name": "FPC 0"}] + + +class TestComponentCreateWiring: + """Through create_components(), so the resolution is proved to be wired in, not just present.""" + + def test_a_created_bay_carries_resolved_ids(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert [p["name"] for p in posted] == ["FPC 0"] + assert posted[0]["module_bay_types"] == [created_id(server, "mx304-lmic")] + + def test_an_older_server_creates_the_bay_without_the_field(self, make_device_types, server): + device_types, _ = make_device_types(module_bay_types_supported=False) + device_types.components.record("module_bay_templates", "device", 3, {}) + + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + posted = server.sent("POST", "module_bay_templates") + assert posted and "module_bay_types" not in posted[0] + + def test_an_unresolvable_bay_is_never_posted(self, make_device_types, server): + device_types, _ = make_device_types() + device_types.components.record("module_bay_templates", "device", 3, {}) + + with device_types.collect_component_errors() as collected: + device_types.create_components( + "module-bays", + [{"name": "FPC 0", "module_bay_types": ["NO-SUCH-CLASS"]}], + 3, + manufacturer={"slug": "juniper"}, + ) + + assert not server.sent("POST", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e] + + +class TestComponentUpdatePayload: + """An existing bay whose restriction changed is patched with ids, not names.""" + + @staticmethod + def _change(name, new_value): + return ComponentChange( + component_type="module-bays", + component_name=name, + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange(property_name="module_bay_types", old_value=[], new_value=new_value)], + ) + + def test_a_changed_restriction_is_patched_as_ids(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + device_types._apply_updates_for_type( + "module-bays", [self._change("FPC 0", ["MX304-LMIC"])], {"manufacturer": {"slug": "juniper"}}, 3, "device" + ) + + assert server.sent("PATCH", "module_bay_templates") == [ + {"id": 55, "module_bay_types": [created_id(server, "mx304-lmic")]} + ] + + def test_a_rejected_patch_is_collected_rather_than_raised(self, make_device_types, server): + """The bay resolves; NetBox refuses the write. That must land in the entity's report.""" + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + server.errors["module_bay_templates"] = 400 + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["MX304-LMIC"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert [e for e in collected if "55" in e] + + def test_an_unresolvable_restriction_is_logged_and_nothing_is_patched(self, make_device_types, server): + device_types, _ = make_device_types() + server.collection("module_bay_templates").append({"id": 55, "name": "FPC 0"}) + device_types.components.record("module_bay_templates", "device", 3, {"FPC 0": NetBoxRecord(55)}) + + with device_types.collect_component_errors() as collected: + device_types._apply_updates_for_type( + "module-bays", + [self._change("FPC 0", ["NO-SUCH-CLASS"])], + {"manufacturer": {"slug": "juniper"}}, + 3, + "device", + ) + + assert not server.sent("PATCH", "module_bay_templates") + assert [e for e in collected if "FPC 0" in e and "NO-SUCH-CLASS" in e] + + +class TestCatalogWiring: + """The catalog is built once per run, from the library checkout the run is using.""" + + def test_the_catalog_is_built_from_the_repo_path_and_reused(self, make_device_types, catalog_root): + device_types, _ = make_device_types() + + catalog = device_types.module_bay_type_catalog + + assert isinstance(catalog, ModuleBayTypeCatalog) + assert device_types.module_bay_type_catalog is catalog + assert not hasattr(device_types, "module_bay_types") + assert catalog.identities_for("juniper", ["MX304-RE"]) == frozenset({("juniper", "mx304-re")}) diff --git a/tests/test_module_bay_types.py b/tests/test_module_bay_types.py new file mode 100644 index 00000000..82a2e4d1 --- /dev/null +++ b/tests/test_module_bay_types.py @@ -0,0 +1,340 @@ +"""Tests for module bay type reference resolution. + +Driven through the public interface (``ids_for`` / ``identities_for``) against catalog files +the tests write themselves and a real ``pynetbox`` client talking HTTP to a local server. +Nothing here is mocked: the client serialises, filters and paginates for real, which is +where the semantics that matter actually live. +""" + +import pytest + +from core.module_bay_types import ModuleBayCatalogError, ModuleBayTypeCatalog, ModuleBayTypeError +from helpers import FakeNetBox, write_module_bay_type + + +class Handle: + """Capture what the catalog logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +DEFAULT_MANUFACTURERS = ( + {"id": 1, "name": "Juniper", "slug": "juniper"}, + {"id": 2, "name": "Cisco", "slug": "cisco"}, + {"id": 3, "name": "Nokia", "slug": "nokia"}, +) + + +@pytest.fixture +def library(tmp_path): + """Write the catalog the resolution tests resolve against, and return its root. + + QSFP-DD is defined twice on purpose, by Juniper and by Generic, so the owner-scope + rule has two candidates to choose between. + """ + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "Juniper MX304 routing-engine slot compatibility") + write_module_bay_type(root, "Juniper", "mx304-lmic", "MX304-LMIC", "Juniper MX304 LMIC slot compatibility") + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "Juniper MX304 QSFP-DD cage") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "QSFP-DD pluggable transceiver form factor") + return root + + +@pytest.fixture +def catalog(library): + """Build a catalog wired to a real pynetbox client and a local NetBox-shaped server.""" + servers = [] + + def _make(module_bay_types=(), manufacturers=None, root=None): + server = FakeNetBox( + manufacturers=DEFAULT_MANUFACTURERS if manufacturers is None else manufacturers, + module_bay_types=module_bay_types, + ) + servers.append(server) + return ModuleBayTypeCatalog(server.api(), str(root or library), Handle()), server + + yield _make + for server in servers: + server.close() + + +@pytest.mark.real_http +class TestResolution: + """Names resolve to ids, in the owning manufacturer's scope and then in Generic.""" + + def test_resolves_in_owner_manufacturer_scope(self, catalog): + cat, server = catalog() + ids = cat.ids_for("juniper", ["MX304-RE"]) + assert len(ids) == 1 + created = server.sent("POST", "module_bay_types") + assert created == [ + { + "name": "MX304-RE", + "slug": "mx304-re", + "manufacturer": 1, + "description": "Juniper MX304 routing-engine slot compatibility", + } + ] + + def test_falls_back_to_generic_for_another_manufacturer(self, catalog): + """A Cisco optic reaches the Generic form factor, and Generic is created on demand.""" + cat, server = catalog() + assert len(cat.ids_for("cisco", ["QSFP-DD"])) == 1 + made = server.sent("POST", "manufacturers") + assert made == [{"name": "Generic", "slug": "generic"}] + + def test_owner_scope_wins_over_generic(self, catalog): + """Juniper and Generic both define QSFP-DD, and a Juniper reference means Juniper's.""" + cat, server = catalog() + cat.ids_for("juniper", ["QSFP-DD"]) + created = server.sent("POST", "module_bay_types") + assert [p["manufacturer"] for p in created] == [1] + assert not server.sent("POST", "manufacturers") + + def test_existing_object_is_reused_not_recreated(self, catalog): + cat, server = catalog( + module_bay_types=[{"id": 77, "name": "MX304-RE", "slug": "mx304-re", "manufacturer": {"id": 1}}] + ) + assert cat.ids_for("juniper", ["MX304-RE"]) == [77] + assert not server.sent("POST", "module_bay_types") + + def test_repeated_resolution_is_cached(self, catalog): + cat, server = catalog() + first = cat.ids_for("juniper", ["MX304-RE"]) + before = len(server.requests) + + assert cat.ids_for("juniper", ["MX304-RE"]) == first + assert len(first) == 1, "an empty result would make the request count meaningless" + assert len(server.requests) == before + + def test_order_is_not_meaningful(self, catalog): + cat, _ = catalog() + a = cat.ids_for("juniper", ["MX304-RE", "MX304-LMIC"]) + b = cat.ids_for("juniper", ["MX304-LMIC", "MX304-RE"]) + assert len(a) == 2 + assert sorted(a) == sorted(b) + + +@pytest.mark.real_http +class TestRefusals: + """An unresolved name and a conflicting identity are reported, never papered over.""" + + def test_unresolved_name_raises_rather_than_dropping(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["NO-SUCH-CLASS"]) + assert "NO-SUCH-CLASS" in str(exc.value) + + def test_same_name_different_slug_is_an_error_not_a_rename(self, catalog): + """The catalog says mx304-re; NetBox holds mx304_re. Never silently rename.""" + cat, server = catalog( + module_bay_types=[{"id": 88, "name": "MX304-RE", "slug": "mx304_re", "manufacturer": {"id": 1}}] + ) + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", ["MX304-RE"]) + assert "mx304_re" in str(exc.value) and "mx304-re" in str(exc.value) + assert not server.sent("POST", "module_bay_types") + + +@pytest.mark.real_http +class TestCatalogReading: + """The catalog is read from real files on disk, including the shapes that are rejected.""" + + def test_entry_without_a_description_is_created_without_one(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + cat, server = catalog(root=tmp_path) + cat.ids_for("generic", ["SFP"]) + created = server.sent("POST", "module_bay_types") + generic = next(m for m in server.collection("manufacturers") if m["slug"] == "generic") + assert created == [{"name": "SFP", "slug": "sfp", "manufacturer": generic["id"]}] + + def test_non_yaml_files_and_empty_documents_are_skipped(self, tmp_path, catalog): + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + (tmp_path / "module-bay-types" / "Generic" / "README.md").write_text("not a catalog entry\n") + (tmp_path / "module-bay-types" / "Generic" / "blank.yaml").write_text("# only a comment\n") + cat, _ = catalog(root=tmp_path) + assert len(cat.ids_for("generic", ["SFP"])) == 1 + + def test_a_yaml_document_that_is_not_a_mapping_is_refused(self, tmp_path, catalog): + """Silently skipping it shrinks the catalog, and a Generic entry then answers instead.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + directory = tmp_path / "module-bay-types" / "Juniper" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("- name: SFP\n slug: sfp\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("Juniper", ["SFP"]) + + def test_an_entry_missing_a_required_field_is_refused_at_load(self, tmp_path, catalog): + """A half-written entry must fail as a catalog error, not as a KeyError mid-run.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: SFP\nmanufacturer: Generic\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "slug" in str(exc.value) and "sfp.yaml" in str(exc.value) + + def test_unparseable_yaml_is_refused_as_a_catalog_error(self, tmp_path, catalog): + """A parser error escaping the boundary ends the run before it reports anything.""" + directory = tmp_path / "module-bay-types" / "Generic" + directory.mkdir(parents=True, exist_ok=True) + (directory / "sfp.yaml").write_text("name: [\n", encoding="utf-8") + cat, _ = catalog(root=tmp_path) + + with pytest.raises(ModuleBayCatalogError) as exc: + cat.identities_for("generic", ["SFP"]) + assert "sfp.yaml" in str(exc.value) + + def test_a_broken_catalog_is_read_once_not_on_every_lookup(self, tmp_path, catalog, monkeypatch): + """The load caches only on success, so a bad catalog re-walked the tree every time.""" + import core.module_bay_types as module + + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + + walks = [] + real_walk = module.os.walk + monkeypatch.setattr(module.os, "walk", lambda *a, **k: walks.append(1) or real_walk(*a, **k)) + + for _ in range(3): + with pytest.raises(ModuleBayCatalogError): + cat.identities_for("generic", ["SFP"]) + assert len(walks) == 1 + + def test_duplicate_scoped_entry_is_refused(self, tmp_path, catalog): + """Two files claiming the same (manufacturer, name) would make a reference ambiguous.""" + write_module_bay_type(tmp_path, "Generic", "sfp", "SFP") + write_module_bay_type(tmp_path, "Generic", "sfp-again", "SFP") + cat, _ = catalog(root=tmp_path) + with pytest.raises(ModuleBayCatalogError) as exc: + cat.ids_for("generic", ["SFP"]) + assert "Duplicate" in str(exc.value) and "SFP" in str(exc.value) + + +@pytest.mark.real_http +class TestMalformedReferences: + """A reference list that is not a list of names is refused, never read as empty.""" + + def test_a_bare_key_is_refused_rather_than_clearing(self, catalog): + """`module_bay_types:` with no value parses as None; treating it as [] would clear.""" + cat, server = catalog() + with pytest.raises(ModuleBayTypeError) as exc: + cat.ids_for("juniper", None) + assert "list of names" in str(exc.value) + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_a_non_string_entry_is_refused(self, catalog): + cat, _ = catalog() + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [{"name": "MX304-RE"}]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", ["MX304-RE", 7]) + with pytest.raises(ModuleBayTypeError): + cat.ids_for("juniper", [" "]) + + def test_an_empty_list_is_allowed_and_holds_no_classes(self, catalog): + """`module_bay_types: []` is an explicit instruction, not a malformed value.""" + cat, server = catalog() + assert cat.ids_for("juniper", []) == [] + assert not server.sent("POST", "module_bay_types") and not server.sent("POST", "manufacturers") + + def test_duplicate_names_collapse(self, catalog): + """The relationship is a set, so a repeated name must not produce a repeated id.""" + cat, _ = catalog() + once = cat.ids_for("juniper", ["MX304-RE"]) + + assert len(once) == 1 + assert cat.ids_for("juniper", ["MX304-RE", "MX304-RE"]) == once + + +@pytest.mark.real_http +class TestACatalogFailureIsNotAPerComponentSkip: + """A broken catalog is terminal for the run, but every caller recovers per component.""" + + @staticmethod + def _broken_catalog(tmp_path, catalog): + root = tmp_path / "broken" + write_module_bay_type(root, "Juniper", "mx304-re", "MX304-RE", "fine") + # A half-written entry: the readers index on name and dereference slug. + (root / "module-bay-types" / "Juniper" / "bad.yaml").write_text( + "name: 123\nslug: bad\nmanufacturer: Juniper\n", encoding="utf-8" + ) + return catalog(root=root)[0] + + def test_a_malformed_entry_is_not_reported_as_a_relation_change(self, tmp_path, catalog): + """_relation_change recovers from ModuleBayTypeError, so a load failure must not be one.""" + from types import SimpleNamespace + + from core.change_detector import _relation_change + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + netbox_comp = SimpleNamespace(name="RE0", module_bay_types=[]) + + with pytest.raises(ModuleBayCatalogError): + _relation_change( + "module_bay_types", + {"name": "RE0", "module_bay_types": ["MX304-RE"]}, + netbox_comp, + catalog=broken, + manufacturer="Juniper", + ) + + def test_the_catalog_failure_is_not_a_module_bay_type_error(self, tmp_path, catalog): + """Sibling, not subclass: an `except ModuleBayTypeError` must not swallow it.""" + from core.module_bay_types import ModuleBayCatalogError + + broken = self._broken_catalog(tmp_path, catalog) + + with pytest.raises(ModuleBayCatalogError) as caught: + broken.identities_for("Juniper", ["MX304-RE"]) + + assert not isinstance(caught.value, ModuleBayTypeError), "per-name catches would swallow it" + + def test_an_unresolved_name_is_still_a_recoverable_module_bay_type_error(self, catalog): + """The split must not promote a per-name miss into a run-ending failure.""" + resolver, _server = catalog() + + with pytest.raises(ModuleBayTypeError): + resolver.identities_for("Juniper", ["NOT-IN-CATALOG"]) + + +@pytest.mark.real_http +class TestUnreadableCatalogDirectory: + """os.walk swallows a directory it cannot read, which silently shrinks the catalog.""" + + def test_an_unreadable_vendor_directory_is_not_silently_skipped(self, tmp_path, catalog): + """The owner-scoped entry would vanish and the name would resolve to Generic instead.""" + import os + + if not hasattr(os, "geteuid"): + pytest.skip("no POSIX ownership, so the permission bits mean nothing here") + if os.geteuid() == 0: + pytest.skip("root ignores the permission bits this test relies on") + + root = tmp_path / "library" + write_module_bay_type(root, "Juniper", "qsfp-dd", "QSFP-DD", "the owner-scoped entry") + write_module_bay_type(root, "Generic", "qsfp-dd", "QSFP-DD", "the fallback entry") + vendor_dir = root / "module-bay-types" / "Juniper" + os.chmod(vendor_dir, 0o000) + try: + resolver, _server = catalog(root=root) + + with pytest.raises(ModuleBayCatalogError): + resolver.identities_for("Juniper", ["QSFP-DD"]) + finally: + os.chmod(vendor_dir, 0o700) diff --git a/tests/test_nb_dt_import.py b/tests/test_nb_dt_import.py index 31245d0b..fea59e41 100644 --- a/tests/test_nb_dt_import.py +++ b/tests/test_nb_dt_import.py @@ -335,13 +335,11 @@ def _make_mock_repo(device_types=None): return mock_repo -def _make_mock_netbox(modules=False, rack_types=False): +def _make_mock_netbox(): """Return a pre-configured NetBox mock.""" from collections import Counter mock_nb = MagicMock() - mock_nb.modules = modules - mock_nb.rack_types = rack_types mock_nb.device_types.existing_device_types = {} mock_nb.device_types.existing_device_types_by_slug = {} mock_nb.count_device_type_images.return_value = 0 @@ -884,7 +882,7 @@ def test_modules_with_types_to_process(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([module_type], {}, []) MockNetBox.return_value = mock_nb MockNetBox.filter_new_module_types.return_value = [] @@ -912,7 +910,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): patch("nb_dt_import.NetBox") as MockNetBox, patch("core.import_run.ChangeDetector") as MockDetector, ): - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_nb.filter_actionable_module_types.return_value = ([], {}, change_log) mock_nb.filter_new_module_types.return_value = [] MockNetBox.return_value = mock_nb @@ -930,7 +928,7 @@ def test_modules_update_mode_logs_change_detection_section(self, nb_dt_import): mock_nb.log_module_type_changes.assert_called_once_with(change_log) def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): - """When netbox.modules is True, module_added/updated counters are logged.""" + """Module counters are always logged: every supported release has module types.""" with ( patch.object(sys, "argv", ["nb-dt-import.py", "--only-new"]), patch("nb_dt_import.DTLRepo") as MockRepo, @@ -938,7 +936,7 @@ def test_settings_netbox_features_modules_logs_module_count(self, nb_dt_import): patch("nb_dt_import.LogHandler") as MockLogHandler, ): MockRepo.return_value = _make_mock_repo() - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() MockNetBox.return_value = mock_nb nb_dt_import.main() @@ -987,19 +985,6 @@ class TestProcessRackTypes: def _make_args(self, only_new=False): return SimpleNamespace(only_new=only_new) - def test_rack_types_disabled_logs_warning_and_returns(self, nb_dt_import): - """netbox.rack_types=False with actual rack types: warning logged, no further processing.""" - handle = MagicMock() - netbox = MagicMock() - netbox.rack_types = False - - rack_type = {"manufacturer": {"slug": "apc"}, "model": "AR1300", "slug": "apc-ar1300"} - import_run_module._process_rack_types(self._make_args(), netbox, handle, None, [rack_type]) - - handle.log.assert_called_once() - assert "4.1" in handle.log.call_args[0][0] - netbox.get_existing_rack_types.assert_not_called() - def test_empty_rack_types_returns_early(self, nb_dt_import): """rack_types=[]: returns immediately without any logging or API calls.""" handle = MagicMock() @@ -1252,7 +1237,7 @@ def test_module_type_only_vendor_uses_scoped_preload(self, nb_dt_import): """ mt = {"manufacturer": {"slug": "acbel"}, "model": "M1", "slug": "acbel-m1"} - mock_nb = _make_mock_netbox(modules=True) + mock_nb = _make_mock_netbox() mock_repo = _make_mock_repo() mock_repo.discover_vendors.return_value = [{"name": "Acbel", "slug": "acbel"}] @@ -1371,8 +1356,6 @@ def test_rack_types_counters_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = True from collections import Counter mock_nb.counter = Counter( @@ -1404,8 +1387,6 @@ def test_duplicate_definitions_are_logged(self, nb_dt_import): handle = MagicMock() mock_nb = MagicMock() - mock_nb.modules = False - mock_nb.rack_types = False from collections import Counter mock_nb.counter = Counter( @@ -1988,7 +1969,7 @@ def test_import_run_processes_slug_fast_path_and_skips_empty_vendor(self, make_c if files == ["cisco-module-types.yaml"] else [] ) - netbox = _make_mock_netbox(modules=True) + netbox = _make_mock_netbox() slug_resolved = { "device_files": {"empty": [], "cisco": ["resolved.yaml"]}, "module_vendors": {"cisco"}, @@ -2190,7 +2171,7 @@ class TestExportDiffVendorFilterOverRealHTTP: """Same run against a local HTTP server, so the filter is asserted as it is serialized on the wire.""" @staticmethod - def _serve(): + def _serve(netbox_version="4.7.0"): """Serve empty GraphQL pages and record every decoded request body.""" import json import threading @@ -2199,16 +2180,22 @@ def _serve(): bodies = [] class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers["Content-Length"]) - bodies.append(json.loads(self.rfile.read(length))) - payload = b'{"data": {}}' + def _reply(self, payload): self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) + def do_GET(self): + """Answer the version probe the export runs before its first query.""" + self._reply(json.dumps({"netbox-version": netbox_version}).encode()) + + def do_POST(self): + length = int(self.headers["Content-Length"]) + bodies.append(json.loads(self.rfile.read(length))) + self._reply(b'{"data": {}}') + def log_message(self, *args): """Silence the default stderr access log.""" @@ -2229,3 +2216,33 @@ def test_vendor_filter_is_serialized_as_a_json_list(self, nb_dt_import, monkeypa assert set(filters) == set(_LIST_FIELDS) for field, variables in filters.items(): assert variables["manufacturer_slugs"] == ["cisco", "juniper"], field + + def _queries_for_version(self, nb_dt_import, monkeypatch, tmp_path, library_root, version): + """Run one whole export against a server reporting *version* and return its queries.""" + url, server, bodies = self._serve(version) + monkeypatch.setenv("NETBOX_URL", url) + try: + _run_export_diff_cli(nb_dt_import, monkeypatch, tmp_path, library_root, "Juniper") + finally: + server.shutdown() + server.server_close() + return [payload["query"] for payload in bodies] + + def test_a_47_server_is_asked_for_the_module_bay_type_relation( + self, nb_dt_import, monkeypatch, tmp_path, _real_library_root + ): + """Not selecting it exports every bay without its restriction, silently. + + Asserted on the module-type query, which this run always issues; the component + queries only run once there is something to export. + """ + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.7.0") + + module_types = [q for q in queries if "module_type_list(" in q] + assert module_types and all("module_bay_types" in q for q in module_types) + + def test_an_older_server_is_never_asked_for_it(self, nb_dt_import, monkeypatch, tmp_path, _real_library_root): + """Selecting a field the schema lacks fails the whole query.""" + queries = self._queries_for_version(nb_dt_import, monkeypatch, tmp_path, _real_library_root, "4.6.9") + + assert not [q for q in queries if "module_bay_types" in q] diff --git a/tests/test_nb_serializer.py b/tests/test_nb_serializer.py index 9250234d..95e9452d 100644 --- a/tests/test_nb_serializer.py +++ b/tests/test_nb_serializer.py @@ -441,13 +441,33 @@ def test_rack_type_manufacturer_as_name_string(self): class TestFrontPortSerialization: - """Tests for front port rear_port extraction.""" + """Front port scalars. The rear-port linkage lives in TestPortMappingsStanza.""" - def test_front_port_rear_port_extracted_from_mapping(self): + def test_components_sorted_by_name(self): from types import SimpleNamespace - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) + iface_z = SimpleNamespace( + name="eth9", + type="1000base-t", + label="", + description="", + mgmt_only=False, + enabled=True, + poe_mode=None, + poe_type=None, + rf_role=None, + ) + iface_a = SimpleNamespace( + name="eth0", + type="1000base-t", + label="", + description="", + mgmt_only=False, + enabled=True, + poe_mode=None, + poe_type=None, + rf_role=None, + ) record = _dotdict( id=1, model="X", @@ -465,115 +485,108 @@ def test_front_port_rear_port_extracted_from_mapping(self): front_image=None, rear_image=None, ) - components = {1: {"front_port_templates": [fp]}} + components = {1: {"interface_templates": [iface_z, iface_a]}} result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] + names = [i["name"] for i in result["interfaces"]] + assert names == ["eth0", "eth9"] - def test_front_port_rear_port_position_included_when_gt_1(self): - from types import SimpleNamespace - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=3) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, +class TestRelationSerialization: + """A module bay's restriction has to survive the trip back out to YAML.""" + + @staticmethod + def _bay(name, module_bay_types=None, **extra): + """Build a module bay template as the GraphQL query returns it.""" + return _dotdict( + name=name, + position=None, + label="", description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + module_bay_types=module_bay_types, + **extra, ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port_position"] == 3 - def test_front_port_rear_port_position_zero_omitted(self): - """Position 0 is not a valid DTL value and should be omitted.""" - from types import SimpleNamespace + def test_a_bay_exports_the_names_of_its_classes(self): + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = self._bay("FPC 0", [_dotdict(id=9, name="MX304-LMIC"), _dotdict(id=8, name="QSFP-DD")]) + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0", "module_bay_types": ["MX304-LMIC", "QSFP-DD"]}] - mapping = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=0) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[mapping]) + def test_a_bay_with_no_classes_writes_no_key(self): + """An empty list here would add the key to every bay in the library. + + See test_an_empty_relation_does_not_make_every_definition_differ for the effect + that has on the export diff. + """ + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + + result = serialize_module_type(record, {1: {"module_bay_templates": [self._bay("FPC 0", [])]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_server_that_never_returned_the_field_omits_the_key(self): + """Below 4.7 the relation is not selected, and absent must not become empty.""" + record = _dotdict(id=1, model="MX304", manufacturer=_make_mfr(), part_number=None) + bay = _dotdict(name="FPC 0", position=None, label="", description="") + + result = serialize_module_type(record, {1: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "FPC 0"}] + + def test_a_module_type_exports_the_classes_it_belongs_to(self): record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, + id=5, + model="JNP304-RE", + manufacturer=_make_mfr(name="Juniper", slug="juniper"), part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + module_bay_types=[_dotdict(id=9, name="MX304-RE")], ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert "rear_port_position" not in result["front-ports"][0] - def test_front_port_multiple_mappings_warns_and_uses_first(self): - """When a front port has >1 mappings a UserWarning is raised and only the first is used.""" - import warnings - from types import SimpleNamespace + assert serialize_module_type(record, {})["module_bay_types"] == ["MX304-RE"] - m1 = SimpleNamespace(rear_port=SimpleNamespace(name="RP1"), rear_port_position=1) - m2 = SimpleNamespace(rear_port=SimpleNamespace(name="RP2"), rear_port_position=1) - fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="", mappings=[m1, m2]) + def test_a_device_type_bay_exports_its_classes_too(self): record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, + id=2, model="MX304", slug="mx304", manufacturer=_make_mfr(), u_height=None, is_full_depth=None ) - components = {1: {"front_port_templates": [fp]}} - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert len(caught) == 1 - assert issubclass(caught[0].category, UserWarning) - assert "FP1" in str(caught[0].message) - assert "2 mappings" in str(caught[0].message) - assert "issue #78" in str(caught[0].message) - - def test_front_port_legacy_rear_port_scalars(self): - """pre-4.5 NetBox: record has rear_port/rear_port_position as direct attrs (no mappings).""" + bay = self._bay("RE0", [_dotdict(id=9, name="MX304-RE")]) + + result = serialize_device_type(record, {2: {"module_bay_templates": [bay]}}) + + assert result["module-bays"] == [{"name": "RE0", "module_bay_types": ["MX304-RE"]}] + + +class TestPortMappingsStanza: + """Export writes the NetBox 4.5 port-mappings stanza the DTL schema now requires.""" + + @staticmethod + def _mapping(rear_port, rear_position=1, front_position=1): from types import SimpleNamespace - fp = SimpleNamespace( - name="FP1", - type="8p8c", + return SimpleNamespace( + rear_port=SimpleNamespace(name=rear_port), + rear_port_position=rear_position, + front_port_position=front_position, + ) + + @staticmethod + def _front_port(name, mappings=None, positions=1, **extra): + from types import SimpleNamespace + + return SimpleNamespace( + name=name, + type="lc-upc", label="", description="", color="", - mappings=None, - rear_port=SimpleNamespace(name="RP1"), - rear_port_position=3, + positions=positions, + mappings=mappings or [], + **extra, ) - record = _dotdict( + + def _device(self): + return _dotdict( id=1, model="X", slug="acme-x", @@ -590,13 +603,63 @@ def test_front_port_legacy_rear_port_scalars(self): front_image=None, rear_image=None, ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert result["front-ports"][0]["rear_port_position"] == 3 - def test_front_port_legacy_rear_port_position_1_omitted(self): - """pre-4.5: rear_port_position == 1 should be omitted (same as mappings path).""" + def test_a_front_port_carries_positions_and_no_inline_rear_port(self): + """The schema dropped rear_port from front-port entries and made positions required.""" + fp = self._front_port("FP1", [self._mapping("RP1")], positions=1) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "lc-upc", "positions": 1}] + + def test_every_mapping_reaches_the_stanza_not_just_the_first(self): + """The issue: a crossover front port mapped to two rear ports lost the second.""" + fp = self._front_port("FP1", [self._mapping("RP1", 1), self._mapping("RP2", 3, front_position=2)], positions=2) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 1}, + {"front_port": "FP1", "front_port_position": 2, "rear_port": "RP2", "rear_port_position": 3}, + ] + + def test_an_mpo_cassette_maps_every_front_port_to_its_rear_position(self): + """The shape the library actually carries: many front ports onto one MPO rear port.""" + ports = [self._front_port(f"FP{i}", [self._mapping("MPO1", i)]) for i in (1, 2, 3)] + + result = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert [(m["front_port"], m["rear_port_position"]) for m in result["port-mappings"]] == [ + ("FP1", 1), + ("FP2", 2), + ("FP3", 3), + ] + + def test_mappings_sort_by_numeric_positions_then_rear_port_name(self): + import yaml + + fp = self._front_port( + "FP1", + [ + self._mapping("RP2", "10.0", "2.0"), + self._mapping("RP2", 2, 2), + self._mapping("RP1", "2.0", 2), + self._mapping("RP1", None, None), + ], + positions=2, + ) + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == yaml.safe_load(""" +- {front_port: FP1, front_port_position: 1, rear_port: RP1, rear_port_position: 1} +- {front_port: FP1, front_port_position: 2, rear_port: RP1, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 2} +- {front_port: FP1, front_port_position: 2, rear_port: RP2, rear_port_position: 10} +""") + + def test_a_pre_45_server_still_exports_its_mappings(self): + """Below 4.5 NetBox returns rear_port scalars; dropping them would lose the linkage.""" from types import SimpleNamespace fp = SimpleNamespace( @@ -605,75 +668,48 @@ def test_front_port_legacy_rear_port_position_1_omitted(self): label="", description="", color="", - mappings=None, rear_port=SimpleNamespace(name="RP1"), - rear_port_position=1, + rear_port_position=4, ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"front_port_templates": [fp]}} - result = serialize_device_type(record, components) - assert result["front-ports"][0]["rear_port"] == "RP1" - assert "rear_port_position" not in result["front-ports"][0] - def test_components_sorted_by_name(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["port-mappings"] == [ + {"front_port": "FP1", "front_port_position": 1, "rear_port": "RP1", "rear_port_position": 4} + ] + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_legacy_front_port_carries_the_schema_required_positions(self): + """The schema requires positions, but it arrived in 4.5, so a pre-4.5 record needs the default.""" from types import SimpleNamespace - iface_z = SimpleNamespace( - name="eth9", - type="1000base-t", - label="", - description="", - mgmt_only=False, - enabled=True, - poe_mode=None, - poe_type=None, - rf_role=None, - ) - iface_a = SimpleNamespace( - name="eth0", - type="1000base-t", - label="", - description="", - mgmt_only=False, - enabled=True, - poe_mode=None, - poe_type=None, - rf_role=None, - ) - record = _dotdict( - id=1, - model="X", - slug="acme-x", - manufacturer=_make_mfr(), - u_height=1, - is_full_depth=True, - part_number=None, - airflow=None, - weight=None, - weight_unit=None, - description="", - comments="", - subdevice_role=None, - front_image=None, - rear_image=None, - ) - components = {1: {"interface_templates": [iface_z, iface_a]}} - result = serialize_device_type(record, components) - names = [i["name"] for i in result["interfaces"]] - assert names == sorted(names) + fp = SimpleNamespace(name="FP1", type="8p8c", label="", description="", color="") + + result = serialize_device_type(self._device(), {1: {"front_port_templates": [fp]}}) + + assert result["front-ports"] == [{"name": "FP1", "type": "8p8c", "positions": 1}] + + def test_a_front_port_with_no_mapping_adds_no_stanza(self): + result = serialize_device_type(self._device(), {1: {"front_port_templates": [self._front_port("FP1")]}}) + + assert "port-mappings" not in result + + def test_a_type_without_front_ports_adds_no_stanza(self): + assert "port-mappings" not in serialize_device_type(self._device(), {1: {}}) + + def test_the_importer_reads_back_what_the_export_wrote(self): + """Serializer to normalizer, both real: the stanza is the seam between them.""" + from core.repo import normalize_port_mappings + + ports = [ + self._front_port("1", [self._mapping("MPO1", 1)]), + self._front_port("2", [self._mapping("MPO1", 2)]), + ] + exported = serialize_device_type(self._device(), {1: {"front_port_templates": ports}}) + + assert normalize_port_mappings(exported) is None + assert [fp["_mappings"] for fp in exported["front-ports"]] == [ + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 1}], + [{"rear_port": "MPO1", "front_port_position": 1, "rear_port_position": 2}], + ] + assert "port-mappings" not in exported, "the normalizer consumes the stanza" diff --git a/tests/test_netbox_api.py b/tests/test_netbox_api.py index 3f7b3315..2423e106 100644 --- a/tests/test_netbox_api.py +++ b/tests/test_netbox_api.py @@ -1,5 +1,6 @@ import os import threading +from types import SimpleNamespace from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -80,7 +81,6 @@ def _factory(nb_api=None, handle=None, counter=None, **kwargs): handle if handle is not None else mock_handle, counter if counter is not None else MagicMock(), False, - False, graphql=kwargs.pop("graphql", graphql_client), repo_path=kwargs.pop("repo_path", mock_settings.repo_path), **kwargs, @@ -112,13 +112,12 @@ def _mark_cache_ready(device_types): def test_netbox_init(mock_settings, mock_pynetbox, mock_handle): # Mock api call - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.url == "http://mock-netbox" assert nb.token == "mock-token" # Verify module support detection - assert nb.modules def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, mock_handle): @@ -128,7 +127,7 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc remove_unmanaged_types=True, verify_images=True, ) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" netbox = NetBox(config, mock_handle) @@ -138,24 +137,30 @@ def test_netbox_init_applies_import_policy_flags(make_config, mock_pynetbox, moc def test_netbox_version_check(mock_settings, mock_pynetbox, mock_handle): - # Test 5.0 - mock_pynetbox.api.return_value.version = "5.0" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters + """Every supported release uses the new filter names; 4.7 adds module bay types.""" + for version, module_bay_types in (("4.3", False), ("4.5", False), ("4.7", True), ("5.0", True)): + mock_pynetbox.api.return_value.version = version + nb = NetBox(mock_settings, mock_handle) + assert nb.module_bay_types is module_bay_types, version - # Test 4.0 - mock_pynetbox.api.return_value.version = "4.0" - nb = NetBox(mock_settings, mock_handle) - assert not nb.new_filters - # Test 4.1 - mock_pynetbox.api.return_value.version = "4.1" - nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters +def test_the_netbox_version_is_fetched_once(mock_settings, mock_pynetbox, mock_handle): + """Each pynetbox `version` access is another HTTP request, and only the first is error-mapped.""" + from unittest.mock import PropertyMock + + api_type = type(mock_pynetbox.api.return_value) + version = PropertyMock(return_value="4.7.0") + api_type.version = version + try: + NetBox(mock_settings, mock_handle) + + assert version.call_count == 1, "the log lines must reuse the version already fetched" + finally: + del api_type.version def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.manufacturers.all.return_value = [] nb = NetBox(mock_settings, mock_handle) @@ -168,7 +173,7 @@ def test_create_manufacturers(mock_settings, mock_pynetbox, mock_handle): def test_create_manufacturers_no_new_is_verbose_only(mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -233,7 +238,7 @@ def test_create_generic_counts_the_created_list_at_the_caller(mock_pynetbox, mak def test_redundant_image_upload(mock_settings, mock_pynetbox, mock_handle): # Setup # Ensure modules check doesn't fail - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = MagicMock() @@ -514,6 +519,41 @@ def test_update_components_legacy_mapping_two_tuple_warns_and_skips( assert any("NetBox < 4.5" in str(c) for c in mock_handle.log.call_args_list) +def test_update_components_legacy_truncation_is_reported( + mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle +): + """The create path says "only first mapping applied"; the update path said nothing at all.""" + from core.change_detector import ChangeType, ComponentChange, PropertyChange + + mock_nb_api = MagicMock() + dt = make_device_types(nb_api=mock_nb_api) + dt.m2m_front_ports = False + + existing_fp = MagicMock(id=10, name="FP1") + rp1 = MagicMock(id=21, name="RP1") + rp2 = MagicMock(id=22, name="RP2") + dt.components.record("front_port_templates", "device", 1, {"FP1": existing_fp}) + dt.components.record("rear_port_templates", "device", 1, {"RP1": rp1, "RP2": rp2}) + + # Two mappings: the legacy model can hold only one of them. + new_mappings_set = frozenset({("RP1", 1, 1), ("RP2", 2, 1)}) + changes = [ + ComponentChange( + component_type="front-ports", + component_name="FP1", + change_type=ChangeType.COMPONENT_CHANGED, + property_changes=[PropertyChange("_mappings", frozenset(), new_mappings_set)], + ), + ] + + mock_handle.log.reset_mock() + dt.update_components({}, 1, changes, parent_type="device") + + logged = " ".join(str(c) for c in mock_handle.log.call_args_list) + assert "FP1" in logged, f"truncation must name the port, got: {logged}" + assert "4.5" in logged, f"truncation must say why, got: {logged}" + + def test_update_components_legacy_mapping_two_tuple_uses_yaml_fallback( mock_settings, mock_pynetbox, graphql_client, make_device_types ): @@ -611,7 +651,7 @@ class TestNetBoxConnectApi: def test_ssl_ignore_sets_verify_false(self, mock_pynetbox, mock_handle, make_config): mock_settings = make_config(ignore_ssl_errors=True) - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) assert nb.netbox.http_session.verify is False @@ -622,7 +662,7 @@ class TestCreateManufacturersError: def test_request_error_logged(self, mock_settings, mock_pynetbox, mock_handle): import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" # Make pynetbox.RequestError in the module under test be the real exception class mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -930,7 +970,7 @@ def test_creates_new_device_type_with_components( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) @@ -1118,14 +1158,14 @@ class TestCreateModuleTypes: """Tests for TestCreateModuleTypes.""" def test_empty_module_types_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) # Should not raise and should not call create nb.create_module_types([]) nb.netbox.dcim.module_types.create.assert_not_called() def test_creates_new_module_type(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1250,7 +1290,7 @@ class TestCreateModuleTypesBody: """Tests for TestCreateModuleTypesBody.""" def test_cached_module_type_skips_creation(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1285,7 +1325,7 @@ def test_create_module_type_request_error_logged( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1310,7 +1350,7 @@ def test_create_module_type_request_error_logged( def test_creates_module_type_with_components( self, mock_settings, mock_pynetbox, mock_graphql_requests, graphql_client, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -1606,7 +1646,7 @@ class TestCreateManufacturersSuccessLog: def test_verbose_log_per_created_manufacturer(self, mock_settings, mock_pynetbox, mock_handle): """verbose_log should be called for each created manufacturer.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_m = MagicMock() @@ -2333,7 +2373,7 @@ def test_create_failure_reaches_the_end_of_run_failure_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b"{\"manufacturer\":[\"Related object not found using the provided attributes: {'slug': 'ribbon'}\"]}" ) @@ -2487,7 +2527,6 @@ def test_creates_all_component_types( nb = NetBox(mock_settings, mock_handle) nb.device_types = dt - nb.modules = True created_dt = MagicMock() created_dt.id = 1 @@ -2574,7 +2613,7 @@ class TestFilterActionableModuleTypesEdge: def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """Empty module_types list returns [], {} immediately.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) result, images, _ = nb.filter_actionable_module_types([], {}, only_new=False) assert result == [] @@ -2582,7 +2621,7 @@ def test_empty_module_types_returns_empty(self, mock_settings, mock_pynetbox, mo def test_only_new_delegates_to_filter_new(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """only_new=True returns only genuinely new module types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() all_mts = {"cisco": {"LC": existing_mt}} @@ -2599,7 +2638,7 @@ def test_new_module_type_added_to_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """Module type not in all_module_types is added to actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2622,7 +2661,7 @@ def test_existing_module_with_new_image_is_actionable( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """Existing module type with an image not yet in NetBox is actionable.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2673,7 +2712,7 @@ def test_existing_module_with_changed_property_is_actionable( """Existing module type with a changed scalar property (e.g. part_number) is actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2726,7 +2765,7 @@ def test_existing_module_with_missing_image_and_property_change_logs_both( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2782,7 +2821,7 @@ def test_existing_module_with_only_missing_image_is_actionable_but_not_logged( """ from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2827,7 +2866,7 @@ def test_existing_module_with_unchanged_property_is_not_actionable( """Existing module type whose properties all match NetBox is not actionable.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -2868,7 +2907,7 @@ class TestCreateModuleTypesEdge: def test_existing_module_type_verbose_logged(self, mock_settings, mock_pynetbox, mock_handle): """When a module type already exists, verbose_log is called with 'Cached'.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = MagicMock() @@ -2894,7 +2933,7 @@ def test_only_new_skips_existing_module_component_creation( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """only_new=True + existing module → skip component creation.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("interface_templates", "module", 5, {}) @@ -2923,7 +2962,7 @@ def test_creates_module_type_with_power_outlets_console_server_ports_front_ports self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """power-outlets, console-server-ports, front-ports branches in create_module_types.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) for endpoint_name in ("power_outlet_templates", "console_server_port_templates", "front_port_templates"): @@ -2955,7 +2994,7 @@ def test_a_new_module_type_gets_its_module_bays( self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle ): """The DTL module-type schema allows module-bays, so creation must not skip them.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.components.record("module_bay_templates", "module", 5, {}) @@ -2981,7 +3020,7 @@ def test_existing_module_type_property_update_calls_api(self, mock_settings, moc """Existing module type with changed part_number calls module_types.update and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3017,7 +3056,7 @@ def test_existing_module_type_property_unchanged_no_api_call(self, mock_settings """Existing module type with matching part_number does not call module_types.update.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3048,7 +3087,7 @@ def test_existing_module_type_only_new_skips_property_update(self, mock_settings """only_new=True skips property update even when part_number differs.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) existing_mt = DotDict( @@ -3082,7 +3121,7 @@ def test_existing_module_type_component_update_calls_update_components( """Existing module type with changed component property calls update_components and increments counter.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3146,7 +3185,7 @@ def test_existing_module_type_property_and_component_update_increments_once( """Both property and component change → module_updated incremented only once.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3214,7 +3253,7 @@ def test_existing_module_type_removal_only_no_counter_increment( """COMPONENT_REMOVED-only changes call update_components but do NOT increment module_updated.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) _mark_cache_ready(nb.device_types) @@ -3275,7 +3314,7 @@ def test_property_update_plus_removal_only_remove_false_counts_as_updated( """Properties changed + removal-only diff with remove_components=False → module_updated incremented.""" from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types = make_device_types(nb_api=mock_pynetbox.api.return_value) nb.device_types.update_components = MagicMock() @@ -3358,7 +3397,7 @@ def test_existing_image_is_skipped( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """If the image name is already in module_type_existing_images, upload is skipped.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -3388,7 +3427,7 @@ def test_new_image_is_uploaded_and_tracked( self, mock_settings, mock_pynetbox, mock_graphql_requests, tmp_path, mock_handle ): """When image is not yet in existing_images, upload_image_attachment is called.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_dir = tmp_path / "module-types" / "vendor" @@ -4182,43 +4221,9 @@ def test_image_file_not_found_logs_error( nb.create_device_types([device_type]) assert any("Error locating image file" in str(c) for c in mock_handle.log.call_args_list) - def test_module_bays_not_created_when_modules_false( - self, mock_settings, mock_pynetbox, graphql_client, make_device_types, mock_handle - ): - """module-bays are only created when self.modules is True.""" - mock_nb_api = mock_pynetbox.api.return_value - dt = make_device_types(nb_api=mock_nb_api) - dt.existing_device_types = {} - dt.existing_device_types_by_slug = {} - dt.components.record("module_bay_templates", "device", 1, {}) - - nb = NetBox(mock_settings, mock_handle) - nb.device_types = dt - nb.modules = False # explicitly disabled - - created_dt = MagicMock() - created_dt.id = 1 - created_dt.manufacturer.name = "Cisco" - created_dt.model = "TestSwitch" - mock_nb_api.dcim.device_types.create.return_value = created_dt - - device_type = { - "manufacturer": {"slug": "cisco"}, - "model": "TestSwitch", - "slug": "testswitch", - "module-bays": [{"name": "MB1"}], - "src": "/tmp/device-types/cisco/testswitch.yaml", - } - nb.create_device_types([device_type]) - mock_nb_api.dcim.module_bay_templates.create.assert_not_called() - - -class TestCreateModuleTypesCornerCases: - """Corner-case tests for create_module_types (cognitive complexity 16).""" - def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When progress is provided, iteration goes through it.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) created_mt = MagicMock() @@ -4252,7 +4257,7 @@ def tracking_iter(): def test_all_module_types_fetched_when_none(self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle): """all_module_types is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4277,7 +4282,7 @@ def test_module_type_existing_images_fetched_when_none( self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): """module_type_existing_images is fetched when not supplied.""" - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { "manufacturer_list": [], @@ -4421,7 +4426,7 @@ class TestGetExistingRackTypes: def test_delegates_to_graphql(self, mock_settings, mock_pynetbox, graphql_client, mock_handle): """get_existing_rack_types() returns whatever graphql.get_rack_types() returns.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.graphql = graphql_client expected = {"apc": {"AR1300": MagicMock()}} @@ -4442,7 +4447,7 @@ class TestCreateRackTypes: """Tests for NetBox.create_rack_types().""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_empty_list_returns_immediately(self, mock_settings, mock_pynetbox, mock_handle): @@ -4510,7 +4515,7 @@ def test_existing_rack_type_fields_differ_calls_update(self, mock_settings, mock def test_new_rack_type_calls_create(self, mock_settings, mock_pynetbox, mock_handle): """Non-existing rack type: create called, counter incremented, added to cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 99 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4532,7 +4537,7 @@ def test_request_error_on_create_logged_no_crash(self, mock_settings, mock_pynet """RequestError during create is logged; processing continues.""" import pynetbox - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4553,7 +4558,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet import pynetbox from core.graphql_client import DotDict - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" err = pynetbox.RequestError(MagicMock(status_code=400, url="u", content=b'{"detail":"bad"}')) mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = err mock_pynetbox.RequestError = pynetbox.RequestError @@ -4573,7 +4578,7 @@ def test_request_error_on_update_logged_no_crash(self, mock_settings, mock_pynet def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, mock_handle): """When all_rack_types=None, get_existing_rack_types() is called to populate the cache.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) nb.get_existing_rack_types = MagicMock(return_value={}) rack_type = { @@ -4587,7 +4592,7 @@ def test_all_rack_types_none_triggers_fetch(self, mock_settings, mock_pynetbox, def test_progress_iterator_used(self, mock_settings, mock_pynetbox, mock_handle): """When a progress wrapper is provided, it is used as the iterator.""" - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" created_rt = MagicMock() created_rt.id = 1 mock_pynetbox.api.return_value.dcim.rack_types.create.return_value = created_rt @@ -4613,49 +4618,47 @@ class TestVerifyCompatibility: """Tests for NetBox.verify_compatibility() version thresholds.""" @pytest.mark.parametrize( - "version_str, expected_modules, expected_new_filters, expected_rack_types, expected_m2m", + "version_str, expected_m2m, expected_module_bay_types", [ - ("3.1", False, False, False, False), - ("3.2", True, False, False, False), - ("4.0", True, False, False, False), - ("4.1", True, True, True, False), - ("4.4", True, True, True, False), - ("4.5", True, True, True, True), - ("4.6", True, True, True, True), + ("4.3", False, False), + ("4.4", False, False), + ("4.5", True, False), + ("4.6", True, False), + ("4.7", True, True), + ("5.0", True, True), # Version strings with non-numeric suffixes - ("4.5-beta", True, True, True, True), - ("4.1.0", True, True, True, False), + ("4.5-beta", True, False), + ("4.3.0", False, False), ], ) def test_version_thresholds( self, version_str, - expected_modules, - expected_new_filters, - expected_rack_types, expected_m2m, + expected_module_bay_types, mock_settings, mock_pynetbox, mock_handle, ): + """Only the flags that still vary above the 4.3 floor are set.""" mock_pynetbox.api.return_value.version = version_str nb = NetBox(mock_settings, mock_handle) - assert nb.modules == expected_modules, f"modules mismatch for {version_str}" - assert nb.new_filters == expected_new_filters, f"new_filters mismatch for {version_str}" - assert nb.rack_types == expected_rack_types, f"rack_types mismatch for {version_str}" assert nb.m2m_front_ports == expected_m2m, f"m2m_front_ports mismatch for {version_str}" + assert nb.module_bay_types == expected_module_bay_types, f"module_bay_types mismatch for {version_str}" def test_single_component_version_string(self, mock_settings, mock_pynetbox, mock_handle): - """Version string with only major component (e.g. '4') does not crash.""" - mock_pynetbox.api.return_value.version = "4" + """A version string with only a major component (e.g. '5') does not crash.""" + mock_pynetbox.api.return_value.version = "5" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is False # 4.0 → no new filters + assert nb.m2m_front_ports is True - def test_version_42_enables_new_filters_not_m2m(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "4.2" + def test_the_oldest_supported_release_has_no_m2m_or_module_bay_types( + self, mock_settings, mock_pynetbox, mock_handle + ): + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) - assert nb.new_filters is True assert nb.m2m_front_ports is False + assert nb.module_bay_types is False # ============================================================ @@ -4855,7 +4858,7 @@ class TestLogModuleTypeChanges: def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, mock_handle): """A non-empty changed_property_log triggers verbose logging.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4868,7 +4871,7 @@ def test_non_empty_log_emits_verbose_output(self, mock_settings, mock_pynetbox, def test_empty_log_emits_nothing(self, mock_settings, mock_pynetbox, mock_handle): """An empty changed_property_log does not trigger any logging calls.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) mock_handle.verbose_log.reset_mock() @@ -4887,7 +4890,7 @@ class TestTryUpdateModuleTypeErrors: """Tests for RequestError and retryable-exception handlers in _try_update_module_type.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def _make_module_type_res(self): @@ -4901,7 +4904,7 @@ def test_request_error_returns_false_and_logs(self, mock_settings, mock_pynetbox """pynetbox.RequestError during update causes (False, False) return and log.""" import pynetbox as real_pynb - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) @@ -4926,7 +4929,7 @@ def test_retryable_exception_returns_false_and_logs(self, mock_settings, mock_py import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = self._make_nb(mock_settings, mock_handle, mock_pynetbox) mock_handle.log.reset_mock() @@ -4969,7 +4972,7 @@ def test_retryable_exception_on_create_returns_false( import requests mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_pynetbox.RequestError = real_pynb.RequestError nb = NetBox(mock_settings, mock_handle) @@ -5777,7 +5780,7 @@ class TestUploadModuleTypeImagesVerify: """Tests for _upload_module_type_images with verify_images=True.""" def _make_nb(self, mock_settings, mock_handle, mock_pynetbox): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.device_types.upload_image_attachment = MagicMock(return_value=True) nb.verify_images = True @@ -5936,7 +5939,7 @@ def test_load_image_hash_cache_returns_empty_dict_on_bad_json(self, tmp_path): def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_settings, mock_pynetbox, mock_handle): from core.graphql_client import GraphQLError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with patch.object(NetBox, "get_manufacturers", side_effect=GraphQLError("bad query")): with pytest.raises(NetBoxError, match="GraphQL error: bad query"): @@ -5945,7 +5948,7 @@ def test_init_raises_typed_graphql_error_from_get_manufacturers(self, mock_setti def test_init_raises_typed_error_when_device_types_initialization_fails( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" with ( patch.object(NetBox, "get_manufacturers", return_value=[]), @@ -6022,7 +6025,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.manufacturers.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6032,7 +6035,7 @@ def test_create_manufacturers_logs_retryable_exception(self, mock_settings, mock assert any("Connection error creating manufacturers" in str(c) for c in mock_handle.log.call_args_list) def test_try_resolve_update_logs_classifier_exception(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") @@ -6049,7 +6052,7 @@ def test_try_resolve_update_truncates_blocker_list(self, mock_settings, mock_pyn from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") resolution = SimpleNamespace( @@ -6072,7 +6075,7 @@ def test_try_resolve_update_logs_auto_resolve_failure(self, mock_settings, mock_ from types import SimpleNamespace from core.update_failure_resolver import FailureKind - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6105,7 +6108,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( from core.update_failure_resolver import FailureKind mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.force_resolve_conflicts = True dt = MagicMock(id=1, model="Model-1") @@ -6132,7 +6135,7 @@ def test_try_resolve_update_logs_retryable_exception_after_auto_resolve( def test_log_device_type_change_outcome_partial_success_mentions_property_failure( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6151,7 +6154,7 @@ def test_log_device_type_change_outcome_partial_success_mentions_property_failur def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) dt = MagicMock(id=1, model="Model-1") dt.manufacturer.name = "Cisco" @@ -6168,7 +6171,7 @@ def test_log_device_type_change_outcome_logs_cached_when_nothing_happened( assert any("Device Type Cached" in str(c) for c in mock_handle.verbose_log.call_args_list) def test_filter_images_for_upload_keeps_changed_image(self, mock_settings, mock_pynetbox, tmp_path, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True @@ -6194,7 +6197,7 @@ def test_handle_existing_device_type_logs_retryable_property_update_error( from core.change_detector import PropertyChange mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.update.side_effect = requests.exceptions.ConnectionError("offline") nb._log_device_type_change_outcome = MagicMock() @@ -6217,7 +6220,7 @@ def test_create_new_device_type_logs_retryable_error(self, mock_settings, mock_p import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.netbox.dcim.device_types.create.side_effect = requests.exceptions.ConnectionError("offline") @@ -6238,7 +6241,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( ): from core.change_detector import ChangeType, ComponentChange, PropertyChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) changes = [ ComponentChange("interfaces", "xe-0", ChangeType.COMPONENT_ADDED), @@ -6261,7 +6264,7 @@ def test_log_module_property_diffs_emits_added_changed_and_removed_components( def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True details = {7: {"linecard.front": {"att_id": 5, "url": "/media/linecard.front.jpg"}}} @@ -6273,7 +6276,7 @@ def test_fetch_module_type_existing_images_uses_detailed_query_in_verify_mode( assert nb._module_image_details == details def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, mock_pynetbox, mock_handle): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(spec=["id", "manufacturer", "model"]) module_type_res.id = 1 @@ -6290,7 +6293,7 @@ def test_try_update_module_type_skips_missing_netbox_fields(self, mock_settings, def test_filter_actionable_module_types_marks_verify_images_module_actionable( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True _mark_cache_ready(nb.device_types) @@ -6314,7 +6317,7 @@ class TestAdditionalModuleTypeCoverage: def test_apply_module_type_component_updates_records_failed_no_actionable_changes( self, mock_settings, mock_pynetbox, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6334,7 +6337,7 @@ def test_apply_module_type_component_updates_marks_partial_on_partial_component_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6362,7 +6365,7 @@ def test_apply_module_type_component_updates_marks_partial_when_properties_only_ ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6383,7 +6386,7 @@ def test_apply_module_type_component_updates_records_failed_when_no_changes_appl ): from core.change_detector import ChangeType, ComponentChange - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) module_type_res = MagicMock(id=7, model="LC") module_type_res.manufacturer.name = "Cisco" @@ -6510,7 +6513,7 @@ def test_create_rack_types_logs_retryable_update_error(self, mock_settings, mock from core.graphql_client import DotDict mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.update.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6530,7 +6533,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock import requests mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.rack_types.create.side_effect = requests.exceptions.ConnectionError( "offline" ) @@ -6547,7 +6550,7 @@ def test_create_rack_types_logs_retryable_create_error(self, mock_settings, mock def test_upload_module_type_images_discards_missing_attachment_before_failed_upload( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=False) @@ -6575,7 +6578,7 @@ def test_upload_module_type_images_discards_missing_attachment_before_failed_upl def test_upload_module_type_images_skips_changed_image_when_delete_fails( self, mock_settings, mock_pynetbox, tmp_path, mock_handle ): - mock_pynetbox.api.return_value.version = "3.5" + mock_pynetbox.api.return_value.version = "4.3" nb = NetBox(mock_settings, mock_handle) nb.verify_images = True nb.device_types.upload_image_attachment = MagicMock(return_value=True) @@ -6814,7 +6817,6 @@ def _device_types(url, handle): handle, MagicMock(), False, - False, graphql=NetBoxGraphQLClient(url, "token", page_size=10), repo_path="/tmp/repo", max_threads=2, @@ -6825,7 +6827,7 @@ def test_filter_actionable_module_types_skips_unchanged_existing_module( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6874,7 +6876,7 @@ def test_filter_actionable_module_types_includes_module_with_missing_component( mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle ): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6922,7 +6924,7 @@ def test_missing_netbox_field_is_not_treated_as_change( ): """When existing module lacks an attribute, it's skipped — no false positive change.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" mock_graphql_requests.side_effect = paginate_dispatch( { @@ -6979,7 +6981,7 @@ def test_remove_components_is_called_when_flag_set( ): """When remove_components=True and there are component changes, remove_components is called.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7037,7 +7039,7 @@ def test_component_reconciliation_continues_when_scalar_patch_fails( skipped entirely. """ mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "3.5" + mock_nb_api.version = "4.3" nb = NetBox(mock_settings, mock_handle) @@ -7106,7 +7108,7 @@ def _netbox(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_rack_type_create_failure_is_reported(self, mock_settings, mock_pynetbox, mock_handle): @@ -7190,7 +7192,7 @@ def test_device_type_create_failure_is_not_called_an_update_failure( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" mock_nb_api.dcim.device_types.create.side_effect = _request_error( b'{"manufacturer":["Related object not found using the provided attributes: slug ribbon"]}' ) @@ -7223,12 +7225,11 @@ def test_module_type_create_failure_is_not_called_an_update_failure( import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" mock_pynetbox.api.return_value.dcim.module_types.create.side_effect = _request_error( b'{"model":["This field may not be blank."]}' ) nb = NetBox(mock_settings, mock_handle) - nb.modules = True nb._process_single_module_type( {"manufacturer": {"slug": "panduit"}, "model": "FAP6WBUSC", "slug": "fap6wbusc"}, "/repo/module-types/Panduit/FAP6WBUSC.yaml", @@ -7247,7 +7248,7 @@ class TestSkippedComponentReasonReachesTheReport: def _dt(self, mock_pynetbox, make_device_types, parent_id=1): mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) return dt @@ -7260,7 +7261,7 @@ def test_unresolvable_power_port_reaches_the_report( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) inlet = MagicMock() @@ -7351,7 +7352,7 @@ def test_component_update_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) existing_iface = MagicMock() @@ -7410,7 +7411,7 @@ def test_component_removal_transport_failure_reports_the_transport_error( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) stale = MagicMock() @@ -7439,7 +7440,7 @@ def test_failed_component_create_reports_the_netbox_message( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 6119, {}) @@ -7484,7 +7485,7 @@ def _nb(self, mock_settings, mock_handle, mock_pynetbox): import pynetbox as real_pynb mock_pynetbox.RequestError = real_pynb.RequestError - mock_pynetbox.api.return_value.version = "4.1" + mock_pynetbox.api.return_value.version = "4.3" return NetBox(mock_settings, mock_handle) def test_new_device_type_with_failing_component_is_partial( @@ -7495,7 +7496,7 @@ def test_new_device_type_with_failing_component_is_partial( mock_pynetbox.RequestError = real_pynb.RequestError mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt.components.record("interface_templates", "device", 900, {}) mock_nb_api.dcim.interface_templates.create.side_effect = _request_error( @@ -7532,7 +7533,7 @@ def test_new_device_type_with_failing_component_is_partial( def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, graphql_client, make_device_types): """Errors buffered for one entity must not surface in the next entity's reason.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) dt._log_component_error("stale error from an earlier entity") @@ -7547,7 +7548,7 @@ def test_component_errors_do_not_leak_between_entities(self, mock_pynetbox, grap def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_client, make_device_types): """The buffer is empty after a scope closes, so the next scope starts clean.""" mock_nb_api = mock_pynetbox.api.return_value - mock_nb_api.version = "4.1" + mock_nb_api.version = "4.3" dt = make_device_types(nb_api=mock_nb_api) with dt.collect_component_errors() as first: @@ -7557,3 +7558,99 @@ def test_collect_component_errors_clears_on_exit(self, mock_pynetbox, graphql_cl with dt.collect_component_errors() as second: pass assert second == [] + + +def test_netbox_below_minimum_version_is_refused_with_a_clear_message(mock_settings, mock_pynetbox, mock_handle): + """A GraphQL schema error is a poor way to learn the server is too old.""" + from core.netbox_api import NetBoxError + + for version in ("4.1", "4.2", "3.5"): + mock_pynetbox.api.return_value.version = version + with pytest.raises(NetBoxError) as exc: + NetBox(mock_settings, mock_handle) + message = str(exc.value) + assert "4.3" in message, f"{version}: the message must name the minimum" + assert version in message, f"{version}: the message must name what was found" + + +class TestAMappingClearNeedsTheRemovalFlag: + """Clearing a front-port mapping removes data, so it obeys --remove-components. + + The tool tells the user "will not remove components from existing models" when the flag + is off. A mapping clear reaches NetBox as a COMPONENT_CHANGED property change, so it + bypassed that promise and cleared the linkage under a plain --update. + """ + + def _module_type_losing_a_mapping(self, nb): + """Record an existing FP1 mapped to RP1, and return YAML whose stanza omits it.""" + existing_module = MagicMock() + existing_module.id = 55 + existing_module.manufacturer.name = "Cisco" + existing_module.model = "CM-Map" + + existing_fp = SimpleNamespace( + name="FP1", + _mappings_canonical=[{"rear_port_name": "RP1", "front_port_position": 1, "rear_port_position": 1}], + _mappings_m2m=True, + ) + nb.device_types.components.record("front_port_templates", "module", 55, {"FP1": existing_fp}) + _mark_cache_ready(nb.device_types) + + curr_mt = { + "manufacturer": {"slug": "cisco"}, + "model": "CM-Map", + "slug": "cm-map", + # normalize_port_mappings assigns [] to a front port the stanza omits + "front-ports": [{"name": "FP1", "type": "8p8c", "_mappings": []}], + } + return {"cisco": {"CM-Map": existing_module}}, curr_mt + + def _mapping_clears_sent(self, nb): + """Return the _mappings property changes that reached update_components.""" + from core.change_detector import ChangeType + + sent = [] + for call_args in nb.device_types.update_components.call_args_list: + for change in call_args.args[2]: + if change.change_type is not ChangeType.COMPONENT_CHANGED: + continue + sent += [pc for pc in change.property_changes if pc.property_name == "_mappings"] + return sent + + @pytest.mark.parametrize("remove_components, expected", [(False, 0), (True, 1)]) + def test_the_flag_decides_whether_a_clear_reaches_netbox( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle, remove_components, expected + ): + """Without the flag the clear must never be sent; with it, it must.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=remove_components + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset()] * expected + + def test_a_changed_mapping_still_applies_without_the_flag( + self, mock_settings, mock_pynetbox, mock_graphql_requests, mock_handle + ): + """Only removal is gated. Repointing FP1 to another rear port is an ordinary update.""" + mock_pynetbox.api.return_value.version = "4.3" + nb = NetBox(mock_settings, mock_handle) + all_module_types, curr_mt = self._module_type_losing_a_mapping(nb) + curr_mt["front-ports"][0]["_mappings"] = [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] + nb.device_types.update_components = MagicMock() + nb.device_types.remove_components = MagicMock() + + nb._process_single_module_type( + curr_mt, "test.yaml", all_module_types, {}, only_new=False, remove_components=False + ) + + clears = self._mapping_clears_sent(nb) + assert [pc.new_value for pc in clears] == [frozenset({("RP2", 1, 1)})] diff --git a/tests/test_relation_scope.py b/tests/test_relation_scope.py new file mode 100644 index 00000000..caee3dd0 --- /dev/null +++ b/tests/test_relation_scope.py @@ -0,0 +1,197 @@ +"""A relation assigned from the wrong manufacturer scope must be detected as a change. + +The catalog resolves a reference name in the owning manufacturer's scope first and only +then in Generic. If NetBox already holds the Generic object where the owner has one of +its own, comparing names alone reports equality and the wrong object survives. +""" + +import pytest + +from core.change_detector import ChangeDetector +from core.module_bay_types import ModuleBayTypeCatalog + + +class Handle: + """Capture what the detector logs.""" + + def __init__(self): + """Start with no recorded lines.""" + self.lines = [] + + def log(self, message): + """Record one line.""" + self.lines.append(message) + + def verbose_log(self, message): + """Record one verbose line.""" + self.lines.append(message) + + +class Related: + """A module bay type as NetBox returns it inside a relation.""" + + def __init__(self, name, slug, manufacturer_slug): + """Store the identity fields the comparison needs.""" + self.name = name + self.slug = slug + self.manufacturer = type("M", (), {"slug": manufacturer_slug})() + + +class NetBoxBay: + """A module bay template as the cache hands it to the detector.""" + + def __init__(self, name, module_bay_types): + """Store the bay name and its assigned classes.""" + self.name = name + self.module_bay_types = module_bay_types + + +@pytest.fixture +def two_scope_catalog(tmp_path): + """Build a catalog where the same class name exists under Acme and under Generic.""" + for manufacturer, slug in (("Acme", "acme-x"), ("Generic", "generic-x")): + directory = tmp_path / "module-bay-types" / manufacturer + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{slug}.yaml").write_text( + f"name: X\nslug: {slug}\nmanufacturer: {manufacturer}\n", encoding="utf-8" + ) + return str(tmp_path) + + +def _detector(catalog, handle=None): + """Build a detector whose device_types exposes the catalog, as the real one does.""" + device_types = type("DeviceTypes", (), {"module_bay_type_catalog": catalog, "module_bay_types_supported": True})() + return ChangeDetector(device_types, handle or Handle()) + + +def test_wrong_scope_assignment_is_detected(two_scope_catalog): + """Acme owns X, but NetBox assigned Generic's X. The names match; the objects do not.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert [c.property_name for c in changes] == ["module_bay_types"], ( + "a bay owned by Acme holding Generic's X must be corrected to Acme's X" + ) + + +def test_right_scope_assignment_is_left_alone(two_scope_catalog): + """The same bay already holding Acme's X is correct and must not be rewritten.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="acme" + ) + assert changes == [] + + +def test_generic_fallback_is_correct_when_the_owner_has_no_such_class(two_scope_catalog): + """A Nokia bay resolves X to Generic's X, so holding Generic's X is right.""" + catalog = ModuleBayTypeCatalog(None, two_scope_catalog, Handle()) + detector = _detector(catalog) + + yaml_comp = {"name": "Slot 0", "module_bay_types": ["X"]} + netbox_comp = NetBoxBay("Slot 0", [Related("X", "generic-x", "generic")]) + + changes = detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer="nokia" + ) + assert changes == [] + + +def _changes(detector, yaml_comp, netbox_comp, manufacturer="acme"): + """Run the real property comparison for the relation and return what it found.""" + return detector._compare_component_properties( + yaml_comp, netbox_comp, ["module_bay_types"], comp_type="module-bays", manufacturer=manufacturer + ) + + +class TestUnmanagedRelations: + """A relation is only rewritten when both sides say something about it.""" + + def test_an_omitted_key_leaves_the_relation_alone(self, two_scope_catalog): + """A definition that never mentions the relation is not asking for it to be cleared.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0"}, netbox_comp) == [] + + def test_a_bare_key_leaves_the_relation_alone_but_says_so(self, two_scope_catalog): + """Parses as None, so nothing is cleared; a typo must still not be invisible.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": None}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_non_name_entry_leaves_the_relation_alone(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [{"name": "X"}]}, netbox_comp) == [] + + def test_a_whitespace_only_entry_leaves_the_relation_alone(self, two_scope_catalog): + """The catalog rejects it later, which would skip the whole component's update.""" + handle = Handle() + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle()), handle) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + assert _changes(detector, {"name": "Slot 0", "module_bay_types": [" "]}, netbox_comp) == [] + assert any("module_bay_types" in line and "Slot 0" in line for line in handle.lines) + + def test_a_field_the_query_did_not_return_is_skipped(self, two_scope_catalog): + """Reading an absent field as empty would report a change on every run.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = type("Bay", (), {"name": "Slot 0"})() + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_an_unresolvable_reference_reaches_the_write_path(self, two_scope_catalog): + """Reporting "no change" leaves the bay unrestricted in silence. + + The write path is the only thing that logs and records an unresolvable name, and it + only ever sees a component the detector reported as changed. + """ + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["NO-SUCH-CLASS"]}, netbox_comp) + + assert [(c.property_name, c.new_value) for c in changes] == [("module_bay_types", ["NO-SUCH-CLASS"])] + + +class TestNameComparisonFallback: + """Where an identity cannot be had, comparing names is still better than doing nothing.""" + + def test_names_are_compared_when_netbox_returned_no_slug(self, two_scope_catalog): + """A read path returning only id and name cannot answer the scope question.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", None, None)]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.property_name, c.old_value, c.new_value) for c in changes] == [("module_bay_types", ["Y"], ["X"])] + + def test_matching_names_are_left_alone_when_netbox_returned_no_slug(self, two_scope_catalog): + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", None, None)]) + assert _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) == [] + + def test_names_are_compared_without_a_catalog(self): + """A caller that has not wired a catalog still gets the name comparison.""" + detector = _detector(None) + netbox_comp = NetBoxBay("Slot 0", [Related("Y", "generic-y", "generic")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": ["X"]}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["Y"], ["X"])] + + def test_an_empty_list_clears_the_relation(self, two_scope_catalog): + """`module_bay_types: []` is an explicit instruction to hold no classes.""" + detector = _detector(ModuleBayTypeCatalog(None, two_scope_catalog, Handle())) + netbox_comp = NetBoxBay("Slot 0", [Related("X", "acme-x", "acme")]) + changes = _changes(detector, {"name": "Slot 0", "module_bay_types": []}, netbox_comp) + assert [(c.old_value, c.new_value) for c in changes] == [(["X"], [])] diff --git a/tests/test_repo.py b/tests/test_repo.py index ab4e695a..09714d48 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,6 +1,7 @@ import os import pytest +import yaml from unittest.mock import MagicMock, call, mock_open, patch from git import Actor, Repo as GitRepo, exc as git_exc from core.repo import ( @@ -1104,6 +1105,44 @@ def test_null_stanza_is_deleted(self): assert err is None assert "port-mappings" not in data + @pytest.mark.parametrize("stanza", ["", "port-mappings:", "port-mappings: []"]) + def test_only_an_explicit_list_manages_front_port_mappings(self, stanza): + data = yaml.safe_load(f""" +front-ports: + - name: FP1 + type: 8p8c + - name: FP2 + type: 8p8c +{stanza} +""") + + assert normalize_port_mappings(data) is None + assert "port-mappings" not in data + for port in data["front-ports"]: + if stanza == "port-mappings: []": + assert port["_mappings"] == [] + else: + assert "_mappings" not in port + + @pytest.mark.parametrize( + ("stanza", "expected"), + [ + ("RP1", "Error: port-mappings must be a list: 'RP1'"), + ("{FP1: RP1}", "Error: port-mappings must be a list: {'FP1': 'RP1'}"), + ("[RP1]", "Error: port-mappings entry must be a mapping: 'RP1'"), + ], + ) + def test_malformed_stanza_returns_an_error(self, stanza, expected): + data = yaml.safe_load(f""" +front-ports: + - {{name: FP1, type: 8p8c}} +rear-ports: + - {{name: RP1, type: 8p8c}} +port-mappings: {stanza} +""") + + assert normalize_port_mappings(data) == expected + def test_empty_stanza_no_front_ports_still_deleted(self): """Empty port-mappings stanza with no front-ports is cleaned up (not silently skipped).""" data = { @@ -1796,3 +1835,170 @@ def test_corrupted_device_json_returns_none(self, tmp_path): repo.cwd = "" assert repo.resolve_slug_files(["nokia"]) is None + + +class TestAnExplicitlyEmptyStanza: + """`port-mappings: []` is an author saying "none", which is not the same as saying nothing.""" + + def test_an_empty_stanza_clears_every_front_port_mapping(self): + """Without _mappings: [] the change detector cannot express removing a mapping.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c"}, {"name": "FP2", "type": "8p8c"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 2}], + "port-mappings": [], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [] + assert data["front-ports"][1]["_mappings"] == [] + + def test_an_empty_stanza_beside_an_inline_linkage_is_a_conflict(self): + """Silently preferring the inline linkage ignores the newer, explicit statement.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [{"name": "RP1", "type": "8p8c", "positions": 1}], + "port-mappings": [], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an empty stanza beside an inline linkage must not pass silently" + assert result.startswith("Error:"), result + + def test_no_stanza_at_all_still_leaves_mappings_unmanaged(self): + """An absent key must keep meaning "no opinion", or every file would clear its mappings.""" + from core.repo import normalize_port_mappings + + data = {"front-ports": [{"name": "FP1", "type": "8p8c"}], "rear-ports": []} + + assert normalize_port_mappings(data) is None + assert "_mappings" not in data["front-ports"][0] + + +class TestAStanzaThatDoesNotListAFrontPort: + """A stanza speaks for the whole file, so a port it omits has no mapping.""" + + def test_a_nonempty_stanza_clears_an_omitted_front_port_mapping(self): + from types import SimpleNamespace + from core.change_detector import ChangeDetector + + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP1, rear_port: RP1} +""") + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [] + + existing = SimpleNamespace( + name="FP2", + _mappings_canonical=[{"rear_port_name": "RP2", "front_port_position": 1, "rear_port_position": 1}], + ) + detector = ChangeDetector(SimpleNamespace(), LogHandler(False)) + changes = detector._compare_component_properties( + data["front-ports"][1], existing, ["_mappings"], comp_type="front-ports" + ) + assert len(changes) == 1 + assert changes[0].property_name == "_mappings" + assert changes[0].old_value == {("RP2", 1, 1)} + assert changes[0].new_value == set() + + def test_a_half_migrated_file_errors_before_assigning_mappings(self): + data = yaml.safe_load(""" +front-ports: + - {name: FP1, type: 8p8c, rear_port: RP1} + - {name: FP2, type: 8p8c} +rear-ports: + - {name: RP1, type: 8p8c} + - {name: RP2, type: 8p8c} +port-mappings: + - {front_port: FP2, rear_port: RP2} +""") + + assert normalize_port_mappings(data) == ( + "Error: front port 'FP1' declares an inline rear_port but the port-mappings " + "stanza does not list it; the stanza is authoritative, so add 'FP1' to it " + "or remove the inline rear_port keys" + ) + assert all("_mappings" not in port for port in data["front-ports"]) + + def test_an_inline_linkage_the_stanza_omits_names_the_stanza_as_authoritative(self): + """The old wording blamed a conflict against a stanza that never mentioned the port.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP2", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None, "an inline linkage the stanza omits must not pass silently" + assert "conflicting mapping definitions" not in result, result + assert "FP1" in result, result + assert "does not list it" in result, result + + def test_a_disagreement_on_a_shared_front_port_still_reads_as_a_conflict(self): + """Both formats naming one port differently is a real conflict, not an omission.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [{"name": "FP1", "type": "8p8c", "rear_port": "RP1"}], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [{"front_port": "FP1", "rear_port": "RP2"}], + } + + result = normalize_port_mappings(data) + + assert result is not None + assert "conflicting mapping definitions" in result, result + + def test_a_stanza_may_add_a_port_the_inline_format_never_linked(self): + """The half-finished migration the two formats exist to allow: both are kept.""" + from core.repo import normalize_port_mappings + + data = { + "front-ports": [ + {"name": "FP1", "type": "8p8c", "rear_port": "RP1"}, + {"name": "FP2", "type": "8p8c"}, + ], + "rear-ports": [ + {"name": "RP1", "type": "8p8c", "positions": 1}, + {"name": "RP2", "type": "8p8c", "positions": 1}, + ], + "port-mappings": [ + {"front_port": "FP1", "rear_port": "RP1"}, + {"front_port": "FP2", "rear_port": "RP2"}, + ], + } + + assert normalize_port_mappings(data) is None + assert data["front-ports"][0]["_mappings"] == [ + {"rear_port": "RP1", "front_port_position": 1, "rear_port_position": 1} + ] + assert data["front-ports"][1]["_mappings"] == [ + {"rear_port": "RP2", "front_port_position": 1, "rear_port_position": 1} + ] diff --git a/tests/test_suite_hygiene.py b/tests/test_suite_hygiene.py index dd8a46bc..e85d411e 100644 --- a/tests/test_suite_hygiene.py +++ b/tests/test_suite_hygiene.py @@ -82,6 +82,25 @@ def test_integration_collection_reads_credentials_from_a_local_env_file(tmp_path assert "1 passed" in result.stdout, result.stdout + result.stderr +def test_the_fake_netbox_releases_its_listening_socket(): + """shutdown() only stops the serve loop. One server per test leaks a descriptor each.""" + import socket + + from helpers import FakeNetBox + + server = FakeNetBox(manufacturers=[{"id": 1, "name": "Juniper", "slug": "juniper"}]) + port = server._server.server_port + server.close() + + probe = socket.socket() + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: # pragma: no cover - only reached when close() leaks + raise AssertionError(f"FakeNetBox.close() left port {port} bound: {exc}") from exc + finally: + probe.close() + + def test_no_test_function_contains_an_orphaned_docstring(): """A dropped ``def`` header silently merges two tests into one. diff --git a/tests/test_update_failure_resolver.py b/tests/test_update_failure_resolver.py index a6135626..93a4bf2f 100644 --- a/tests/test_update_failure_resolver.py +++ b/tests/test_update_failure_resolver.py @@ -314,57 +314,18 @@ def test_classifier_count_fallback_when_count_query_fails(): assert res.dependent_devices_count == 5 -def test_new_filters_uses_device_type_id_key(): - """new_filters=True must call filter(device_type_id=...) not devicetype_id=... +def test_the_device_bay_template_lookup_uses_the_supported_filter_key(): + """The 4.1 rename is below the supported floor, so only device_type_id is ever sent. - This matters because NetBox >= 4.1 changed the query param name. - Passing the wrong key causes pynetbox to silently return ALL templates. + The wrong key does not raise: pynetbox silently returns every template, so the + classifier would report an unrelated device type as the blocker. """ nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=99, device_type_yaml={}) nb.dcim.device_bay_templates.filter.assert_called_once_with(device_type_id=99) -def test_old_filters_uses_devicetype_id_key(): - """new_filters=False (default) must call filter(devicetype_id=...) for NetBox < 4.1.""" - nb = _make_netbox() - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=99, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.device_bay_templates.filter.assert_called_once_with(devicetype_id=99) - - -def test_count_dependent_devices_uses_new_filter_key(): - """When new_filters=True, dcim.devices must be queried with device_type_id= not devicetype_id=.""" +def test_the_dependent_device_count_uses_the_supported_filter_key(): nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=True, - ) + classify_device_type_update_failure(SUBDEVICE_ROLE_ERROR_DICT, netbox=nb, device_type_id=77, device_type_yaml={}) nb.dcim.devices.filter.assert_called_once_with(device_type_id=77, limit=5) - - -def test_count_dependent_devices_uses_legacy_filter_key(): - """When new_filters=False (default), dcim.devices must be queried with devicetype_id=.""" - nb = _make_netbox(devices=[], device_count=0) - classify_device_type_update_failure( - SUBDEVICE_ROLE_ERROR_DICT, - netbox=nb, - device_type_id=77, - device_type_yaml={}, - new_filters=False, - ) - nb.dcim.devices.filter.assert_called_once_with(devicetype_id=77, limit=5)