diff --git a/docs/source_vmware.md b/docs/source_vmware.md index 9716d9d..247083b 100644 --- a/docs/source_vmware.md +++ b/docs/source_vmware.md @@ -149,6 +149,39 @@ Custom Fields: VMware Guest Hostname: appprd01.corp.example.com ``` +### Cables to CDP/LLDP neighbors + +An ESXi host reports the switch and the switch port each of its physical interfaces (pNICs) is +connected to, if CDP or LLDP is enabled on the switch. With the option `sync_host_cables` enabled +netbox-sync uses this information to create cables in NetBox between the host interface and the +switch port. + +```ini +sync_host_cables = True +``` + +Cables are objects which are usually maintained by hand, that's why this option is disabled by +default. With the option disabled no cable is read from or written to NetBox at all. NetBox 3.3 or +newer is needed, on older versions the option is ignored. + +netbox-sync only connects things it can find, it never creates the other end of a cable: + +* the device the neighbor reports as its system name must already exist in NetBox. The name is + matched exactly first, a short name is only matched against a FQDN if that match is unambiguous +* the port the neighbor reports must already exist as an interface of that device. Long and short + interface names are matched against each other, so a reported `FastEthernet0/16` also matches an + interface named `Fa0/16` in NetBox. CDP reports the port ID, LLDP additionally reports a port + description and both are tried +* both interfaces must already exist in NetBox. An interface which was just discovered gets its + cable during the next run +* neither of the two interfaces may be connected already. A cable which was created by hand or which + connects to a different port is never changed or deleted, it is reported at log level `DEBUG` + instead + +Cables created by this source are tagged like every other object and are marked as orphaned and +pruned once the host stops reporting that neighbor (see `prune_enabled`). Disabling the option again +leaves all previously created cables untouched in NetBox. + ### Filtering VM Disk Information VM disks are synchronized between vCenter and NetBox. Since NetBox 3.7.0, virtual disks are tracked as separate objects linked to VMs. In some scenarios, such as when temporary disks are attached to VMs during backup operations diff --git a/module/netbox/__init__.py b/module/netbox/__init__.py index 8f5696c..f0323a0 100644 --- a/module/netbox/__init__.py +++ b/module/netbox/__init__.py @@ -40,7 +40,8 @@ NBMACAddress, NBFHRPGroupItem, NBInventoryItem, - NBPowerPort + NBPowerPort, + NBCable ) primary_tag_name = "NetBox-synced" diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 377c12b..2e57bc0 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -13,7 +13,7 @@ # noinspection PyUnresolvedReferences from packaging import version -from module.common.misc import grab +from module.common.misc import grab, get_string_or_none from module.common.logging import get_logger from module.netbox.manufacturer_mapping import sanitize_manufacturer_name @@ -2428,4 +2428,93 @@ def update(self, data=None, read_from_netbox=False, source=None): super().update(data=data, read_from_netbox=read_from_netbox, source=source) + +class NBCable(NetBoxObject): + name = "cable" + api_path = "dcim/cables" + object_type = "dcim.cable" + # a cable has no natural name, the label is the only free form text attribute it has + primary_key = "label" + prune = True + # cable terminations are lists of objects since NetBox 3.3 + min_netbox_version = "3.3" + + def __init__(self, *args, **kwargs): + self.data_model = { + "label": 100, + "a_terminations": list, + "b_terminations": list, + "status": ["connected", "planned", "decommissioning"], + "type": [ + "cat3", "cat5", "cat5e", "cat6", "cat6a", "cat7", "cat7a", "cat8", + "dac-active", "dac-passive", + "mmf", "mmf-om1", "mmf-om2", "mmf-om3", "mmf-om4", "mmf-om5", + "smf", "smf-os1", "smf-os2", "aoc", "power", "usb", "coaxial" + ], + "description": 200, + "color": str, + "length": float, + "length_unit": ["km", "m", "cm", "mi", "ft", "in"], + "tags": NBTagList + } + super().__init__(*args, **kwargs) + + def format_termination(self, termination): + """ + format a single cable termination as string + + Parameters + ---------- + termination: dict + a single entry of a cable "a_terminations"/"b_terminations" list + + Returns + ------- + (str, None): the name of the terminated object, None if it can't be determined + """ + + if not isinstance(termination, dict): + return None + + # data read from NetBox contains the terminated object, data compiled by a source only the ID + termination_object = termination.get("object") + if isinstance(termination_object, dict) and termination_object.get("display") is not None: + return f"{termination_object.get('display')}" + + object_id = termination.get("object_id") + if object_id is None: + return None + + # a source only knows the ID of an interface it compiled a cable for + if termination.get("object_type") == NBInterface.object_type and self.inventory is not None: + interface_object = self.inventory.get_by_id(NBInterface, nb_id=object_id) + if interface_object is not None: + return interface_object.get_display_name(including_second_key=True) + + return f"{termination.get('object_type')} {object_id}" + + def get_display_name(self, data=None, including_second_key=False): + """ + A cable label is optional and mostly unset. Fall back to the objects this cable + connects to get a name which actually says something. + """ + + this_data = data if data is not None else self.data + + label = get_string_or_none(this_data.get(self.primary_key)) + if label is not None: + return label + + terminations = list() + for side in ["a_terminations", "b_terminations"]: + side_names = [self.format_termination(x) for x in this_data.get(side) or list()] + side_names = [x for x in side_names if x is not None] + if len(side_names) > 0: + terminations.append(", ".join(side_names)) + + if len(terminations) == 0: + return None + + return " <> ".join(terminations) + # EOF diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index fe68a71..24524f1 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -560,6 +560,16 @@ def __init__(self): will maintain all physical nics in netbox. This option will skip this part.""" , default_value=False ), + ConfigOption("sync_host_cables", + bool, + description="""Create cables in NetBox between the physical interfaces (pNICs) of an + ESXi host and the switch ports which are reported as CDP/LLDP neighbors by this host. + A cable is only created if the reported switch and the reported switch port both + already exist in NetBox and if neither of the two interfaces is cabled yet. Cables + are visible objects which are usually maintained by hand, that's why this is + disabled by default.""", + default_value=False + ), # removed settings ConfigOption("netbox_host_device_role", diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 63e97d8..1732a7f 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -80,6 +80,22 @@ class VMWareHandler(SourceBase): NBMACAddress ] + # maps the long interface name a CDP/LLDP neighbor can report to the short forms which are + # commonly used as interface name in NetBox and the other way around: Fa0/16 <> FastEthernet0/16 + # the first entry which matches a reported name wins, longer prefixes need to be listed first + interface_name_prefixes = [ + ("HundredGigabitEthernet", ["HundredGigE", "Hu"]), + ("FiftyGigabitEthernet", ["FiftyGigE", "Fi"]), + ("FortyGigabitEthernet", ["FortyGigE", "Fo"]), + ("TwentyFiveGigabitEthernet", ["TwentyFiveGigE", "25GigE", "Twe", "TF"]), + ("TenGigabitEthernet", ["TenGigE", "Te"]), + # Huawei style 10G + ("XGigabitEthernet", ["XGE", "XGi"]), + ("GigabitEthernet", ["GigE", "Gi", "GE"]), + ("FastEthernet", ["Fa"]), + ("Ethernet", ["Eth", "Et"]) + ] + source_type = "vmware" recursion_level = 0 @@ -111,6 +127,18 @@ def __init__(self, name=None): self.set_source_tag() self.site_name = f"vCenter: {name}" + # index of NetBox interface id to the cable terminated on it, compiled on demand + self.cable_index = None + + # cables are only read from and written to NetBox if this source is meant to maintain them + if self.settings.sync_host_cables is True: + if version.parse(self.inventory.netbox_api_version) < version.parse(NBCable.min_netbox_version): + log.warning(f"Option 'sync_host_cables' needs NetBox version {NBCable.min_netbox_version} " + f"or newer. Disabling it for source '{name}'.") + self.settings.sync_host_cables = False + else: + self.dependent_netbox_objects = self.dependent_netbox_objects + [NBCable] + if self.settings.enabled is False: log.info(f"Source '{name}' is currently disabled. Skipping") return @@ -142,6 +170,350 @@ def __init__(self, name=None): self.objects_to_reevaluate = list() self.parsing_objects_to_reevaluate = False + @classmethod + def get_interface_name_variants(cls, name): + """ + return all spellings of an interface name a CDP/LLDP neighbor reported + + A neighbor can report the long name of a port ("FastEthernet0/16") while the very same + interface is named with a short form in NetBox ("Fa0/16") or the other way around. + Comparing names is done case-insensitive, that's why only one spelling per variant + is returned. + + Parameters + ---------- + name: str + interface name as reported by the neighbor + + Returns + ------- + list: of all name variants, empty if no name was reported + """ + + name = get_string_or_none(name) + if name is None: + return list() + + variants = [name] + name_lower = name.lower() + + for long_prefix, short_prefixes in cls.interface_name_prefixes: + + remainder = None + if name_lower.startswith(long_prefix.lower()): + remainder = name[len(long_prefix):] + else: + for short_prefix in short_prefixes: + if not name_lower.startswith(short_prefix.lower()): + continue + short_remainder = name[len(short_prefix):] + # "Te0/1" uses the short form, "TenGigE0/1" just starts with the same letters + if len(short_remainder) > 0 and (short_remainder[0].isdigit() or short_remainder[0] in "/-"): + remainder = short_remainder + break + + if remainder is None: + continue + + variants.append(f"{long_prefix}{remainder}") + variants.extend([f"{x}{remainder}" for x in short_prefixes]) + break + + return list(dict.fromkeys(variants)) + + @staticmethod + def get_pnic_neighbor(hint): + """ + extract the neighbor a physical host interface reported via CDP or LLDP + + CDP is preferred as it reports the name of the connected switch directly. LLDP + reports the same information in a list of key/value parameters. + + Parameters + ---------- + hint: vim.host.PhysicalNic.NetworkHint + network hint of a single physical interface as returned by QueryNetworkHint() + + Returns + ------- + (dict, None): "system_name", "port_id", "port_description" and "protocol" of the + reported neighbor, None if this interface reported no usable neighbor + """ + + if hint is None: + return None + + connected_switch_port = grab(hint, "connectedSwitchPort") + if connected_switch_port is not None: + system_name = get_string_or_none(grab(connected_switch_port, "systemName")) or \ + get_string_or_none(grab(connected_switch_port, "devId")) + + if system_name is not None: + return { + "system_name": system_name, + "port_id": get_string_or_none(grab(connected_switch_port, "portId")), + "port_description": None, + "protocol": "CDP" + } + + lldp_info = grab(hint, "lldpInfo") + if lldp_info is not None: + + parameters = dict() + for parameter in grab(lldp_info, "parameter", fallback=list()): + key = get_string_or_none(grab(parameter, "key")) + value = get_string_or_none(grab(parameter, "value")) + if key is not None and value is not None: + parameters[key.lower()] = value + + system_name = parameters.get("system name") or parameters.get("systemname") + + # the port id is the interface name of the neighbor (i.e.: "XGigabitEthernet0/0/14") + port_id = parameters.get("port id") or parameters.get("portid") + if port_id is None: + port_id = get_string_or_none(grab(lldp_info, "portId")) + + # the port description is maintained by the switch admin (i.e.: "MAIN-DETAIL12/Eth1") + port_description = parameters.get("port description") or parameters.get("portdescription") + + if system_name is not None: + return { + "system_name": system_name, + "port_id": port_id, + "port_description": port_description, + "protocol": "LLDP" + } + + return None + + @staticmethod + def get_cable_interface_ids(cable): + """ + return the NetBox IDs of all interfaces a cable is terminated on + + Parameters + ---------- + cable: NBCable + the cable object to read the terminations from + + Returns + ------- + list: of NetBox interface IDs + """ + + interface_ids = list() + for side in ["a_terminations", "b_terminations"]: + for termination in grab(cable, f"data.{side}", fallback=list()): + + if not isinstance(termination, dict): + continue + if termination.get("object_type") != NBInterface.object_type: + continue + if isinstance(termination.get("object_id"), int): + interface_ids.append(termination.get("object_id")) + + return interface_ids + + def get_cable_for_interface_id(self, interface_id): + """ + return the cable which is terminated on a NetBox interface + + All cables are looked at only once, cables added afterwards are added to the index + by add_cable_to_neighbor(). + + Parameters + ---------- + interface_id: int + NetBox ID of the interface to find the cable for + + Returns + ------- + (NBCable, None): the cable terminated on this interface, None if there is none + """ + + if self.cable_index is None: + self.cable_index = dict() + for cable in self.inventory.get_all_items(NBCable): + for cable_interface_id in self.get_cable_interface_ids(cable): + self.cable_index.setdefault(cable_interface_id, cable) + + return self.cable_index.get(interface_id) + + def get_device_by_neighbor_name(self, name): + """ + find the NetBox device a CDP/LLDP neighbor reported as its system name + + An exact match always wins. A neighbor can report a FQDN while the device is named + with its short name in NetBox (or the other way around), that's why short names are + compared as well. A short name match is only accepted if it is unambiguous and if it + does not compare two different domains with each other. + + Parameters + ---------- + name: str + system name the neighbor reported + + Returns + ------- + (NBDevice, None): the matching device, None if there was no or no unique match + """ + + name = get_string_or_none(name) + if name is None: + return None + + name = name.lower() + short_name = name.split(".")[0] + + short_name_matches = list() + for device in self.inventory.get_all_items(NBDevice): + + device_name = get_string_or_none(grab(device, "data.name")) + if device_name is None: + continue + + device_name = device_name.lower() + if device_name == name: + return device + + # "sw01.dc1.example.com" and "sw01.dc2.example.com" are not the same device + if "." in name and "." in device_name: + continue + + if device_name.split(".")[0] == short_name: + short_name_matches.append(device) + + if len(short_name_matches) == 1: + return short_name_matches[0] + + if len(short_name_matches) > 1: + log.debug(f"Neighbor '{name}' matches more than one {NBDevice.name} in NetBox: " + f"{[grab(x, 'data.name') for x in short_name_matches]}") + + return None + + def get_interface_by_neighbor_port(self, device, port_names): + """ + find the interface of a device which matches one of the port names a neighbor reported + + Parameters + ---------- + device: NBDevice + the device to look for the interface on + port_names: list + port names reported by the neighbor, in the order they should be tried + + Returns + ------- + (NBInterface, None): the matching interface, None if none of the names matched + """ + + if device is None: + return None + + wanted_names = list() + for port_name in port_names: + wanted_names.extend([x.lower() for x in self.get_interface_name_variants(port_name)]) + + if len(wanted_names) == 0: + return None + + device_interfaces = dict() + for interface in self.inventory.get_all_interfaces(device): + interface_name = get_string_or_none(grab(interface, "data.name")) + if interface_name is not None: + device_interfaces.setdefault(interface_name.lower(), interface) + + for wanted_name in dict.fromkeys(wanted_names): + if device_interfaces.get(wanted_name) is not None: + return device_interfaces.get(wanted_name) + + return None + + def add_cable_to_neighbor(self, host_interface, neighbor, host_name, pnic_name): + """ + add a cable between a physical host interface and the switch port its CDP/LLDP neighbor reported + + A cable is only added if the reported switch and switch port were both found in NetBox and + if neither of the two interfaces is connected with a cable already. Cables which were created + by this source before are claimed again so they don't end up being marked as orphaned. + + Parameters + ---------- + host_interface: NBInterface + interface object of the physical host interface + neighbor: dict + neighbor data as returned by get_pnic_neighbor() + host_name: str + name of the host this interface belongs to, used for logging + pnic_name: str + name of the physical interface, used for logging + """ + + if host_interface is None or neighbor is None: + return + + log_name = f"Neighbor of interface '{pnic_name}' on host '{host_name}'" + + switch_object = self.get_device_by_neighbor_name(neighbor.get("system_name")) + if switch_object is None: + log.debug2(f"{log_name}: {NBDevice.name} '{neighbor.get('system_name')}' not found in NetBox. " + f"Not adding a cable.") + return + + port_names = [neighbor.get("port_id"), neighbor.get("port_description")] + switch_interface = self.get_interface_by_neighbor_port(switch_object, port_names) + if switch_interface is None: + log.debug2(f"{log_name}: no interface matching {[x for x in port_names if x is not None]} found on " + f"{NBDevice.name} '{grab(switch_object, 'data.name')}'. Not adding a cable.") + return + + host_interface_id = getattr(host_interface, "nb_id", 0) + switch_interface_id = getattr(switch_interface, "nb_id", 0) + + # a cable can only reference interfaces which exist in NetBox. + # an interface which was just discovered gets its cable during the next run + if host_interface_id == 0 or switch_interface_id == 0: + log.debug2(f"{log_name}: {NBInterface.name} '{host_interface.get_display_name()}' or " + f"'{switch_interface.get_display_name()}' does not exist in NetBox yet. " + f"A cable can be added during the next run.") + return + + existing_cable = self.get_cable_for_interface_id(host_interface_id) or \ + self.get_cable_for_interface_id(switch_interface_id) + + if existing_cable is not None: + + existing_interface_ids = self.get_cable_interface_ids(existing_cable) + + if host_interface_id in existing_interface_ids and switch_interface_id in existing_interface_ids: + log.debug2(f"{log_name}: cable '{existing_cable.get_display_name()}' already exists") + + # a cable this source added before is still valid and must not be marked as orphaned. + # a cable which somebody else created stays untouched and unmanaged + if self.source_tag in existing_cable.get_tags(): + existing_cable.set_source(self) + else: + log.debug(f"{log_name}: {NBInterface.name} '{host_interface.get_display_name()}' or " + f"'{switch_interface.get_display_name()}' is already connected with cable " + f"'{existing_cable.get_display_name()}'. Not adding a cable.") + + return + + log.debug2(f"{log_name}: reported via {neighbor.get('protocol')} as " + f"'{neighbor.get('system_name')}' port '{neighbor.get('port_id')}'") + + cable_object = self.inventory.add_object(NBCable, source=self, data={ + # a label is not mandatory in NetBox and stays empty, the terminations name this cable + "label": "", + "a_terminations": [{"object_type": NBInterface.object_type, "object_id": host_interface_id}], + "b_terminations": [{"object_type": NBInterface.object_type, "object_id": switch_interface_id}], + "status": "connected" + }) + + for interface_id in [host_interface_id, switch_interface_id]: + self.cable_index[interface_id] = cable_object + def create_sdk_session(self): """ Initialize SDK session with vCenter @@ -1170,6 +1542,10 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v disk_data: list data of discs which belong to a VM + Returns + ------- + tuple: the added/updated (NBDevice, NBVM) object and a dict of all interface objects + which were added/updated for it, discovered interface name as key """ if object_type not in [NBDevice, NBVM]: @@ -1396,6 +1772,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v except ValueError: log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") + interface_objects = dict() + for int_name, int_data in nic_data.items(): if nic_object_dict.get(int_name) is not None: @@ -1409,6 +1787,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v int_data, nic_ips.get(int_name, list()), vmware_object=vmware_object) + interface_objects[int_name] = nic_object + # add all interface IPs for ip_object in ip_address_objects: @@ -1457,7 +1837,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v f"'{device_vm_object.get_display_name()}'") device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) - return + return device_vm_object, interface_objects def get_parent_object_by_class(self, obj, object_class_to_find): @@ -2012,6 +2392,7 @@ def add_host(self, obj): # now iterate over all physical interfaces and collect data pnic_data_dict = dict() + pnic_neighbors = dict() pnic_hints = dict() # noinspection PyBroadException try: @@ -2100,6 +2481,18 @@ def add_host(self, obj): log.debug2(f"Host NIC with MAC '{pnic_mac_address}' excluded from sync. Skipping") continue + # collect the reported neighbor to add a cable for this interface later on + if self.settings.sync_host_cables is True: + pnic_neighbor = self.get_pnic_neighbor(pnic_hints.get(pnic_name)) + + if pnic_neighbor is not None: + pnic_neighbors[pnic_name] = pnic_neighbor + + # a CDP neighbor is already part of the description + if pnic_neighbor.get("protocol") == "LLDP": + neighbor_port = pnic_neighbor.get("port_id") or pnic_neighbor.get("port_description") + pnic_description += f" (conn: {pnic_neighbor.get('system_name')} - {neighbor_port})" + pnic_data = { "name": unquote(pnic_name), "device": None, # will be set once we found the correct device @@ -2300,9 +2693,15 @@ def add_host(self, obj): host_primary_ip6 = int_v6 # add host to inventory - self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, - vnic_data=vnic_data_dict, nic_ips=vnic_ips, - p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6, vmware_object=obj) + device_object, interface_objects = \ + self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, + vnic_data=vnic_data_dict, nic_ips=vnic_ips, + p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6, vmware_object=obj) + + # add cables to the switch ports which were reported via CDP/LLDP + if device_object is not None: + for pnic_name, pnic_neighbor in pnic_neighbors.items(): + self.add_cable_to_neighbor(interface_objects.get(pnic_name), pnic_neighbor, name, pnic_name) return diff --git a/settings-example.ini b/settings-example.ini index 3f842ac..dcf8467 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -528,6 +528,13 @@ password = super-secret ; disk synchronization. A VM with this tag will still be synced to NetBox, but its disk information won't be updated. ;vm_exclude_disk_sync_by_tag = backup-vm, veeam-job +; Create cables in NetBox between the physical interfaces (pNICs) of an ESXi host and the +; switch ports which are reported as CDP/LLDP neighbors by this host. A cable is only +; created if the reported switch and the reported switch port both already exist in NetBox +; and if neither of the two interfaces is cabled yet. Cables are visible objects which are +; usually maintained by hand, that's why this is disabled by default. +;sync_host_cables = False + [source/my-redfish-example] ; Defines if this source is enabled or not diff --git a/tests/test_vmware_host_cables.py b/tests/test_vmware_host_cables.py new file mode 100644 index 0000000..89d034e --- /dev/null +++ b/tests/test_vmware_host_cables.py @@ -0,0 +1,472 @@ +""" +Cables from the CDP/LLDP neighbors an ESXi host reports for its physical interfaces. + +The vcsim captures carry no CDP/LLDP data, so the parts which turn a reported neighbor +into a cable are tested against a hand built inventory. What the vcsim run has to prove +is that the feature stays completely out of the way while `sync_host_cables` is disabled. +""" +from types import SimpleNamespace + +import pytest + +from module.netbox.object_classes import NBCable, NBDevice, NBInterface, NBTag +from module.sources import instantiate_sources +from module.sources.vmware.connection import VMWareHandler + + +def cdp_hint(system_name=None, device_id=None, port_id=None): + """a QueryNetworkHint() result of an interface which sees a CDP neighbor""" + + return SimpleNamespace( + connectedSwitchPort=SimpleNamespace(systemName=system_name, devId=device_id, portId=port_id), + lldpInfo=None + ) + + +def lldp_hint(port_id=None, **parameters): + """a QueryNetworkHint() result of an interface which sees an LLDP neighbor""" + + return SimpleNamespace( + connectedSwitchPort=None, + lldpInfo=SimpleNamespace( + portId=port_id, + parameter=[SimpleNamespace(key=key, value=value) for key, value in parameters.items()] + ) + ) + + +def termination(object_id, object_type="dcim.interface"): + return {"object_type": object_type, "object_id": object_id} + + +@pytest.fixture +def cable_source(inventory): + """ + A VMware source handler with nothing but the state the cable code touches, on the + fresh in-memory inventory. Building it without __init__ keeps vCenter out of the way. + """ + source = object.__new__(VMWareHandler) + source.inventory = inventory + source.name = "test" + source.source_tag = "Source: test" + source.cable_index = None + + inventory.add_update_object(NBTag, data={"name": source.source_tag}) + + return source + + +@pytest.fixture +def netbox_interface(inventory, cable_source): + """Returns a function adding a device interface which already exists in NetBox.""" + + devices = {} + + def _add(device_name, interface_name, nb_id): + device = devices.get(device_name) + if device is None: + device = inventory.add_object(NBDevice, data={"name": device_name}, source=cable_source) + devices[device_name] = device + + interface = inventory.add_object(NBInterface, data={"name": interface_name, "device": device}, + source=cable_source) + interface.nb_id = nb_id + interface.is_new = False + + return interface + + return _add + + +# --- interface name variants ------------------------------------------------------------------- + +@pytest.mark.parametrize("reported, expected", [ + ("FastEthernet0/16", "Fa0/16"), + ("Fa0/16", "FastEthernet0/16"), + ("Te1/0/1", "TenGigabitEthernet1/0/1"), + ("Te1/0/1", "TenGigE1/0/1"), + ("TenGigE0/0/1", "Te0/0/1"), + ("Twe1/0/5", "TwentyFiveGigabitEthernet1/0/5"), + ("TwentyFiveGigabitEthernet1/0/5", "TF1/0/5"), + ("XGigabitEthernet0/0/14", "XGE0/0/14"), + ("GE1/0/1", "GigabitEthernet1/0/1"), + ("GigabitEthernet1/0/1", "Gi1/0/1"), + ("Ethernet1/1", "Eth1/1"), + ("Eth1/1", "Ethernet1/1"), +]) +def test_interface_name_variants_contain_the_other_spelling(reported, expected): + assert expected in VMWareHandler.get_interface_name_variants(reported) + + +def test_interface_name_variants_start_with_the_reported_name_and_are_unique(): + variants = VMWareHandler.get_interface_name_variants("FastEthernet0/16") + + assert variants[0] == "FastEthernet0/16" + assert len(variants) == len(set(variants)) + + +@pytest.mark.parametrize("reported", [None, "", " "]) +def test_interface_name_variants_of_an_unreported_port(reported): + assert VMWareHandler.get_interface_name_variants(reported) == [] + + +def test_interface_name_variants_do_not_confuse_ten_and_twentyfive_gigabit(): + variants = VMWareHandler.get_interface_name_variants("Te1/0/1") + + assert "TwentyFiveGigabitEthernet1/0/1" not in variants + + +def test_interface_name_variants_keep_names_which_only_start_like_a_short_form(): + # "TenGigE" is not "Te" plus a port number and "Bundle-Ether1" is no Ethernet port at all + assert VMWareHandler.get_interface_name_variants("TenGigE") == ["TenGigE"] + assert VMWareHandler.get_interface_name_variants("Bundle-Ether1") == ["Bundle-Ether1"] + + +# --- reading the neighbor of a physical interface ------------------------------------------------ + +def test_no_neighbor_reported(): + assert VMWareHandler.get_pnic_neighbor(None) is None + assert VMWareHandler.get_pnic_neighbor(SimpleNamespace(connectedSwitchPort=None, lldpInfo=None)) is None + + +def test_cdp_neighbor(): + neighbor = VMWareHandler.get_pnic_neighbor(cdp_hint(system_name=" sw01.example.com ", port_id=" Gi1/0/1 ")) + + assert neighbor == { + "system_name": "sw01.example.com", + "port_id": "Gi1/0/1", + "port_description": None, + "protocol": "CDP" + } + + +def test_cdp_neighbor_falls_back_to_the_device_id(): + neighbor = VMWareHandler.get_pnic_neighbor(cdp_hint(system_name="", device_id="sw01", port_id=None)) + + assert neighbor.get("system_name") == "sw01" + assert neighbor.get("port_id") is None + + +def test_cdp_without_a_name_falls_through_to_lldp(): + hint = lldp_hint(**{"System Name": "sw01", "Port ID": "Gi1/0/1"}) + hint.connectedSwitchPort = SimpleNamespace(systemName=None, devId=None, portId="Gi1/0/1") + + assert VMWareHandler.get_pnic_neighbor(hint).get("protocol") == "LLDP" + + +def test_lldp_neighbor(): + neighbor = VMWareHandler.get_pnic_neighbor(lldp_hint(**{ + "System Name": "sw01.example.com", + "Port ID": "XGigabitEthernet0/0/14", + "Port Description": "MAIN-DETAIL12/Eth1" + })) + + assert neighbor == { + "system_name": "sw01.example.com", + "port_id": "XGigabitEthernet0/0/14", + "port_description": "MAIN-DETAIL12/Eth1", + "protocol": "LLDP" + } + + +def test_lldp_neighbor_port_id_attribute_is_used_if_no_parameter_was_reported(): + neighbor = VMWareHandler.get_pnic_neighbor(lldp_hint(port_id=42, **{"System Name": "sw01"})) + + assert neighbor.get("port_id") == "42" + + +def test_lldp_neighbor_without_a_system_name_is_unusable(): + assert VMWareHandler.get_pnic_neighbor(lldp_hint(**{"Port ID": "Gi1/0/1"})) is None + + +def test_lldp_parameters_which_are_not_a_name_are_ignored(): + hint = lldp_hint(**{"System Name": ["sw01", "sw02"], "Port ID": "Gi1/0/1"}) + + assert VMWareHandler.get_pnic_neighbor(hint) is None + + +# --- cable terminations --------------------------------------------------------------------------- + +def test_cable_interface_ids_of_both_sides(inventory, cable_source): + cable = inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 5, "label": "", "a_terminations": [termination(10)], "b_terminations": [termination(20)] + }) + + assert VMWareHandler.get_cable_interface_ids(cable) == [10, 20] + + +def test_cable_interface_ids_ignore_terminations_which_are_no_interface(inventory, cable_source): + cable = inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 5, + "label": "", + "a_terminations": [termination(10, object_type="dcim.frontport"), "broken", {"object_id": None}], + "b_terminations": [termination(20)] + }) + + assert VMWareHandler.get_cable_interface_ids(cable) == [20] + + +# --- how a cable is named ----------------------------------------------------------------------- + +def test_a_cable_is_named_after_the_interfaces_it_connects(inventory, cable_source, netbox_interface): + netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Fa0/16", 200) + cable = inventory.add_object(NBCable, source=cable_source, data={ + "label": "", "a_terminations": [termination(100)], "b_terminations": [termination(200)] + }) + + assert cable.get_display_name() == "vmnic0 (esx01) <> Fa0/16 (sw01)" + + +def test_a_cable_read_from_netbox_is_named_after_its_terminations(inventory, cable_source): + cable = inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 9, + "label": "", + "a_terminations": [{"object_type": "dcim.interface", "object_id": 1, + "object": {"display": "Gi1/0/2 (sw01)"}}], + "b_terminations": [{"object_type": "dcim.interface", "object_id": 2, + "object": {"display": "vmnic1 (esx02)"}}] + }) + + assert cable.get_display_name() == "Gi1/0/2 (sw01) <> vmnic1 (esx02)" + + +def test_a_label_someone_set_names_the_cable(inventory, cable_source): + cable = inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 10, "label": "patch-42", "a_terminations": [termination(1)], "b_terminations": [termination(2)] + }) + + assert cable.get_display_name() == "patch-42" + + +# --- adding a cable to the reported neighbor --------------------------------------------------- + +def test_cable_is_added_between_both_reported_ends(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Fa0/16", 200) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "FastEthernet0/16", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + cables = list(inventory.get_all_items(NBCable)) + assert len(cables) == 1 + assert VMWareHandler.get_cable_interface_ids(cables[0]) == [100, 200] + assert cables[0].data.get("status") == "connected" + assert cables[0].source is cable_source + + +def test_cable_is_added_for_a_port_matched_by_its_description(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "MAIN-DETAIL12/Eth1", 200) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "XGigabitEthernet0/0/14", + "port_description": "MAIN-DETAIL12/Eth1", "protocol": "LLDP"}, "esx01", "vmnic0") + + assert len(list(inventory.get_all_items(NBCable))) == 1 + + +def test_a_neighbor_reporting_a_fqdn_matches_the_short_device_name(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Gi1/0/1", 200) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01.example.com", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert len(list(inventory.get_all_items(NBCable))) == 1 + + +def test_an_ambiguous_short_device_name_is_not_matched(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01.dc1.example.com", "Gi1/0/1", 200) + netbox_interface("sw01.dc2.example.com", "Gi1/0/1", 300) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +def test_two_different_domains_are_two_different_devices(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01.dc1.example.com", "Gi1/0/1", 200) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01.dc2.example.com", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +def test_no_cable_if_the_neighbor_is_not_in_netbox(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +def test_no_cable_if_the_reported_port_is_not_in_netbox(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Gi1/0/2", 200) + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +def test_no_cable_before_both_interfaces_exist_in_netbox(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + switch_interface = netbox_interface("sw01", "Gi1/0/1", 200) + # a switch port which was discovered during this very run has no NetBox ID yet + switch_interface.nb_id = 0 + + cable_source.add_cable_to_neighbor( + host_interface, {"system_name": "sw01", "port_id": "Gi1/0/1", + "port_description": None, "protocol": "CDP"}, "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +def test_an_existing_cable_is_not_added_a_second_time(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Gi1/0/1", 200) + inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 7, "label": "", "a_terminations": [termination(200)], "b_terminations": [termination(100)], + "tags": [{"name": cable_source.source_tag}] + }) + inventory.resolve_relations() + + neighbor = {"system_name": "sw01", "port_id": "Gi1/0/1", "port_description": None, "protocol": "CDP"} + cable_source.add_cable_to_neighbor(host_interface, neighbor, "esx01", "vmnic0") + + cables = list(inventory.get_all_items(NBCable)) + assert len(cables) == 1 + # the cable is still reported by this source, so it must not be marked as orphaned + assert cables[0].source is cable_source + + +def test_a_cable_created_by_somebody_else_is_not_claimed(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Gi1/0/1", 200) + inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 7, "label": "", "a_terminations": [termination(100)], "b_terminations": [termination(200)] + }) + inventory.resolve_relations() + + neighbor = {"system_name": "sw01", "port_id": "Gi1/0/1", "port_description": None, "protocol": "CDP"} + cable_source.add_cable_to_neighbor(host_interface, neighbor, "esx01", "vmnic0") + + cables = list(inventory.get_all_items(NBCable)) + assert len(cables) == 1 + assert cables[0].source is None + + +def test_an_interface_which_is_cabled_somewhere_else_is_left_alone(inventory, cable_source, netbox_interface): + host_interface = netbox_interface("esx01", "vmnic0", 100) + netbox_interface("sw01", "Gi1/0/1", 200) + inventory.add_object(NBCable, read_from_netbox=True, data={ + "id": 7, "label": "", "a_terminations": [termination(100)], "b_terminations": [termination(999)] + }) + inventory.resolve_relations() + + neighbor = {"system_name": "sw01", "port_id": "Gi1/0/1", "port_description": None, "protocol": "CDP"} + cable_source.add_cable_to_neighbor(host_interface, neighbor, "esx01", "vmnic0") + + assert len(list(inventory.get_all_items(NBCable))) == 1 + + +def test_two_hosts_reporting_each_other_get_one_cable(inventory, cable_source, netbox_interface): + """A direct link between two ESXi hosts is reported from both sides during the same run.""" + + first = netbox_interface("esx01", "vmnic0", 100) + second = netbox_interface("esx02", "vmnic0", 200) + + cable_source.add_cable_to_neighbor( + first, {"system_name": "esx02", "port_id": "vmnic0", "port_description": None, "protocol": "LLDP"}, + "esx01", "vmnic0") + cable_source.add_cable_to_neighbor( + second, {"system_name": "esx01", "port_id": "vmnic0", "port_description": None, "protocol": "LLDP"}, + "esx02", "vmnic0") + + assert len(list(inventory.get_all_items(NBCable))) == 1 + + +def test_a_pnic_without_an_interface_object_is_skipped(inventory, cable_source, netbox_interface): + netbox_interface("sw01", "Gi1/0/1", 200) + + cable_source.add_cable_to_neighbor( + None, {"system_name": "sw01", "port_id": "Gi1/0/1", "port_description": None, "protocol": "CDP"}, + "esx01", "vmnic0") + + assert list(inventory.get_all_items(NBCable)) == [] + + +# --- the option gates the whole feature --------------------------------------------------------- + +def test_disabled_by_default(vcsim, inventory, load_config, vmware_settings): + load_config(vmware_settings) + sources = instantiate_sources() + assert sources and sources[0].init_successful + + assert sources[0].settings.sync_host_cables is False + + +def test_nothing_cable_related_happens_while_the_option_is_disabled(vcsim, inventory, load_config, + vmware_settings, monkeypatch): + looked_at = list() + monkeypatch.setattr(VMWareHandler, "get_pnic_neighbor", staticmethod(looked_at.append)) + + load_config(vmware_settings) + sources = instantiate_sources() + assert sources and sources[0].init_successful + source = sources[0] + + # a cable which is not read from NetBox can not be changed, tagged or pruned by this run + assert NBCable not in source.dependent_netbox_objects + + inventory.resolve_relations() + source.apply() + + assert looked_at == [], "the CDP/LLDP neighbor of a pNIC must not be read while the option is disabled" + assert list(inventory.get_all_items(NBCable)) == [], "no cable may be created while the option is disabled" + + +def test_enabling_the_option_reads_cables_from_netbox(vcsim, inventory, load_config, vmware_settings): + load_config(vmware_settings + "\nsync_host_cables = True\n") + sources = instantiate_sources() + assert sources and sources[0].init_successful + + assert sources[0].settings.sync_host_cables is True + assert NBCable in sources[0].dependent_netbox_objects + + +def test_the_option_is_disabled_on_a_netbox_which_is_too_old(vcsim, inventory, load_config, vmware_settings): + inventory.netbox_api_version = "3.2.0" + + load_config(vmware_settings + "\nsync_host_cables = True\n") + sources = instantiate_sources() + assert sources and sources[0].init_successful + + assert sources[0].settings.sync_host_cables is False + assert NBCable not in sources[0].dependent_netbox_objects + + +def test_a_sync_with_the_option_enabled_still_works(vcsim, inventory, load_config, vmware_settings): + load_config(vmware_settings + "\nsync_host_cables = True\n") + sources = instantiate_sources() + assert sources and sources[0].init_successful + + inventory.resolve_relations() + sources[0].apply() + + assert list(inventory.get_all_items(NBDevice)), "hosts must still be synced" + # none of the captured vcsim inventories reports a CDP/LLDP neighbor + assert list(inventory.get_all_items(NBCable)) == []