Tests for sw upgrade testing (#37)

* Tests for sw upgrade testing

* Fix diff_helpers.py for flat data structures.

* Remove xfail from tests passing after fix.

* Add bgp prefix tolerance test + improve tolerance check

* Update tests

* Post-review updates.

* Extend sw_upgrade tests with cisco_nxos platform.

Co-authored-by: Patryk Szulczewski <[email protected]>

Author: pszulczewski

netcompare/check_types.py

@@ -149,9 +149,17 @@ def evaluate(self, value_to_compare: Any, reference_data: Any, tolerance: int) -
     def _remove_within_tolerance(self, diff: Dict, tolerance: int) -> None:
         """Recursively look into diff and apply tolerance check, remove reported difference when within tolerance."""
 
-        def _within_tolerance(*, old_value: float, new_value: float) -> bool:
+        def _make_float(value):
+            """Make float, treat non-convertable as 0."""
+            try:
+                return float(value)
+            except ValueError:
+                return 0
+
+        def _within_tolerance(*, old_value: Union[str, int, float], new_value: Union[str, int, float]) -> bool:
             """Return True if new value is within the tolerance range of the previous value."""
             tolerance_factor = tolerance / 100
+            old_value, new_value = _make_float(old_value), _make_float(new_value)
             max_diff = old_value * tolerance_factor
             return (old_value - max_diff) < new_value < (old_value + max_diff)
 

netcompare/utils/diff_helpers.py

@@ -29,13 +29,17 @@ def get_diff_iterables_items(diff_result: Mapping) -> Dict:
     items_removed = diff_result.get("iterable_item_removed")
     if items_removed:
         for key, value in items_removed.items():
-            key, *_ = get_dict_keys.match(key).groups()
+            re_key = get_dict_keys.match(key)
+            if re_key:  # Change only if there is a match, otherwise retain the key.
+                key, *_ = re_key.groups()
             result[key]["missing"].append(value)
 
     items_added = diff_result.get("iterable_item_added")
     if items_added:
         for key, value in items_added.items():
-            key, *_ = get_dict_keys.match(key).groups()
+            re_key = get_dict_keys.match(key)
+            if re_key:
+                key, *_ = re_key.groups()
             result[key]["new"].append(value)
 
     return result
@@ -58,6 +62,8 @@ def fix_deepdiff_key_names(obj: Mapping) -> Dict:
     result = {}
     for key, value in obj.items():
         key_parts = re.findall(REGEX_PATTERN_RELEVANT_KEYS, key)
+        if not key_parts:  # If key parts can't be find, keep original key so data is not lost.
+            key_parts = [key.replace("root", "index_element")]  # replace root from DeepDiff with more meaningful name.
         partial_res = group_value(key_parts, value)
         dict_merger(result, partial_res)
     return result

tests/mock/sw_upgrade/arista_eos_show_interface.json

@@ -0,0 +1,72 @@
+[
+    {
+        "interface": "Ethernet1",
+        "link_status": "up",
+        "protocol_status": "up (connected)",
+        "hardware_type": "Ethernet",
+        "address": "0800.27dc.5443",
+        "bia": "",
+        "description": "",
+        "ip_address": "172.16.1.1/24",
+        "mtu": "1500",
+        "bandwidth": "10000000 kbit",
+        "interface_up_time": "14 minutes, 2 seconds",
+        "link_status_change": "1"
+    },
+    {
+        "interface": "Ethernet49/1",
+        "link_status": "administratively down",
+        "protocol_status": "notpresent (disabled)",
+        "hardware_type": "Ethernet",
+        "address": "fcbd.67e2.b922",
+        "bia": "fcbd.67e2.b922",
+        "description": "Connects to Ethernet1 on localhost",
+        "ip_address": "172.16.2.1/24",
+        "mtu": "9214",
+        "bandwidth": "100000000 kbit",
+        "interface_up_time": "6 days, 11 hours, 16 minutes, 54 seconds",
+        "link_status_change": "1"
+    },
+    {
+        "interface": "Loopback0",
+        "link_status": "up",
+        "protocol_status": "up (connected)",
+        "hardware_type": "Loopback",
+        "address": "",
+        "bia": "",
+        "description": "",
+        "ip_address": "1.1.1.1/32",
+        "mtu": "65535",
+        "bandwidth": "",
+        "interface_up_time": "7 seconds",
+        "link_status_change": ""
+    },
+    {
+        "interface": "Port-Channel1",
+        "link_status": "down",
+        "protocol_status": "lowerlayerdown (notconnect)",
+        "hardware_type": "Port-Channel",
+        "address": "0000.0000.0000",
+        "bia": "",
+        "description": "",
+        "ip_address": "",
+        "mtu": "9214",
+        "bandwidth": "",
+        "interface_up_time": "10 minutes, 21 seconds",
+        "link_status_change": "1"
+    },
+    {
+        "interface": "Management1",
+        "link_status": "up",
+        "protocol_status": "up (connected)",
+        "hardware_type": "Ethernet",
+        "address": "0800.27a5.02b7",
+        "bia": "0800.27a5.02b7",
+        "description": "",
+        "ip_address": "",
+        "mtu": "1500",
+        "bandwidth": "1000000 kbit",
+        "interface_up_time": "14 minutes, 4 seconds",
+        "link_status_change": "3"
+    }
+]

tests/mock/sw_upgrade/arista_eos_show_ip_bgp_summary.json

@@ -0,0 +1,47 @@
+[
+    {
+        "router_id": "10.0.65.72",
+        "local_as": "64911",
+        "vrf": "default",
+        "bgp_neigh": "10.10.128.0",
+        "neigh_as": "65292",
+        "msg_rcvd": "0",
+        "msg_sent": "0",
+        "in_queue": "0",
+        "out_queue": "0",
+        "up_down": "79d05h",
+        "state": "Active",
+        "state_pfxrcd": "",
+        "state_pfxacc": ""
+    },
+    {
+        "router_id": "10.0.65.72",
+        "local_as": "64911",
+        "vrf": "default",
+        "bgp_neigh": "10.106.0.153",
+        "neigh_as": "64832",
+        "msg_rcvd": "0",
+        "msg_sent": "0",
+        "in_queue": "0",
+        "out_queue": "0",
+        "up_down": "79d05h",
+        "state": "Idle(NoIf)",
+        "state_pfxrcd": "",
+        "state_pfxacc": ""
+    },
+    {
+        "router_id": "10.0.65.72",
+        "local_as": "64911",
+        "vrf": "default",
+        "bgp_neigh": "10.106.0.157",
+        "neigh_as": "64833",
+        "msg_rcvd": "114056",
+        "msg_sent": "118481",
+        "in_queue": "0",
+        "out_queue": "0",
+        "up_down": "79d05h",
+        "state": "Estab",
+        "state_pfxrcd": "1",
+        "state_pfxacc": "1"
+    }
+]

tests/mock/sw_upgrade/arista_eos_show_ip_ospf_neighbors.json

@@ -0,0 +1,20 @@
+[
+    {
+        "address": "10.3.4.3",
+        "interface": "Ethernet2",
+        "state": "FULL/BDR",
+        "neighbor_id": "3.3.3.3",
+        "vrf": "default",
+        "priority": "1",
+        "dead_time": "00:00:34"
+    },
+    {
+        "address": "10.2.4.2",
+        "interface": "Ethernet3",
+        "state": "FULL",
+        "neighbor_id": "2.2.2.2",
+        "vrf": "default",
+        "priority": "1",
+        "dead_time": "00:00:38"
+    }
+]