Unit tests for validate methods (#43)

* draft regex base function

* fix file import

* draft regex implementation

* fix logic into regex result

* remove assertion

* draft test for range check type

* add match/no-match logic

* commit save

* add assertion for check option

* add assertions for range type

* add assertion for strings

* commit save

* draft operator check type

* complete validate method for check_type operator

* work on oeprator_evaluator method

* drat operator all-same

* add all-same operator

* add contains operator

* add not-contains. Update contains logic

* add is-gt and is-lt operators

* add range operators

* add operator lib for DRY code

* fix some lint errors

* Update netcompare/operator.py

Not sure about this...if you look at the if statements, your suggestion would brake the logic...or am I missing something?

Co-authored-by: Patryk Szulczewski <[email protected]>

* work on comments

* working on operator args

* fix tests

* improve validator switch logic

* fix pytests

* tolerance validate tests

* add validate parameter test

* add mode to param match check type

* add regex validate tests

* working on operator validato tests

* complete validate tests for operator

* paramet tests

* Update as per comments

* fix as in comment PR

Co-authored-by: Patryk Szulczewski <[email protected]>

Author: lvrfrc87

netcompare/check_types.py

@@ -134,9 +134,9 @@ def validate(**kwargs) -> None:
         # reference_data = getattr(kwargs, "reference_data")
         tolerance = kwargs.get("tolerance")
         if not tolerance:
-            raise ValueError("Tolerance argument is mandatory for Tolerance Check Type.")
+            raise ValueError("'tolerance' argument is mandatory for Tolerance Check Type.")
         if not isinstance(tolerance, int):
-            raise ValueError(f"Tolerance argument must be an integer, and it's {type(tolerance)}.")
+            raise ValueError(f"Tolerance argument's value must be an integer. You have: {type(tolerance)}.")
 
     def evaluate(self, value_to_compare: Any, reference_data: Any, tolerance: int) -> Tuple[Dict, bool]:
         """Returns the difference between values and the boolean. Overwrites method in base class."""
@@ -181,23 +181,23 @@ def validate(**kwargs) -> None:
         mode_options = ["match", "no-match"]
         params = kwargs.get("params")
         if not params:
-            raise ValueError("Params argument is mandatory for ParameterMatch Check Type.")
+            raise ValueError("'params' argument is mandatory for ParameterMatch Check Type.")
         if not isinstance(params, dict):
-            raise ValueError(f"Params argument must be a dict, and it's {type(params)}.")
+            raise ValueError(f"'params' argument must be a dict. You have: {type(params)}.")
 
         mode = kwargs.get("mode")
         if not mode:
-            raise ValueError("Mode argument is mandatory for ParameterMatch Check Type.")
-        if not isinstance(mode, str):
-            raise ValueError(f"Mode argument must be a string, and it's {type(mode)}.")
+            raise ValueError("'mode' argument is mandatory for ParameterMatch Check Type.")
         if mode not in mode_options:
-            raise ValueError(f"Mode argument should be {mode_options}, and it's {mode}")
+            raise ValueError(
+                f"'mode' argument should be one of the following: {', '.join(mode_options)}. You have: {mode}"
+            )
 
     def evaluate(self, value_to_compare: Mapping, params: Dict, mode: str) -> Tuple[Dict, bool]:
         """Parameter Match evaluator implementation."""
         self.validate(params=params, mode=mode)
         # TODO: we don't use the mode?
-        evaluation_result = parameter_evaluator(value_to_compare, params)
+        evaluation_result = parameter_evaluator(value_to_compare, params, mode)
         return evaluation_result, not evaluation_result
 
 
@@ -210,17 +210,15 @@ def validate(**kwargs) -> None:
         mode_options = ["match", "no-match"]
         regex = kwargs.get("regex")
         if not regex:
-            raise ValueError("Params argument is mandatory for Regex Match Check Type.")
+            raise ValueError("'regex' argument is mandatory for Regex Check Type.")
         if not isinstance(regex, str):
-            raise ValueError(f"Params argument must be a string, and it's {type(regex)}.")
+            raise ValueError(f"'regex' argument must be a string. You have: {type(regex)}.")
 
         mode = kwargs.get("mode")
         if not mode:
-            raise ValueError("Mode argument is mandatory for Regex Match Check Type.")
-        if not isinstance(mode, str):
-            raise ValueError(f"Mode argument must be a string, and it's {type(mode)}.")
+            raise ValueError("'mode' argument is mandatory for Regex Check Type.")
         if mode not in mode_options:
-            raise ValueError(f"Mode argument should be {mode_options}, and it's {mode}")
+            raise ValueError(f"'mode' argument should be {mode_options}. You have: {mode}")
 
     def evaluate(self, value_to_compare: Mapping, regex: str, mode: str) -> Tuple[Mapping, bool]:
         """Regex Match evaluator implementation."""
@@ -250,13 +248,19 @@ def validate(**kwargs) -> None:
         )
 
         # Validate "params" argument is not None.
-        if not kwargs:
-            raise KeyError(f"'params' argument must be provided. You have {kwargs}. Read the docs for more info.")
+        if not kwargs or list(kwargs.keys())[0] != "params":
+            raise ValueError(f"'params' argument must be provided. You have: {list(kwargs.keys())[0]}.")
+
+        params_key = kwargs["params"].get("mode")
+        params_value = kwargs["params"].get("operator_data")
+
+        if not params_key or not params_value:
+            raise ValueError(
+                f"'mode' and 'operator_data' arguments must be provided. You have: {list(kwargs['params'].keys())}."
+            )
 
-        params_key = kwargs["mode"]
-        params_value = kwargs["operator_data"]
         # Validate "params" value is legal.
-        if all(params_key in operator for operator in valid_options):
+        if all(params_key not in sub_element for element in valid_options for sub_element in element):
             raise ValueError(
                 f"'params' value must be one of the following: {[sub_element for element in valid_options for sub_element in element]}. You have: {params_key}"
             )
@@ -265,42 +269,46 @@ def validate(**kwargs) -> None:
             # "is-in", "not-in", "in-range", "not-range" requires an iterable
             if not isinstance(params_value, (list, tuple)):
                 raise ValueError(
-                    f"Range check-option {in_operators} must have value of type list or tuple. i.e: dict(not-in=('Idle', 'Down'). You have: {params_value} of type {type(params_value)}You have: {params_value} of type {type(params_value)}"
+                    f"check options {in_operators} must have value of type list or tuple. i.e: dict(not-in=('Idle', 'Down'). You have: {params_value} of type {type(params_value)}."
                 )
 
             # "in-range", "not-range" requires int or float where value at index 0 is lower than value at index 1
             if params_key in ("in-range", "not-range"):
-                if not isinstance(params_value[0], (int, float)) and not isinstance(params_value[1], float, int):
+                if (
+                    len(params_value) != 2
+                    or not isinstance(params_value[0], (int, float))
+                    or not isinstance(params_value[1], (float, int))
+                ):
                     raise ValueError(
-                        f"Range check-option {params_key} must have value of type list or tuple with items of type float or int. i.e: dict(not-range=(70000000, 80000000). You have: {params_value} of type {type(params_value)}"
+                        f"'range' check-option {params_key} must have value of type list or tuple with items of type float or int. i.e: dict(not-range=(70000000, 80000000). You have: {params_value}."
                     )
                 if not params_value[0] < params_value[1]:
                     raise ValueError(
-                        f"'range' and 'not-range' must have value at index 0 lower than value at index 1. i.e: dict(not-range=(70000000, 80000000). You have: {params_value} of type {type(params_value)}"
+                        f"'range' and 'not-range' must have value at index 0 lower than value at index 1. i.e: dict(not-range=(70000000, 80000000). You have: {params_value}."
                     )
 
         # "is-gt","is-lt"  require either int() or float()
         elif params_key in number_operators and not isinstance(params_value, (float, int)):
             raise ValueError(
-                f"Check-option {number_operators} must have value of type float or int. i.e: dict(is-lt=50). You have: {params_value} of type {type(params_value)}"
+                f"check options {number_operators} must have value of type float or int. You have: {params_value} of type {type(params_value)}"
             )
 
         # "contains", "not-contains" require string.
         elif params_key in string_operators and not isinstance(params_value, str):
             raise ValueError(
-                f"Range check-option {string_operators} must have value of type string. i.e: dict(contains='EVPN'). You have: {params_value} of type {type(params_value)}"
+                f"check options {string_operators} must have value of type string. You have: {params_value} of type {type(params_value)}"
             )
 
         # "all-same" requires boolean True or False
         elif params_key in bool_operators and not isinstance(params_value, bool):
             raise ValueError(
-                f"Range check-option {bool_operators} must have value of type bool. i.e: dict(all-same=True). You have: {params_value} of type {type(params_value)}"
+                f"check option all-same must have value of type bool. You have: {params_value} of type {type(params_value)}"
             )
 
     def evaluate(self, value_to_compare: Any, params: Any) -> Tuple[Mapping, bool]:
         """Operator evaluator implementation."""
         self.validate(**params)
         # For naming consistency
         reference_data = params
-        evaluation_result = operator_evaluator(reference_data, value_to_compare)
+        evaluation_result = operator_evaluator(reference_data["params"], value_to_compare)
         return evaluation_result, not evaluation_result

netcompare/evaluators.py

@@ -36,7 +36,7 @@ def diff_generator(pre_result: Any, post_result: Any) -> Dict:
     return fix_deepdiff_key_names(result)
 
 
-def parameter_evaluator(values: Mapping, parameters: Mapping) -> Dict:
+def parameter_evaluator(values: Mapping, parameters: Mapping, mode: str) -> Dict:
     """Parameter Match evaluator engine.
 
     Args:
@@ -71,7 +71,9 @@ def parameter_evaluator(values: Mapping, parameters: Mapping) -> Dict:
         inner_value = list(value.values())[0]
 
         for parameter_key, parameter_value in parameters.items():
-            if inner_value[parameter_key] != parameter_value:
+            if mode == "match" and inner_value[parameter_key] != parameter_value:
+                result_item[parameter_key] = inner_value[parameter_key]
+            elif mode == "no-match" and inner_value[parameter_key] == parameter_value:
                 result_item[parameter_key] = inner_value[parameter_key]
 
         if result_item:

tests/test_operators.py

@@ -198,7 +198,7 @@ def test_operator(filename, check_type_str, evaluate_args, path, expected_result
     # There is not concept of "pre" and "post" in operator.
     data = load_json_file("api", filename)
     value = check.get_value(data, path)
-    actual_results = check.evaluate(value, **evaluate_args)
+    actual_results = check.evaluate(value, evaluate_args)
     assert actual_results == expected_result, ASSERT_FAIL_MESSAGE.format(
         output=actual_results, expected_output=expected_result
     )

tests/test_type_checks.py

@@ -64,31 +64,6 @@ def tests_exceptions_init(check_type_str, exception_type, expected_in_output):
     assert expected_in_output in error.value.__str__()
 
 
-exception_tests_eval = [
-    (
-        "parameter_match",
-        {"value_to_compare": {}, "mode": "some mode", "params": {"some": "thing"}},
-        ValueError,
-        "Mode argument should be",
-    ),
-    (
-        "regex",
-        {"value_to_compare": {}, "mode": "some mode", "regex": "some regex"},
-        ValueError,
-        "Mode argument should be",
-    ),
-]
-
-
-@pytest.mark.parametrize("check_type_str, evaluate_args, exception_type, expected_in_output", exception_tests_eval)
-def tests_exceptions_eval(check_type_str, evaluate_args, exception_type, expected_in_output):
-    """Tests exceptions when calling .evaluate() method."""
-    with pytest.raises(exception_type) as error:
-        check = CheckType.init(check_type_str)
-        check.evaluate(**evaluate_args)
-    assert expected_in_output in error.value.__str__()
-
-
 exact_match_test_values_no_change = (
     "exact_match",
     {},
@@ -287,9 +262,25 @@ def test_checks(folder_name, check_type_str, evaluate_args, path, expected_resul
         False,
     ),
 )
+parameter_no_match_api = (
+    "pre.json",
+    "parameter_match",
+    {"mode": "no-match", "params": {"localAsn": "65130.1100", "linkType": "external"}},
+    "result[0].vrfs.default.peerList[*].[$peerAddress$,localAsn,linkType]",
+    (
+        {
+            "10.1.0.0": {"linkType": "external"},
+            "10.2.0.0": {"localAsn": "65130.1100", "linkType": "external"},
+            "10.64.207.255": {"localAsn": "65130.1100", "linkType": "external"},
+        },
+        False,
+    ),
+)
 
 
-@pytest.mark.parametrize("filename, check_type_str, evaluate_args, path, expected_result", [parameter_match_api])
+@pytest.mark.parametrize(
+    "filename, check_type_str, evaluate_args, path, expected_result", [parameter_match_api, parameter_no_match_api]
+)
 def test_param_match(filename, check_type_str, evaluate_args, path, expected_result):
     """Validate parameter_match check type."""
     check = CheckType.init(check_type_str)

tests/test_validates.py

@@ -0,0 +1,132 @@
+"""Unit tests for validator CheckType method."""
+import pytest
+from netcompare.check_types import CheckType
+
+tolerance_wrong_argumet = (
+    "tolerance",
+    {"gt": 10},
+    "'tolerance' argument is mandatory for Tolerance Check Type.",
+)
+tolerance_wrong_value = (
+    "tolerance",
+    {"tolerance": "10"},
+    "Tolerance argument's value must be an integer. You have: <class 'str'>.",
+)
+parameter_no_params = (
+    "parameter_match",
+    {"mode": "match", "wrong_key": {"localAsn": "65130.1100", "linkType": "external"}},
+    "'params' argument is mandatory for ParameterMatch Check Type.",
+)
+parameter_wrong_type = (
+    "parameter_match",
+    {"mode": "match", "params": [{"localAsn": "65130.1100", "linkType": "external"}]},
+    "'params' argument must be a dict. You have: <class 'list'>.",
+)
+parameter_no_mode = (
+    "parameter_match",
+    {"mode-no-mode": "match", "params": {"localAsn": "65130.1100", "linkType": "external"}},
+    "'mode' argument is mandatory for ParameterMatch Check Type.",
+)
+parameter_mode_value = (
+    "parameter_match",
+    {"mode": ["match"], "params": {"localAsn": "65130.1100", "linkType": "external"}},
+    "'mode' argument should be one of the following: match, no-match. You have: ['match']",
+)
+regex_no_params = (
+    "regex",
+    {"regexregex": ".*UNDERLAY.*", "mode": "match"},
+    "'regex' argument is mandatory for Regex Check Type.",
+)
+regex_wrong_type = (
+    "regex",
+    {"regex": [".*UNDERLAY.*"], "mode-no-mode": "match"},
+    "'regex' argument must be a string. You have: <class 'list'>.",
+)
+regex_no_mode = (
+    "regex",
+    {"regex": ".*UNDERLAY.*", "mode-no-mode": "match"},
+    "'mode' argument is mandatory for Regex Check Type.",
+)
+regex_mode_value = (
+    "regex",
+    {"regex": ".*UNDERLAY.*", "mode": "match-no-match"},
+    "'mode' argument should be ['match', 'no-match']. You have: match-no-match",
+)
+operator_params = (
+    "operator",
+    {"my_params": {"mode": "not-in", "operator_data": [20, 40, 60]}},
+    "'params' argument must be provided. You have: my_params.",
+)
+operator_params_mode = (
+    "operator",
+    {"params": {"no-mode": "not-in", "not-operator_data": [20, 40, 60]}},
+    "'mode' and 'operator_data' arguments must be provided. You have: ['no-mode', 'not-operator_data'].",
+)
+operator_params_wrong_operator = (
+    "operator",
+    {"params": {"mode": "random", "operator_data": [20, 40, 60]}},
+    "'params' value must be one of the following: ['is-in', 'not-in', 'in-range', 'not-range', 'all-same', 'is-gt', 'is-lt', 'contains', 'not-contains']. You have: random",
+)
+operator_params_in = (
+    "operator",
+    {"params": {"mode": "in-range", "operator_data": "string"}},
+    "check options ('is-in', 'not-in', 'in-range', 'not-range') must have value of type list or tuple. i.e: dict(not-in=('Idle', 'Down'). You have: string of type <class 'str'>.",
+)
+operator_params_in_range = (
+    "operator",
+    {"params": {"mode": "in-range", "operator_data": (0, "1")}},
+    "'range' check-option in-range must have value of type list or tuple with items of type float or int. i.e: dict(not-range=(70000000, 80000000). You have: (0, '1').",
+)
+operator_params_in_range_lower_than = (
+    "operator",
+    {"params": {"mode": "in-range", "operator_data": (1, 0)}},
+    "'range' and 'not-range' must have value at index 0 lower than value at index 1. i.e: dict(not-range=(70000000, 80000000). You have: (1, 0).",
+)
+operator_params_number = (
+    "operator",
+    {"params": {"mode": "is-gt", "operator_data": "1"}},
+    "check options ('is-gt', 'is-lt') must have value of type float or int. You have: 1 of type <class 'str'>",
+)
+operator_params_contains = (
+    "operator",
+    {"params": {"mode": "contains", "operator_data": 1}},
+    "check options ('contains', 'not-contains') must have value of type string. You have: 1 of type <class 'int'>",
+)
+operator_params_bool = (
+    "operator",
+    {"params": {"mode": "all-same", "operator_data": 1}},
+    "check option all-same must have value of type bool. You have: 1 of type <class 'int'>",
+)
+
+all_tests = [
+    tolerance_wrong_argumet,
+    tolerance_wrong_value,
+    parameter_no_params,
+    parameter_wrong_type,
+    parameter_no_mode,
+    parameter_mode_value,
+    regex_no_params,
+    regex_wrong_type,
+    regex_no_mode,
+    regex_mode_value,
+    operator_params,
+    operator_params_mode,
+    operator_params_wrong_operator,
+    operator_params_in,
+    operator_params_in_range,
+    operator_params_in_range_lower_than,
+    operator_params_number,
+    operator_params_contains,
+    operator_params_bool,
+]
+
+
+@pytest.mark.parametrize("check_type_str, evaluate_args, expected_results", all_tests)
+def test_tolerance_key_name(check_type_str, evaluate_args, expected_results):
+    """Test CheckType validate method for each check-type."""
+    check = CheckType.init(check_type_str)
+
+    with pytest.raises(ValueError) as exc_info:
+        check.validate(**evaluate_args)
+
+    assert exc_info.type is ValueError and exc_info.value.args[0] == expected_results