Run compatibility tests from the new 'compatTest' distribution

Author: sethmlarson

.ci/run-repository.sh

@@ -32,7 +32,6 @@ docker build \
 echo -e "\033[1m>>>>> Run [elastic/elasticsearch-py container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
 if [[ "$STACK_VERSION" == "8.0.0-SNAPSHOT" ]]; then
-  export STACK_VERSION="7.x-SNAPSHOT"
   export ELASTIC_CLIENT_APIVERSIONING="true"
 fi
 

test_elasticsearch/test_async/test_server/test_rest_api_spec.py

@@ -21,6 +21,7 @@
 clients.
 """
 import inspect
+import re
 import warnings
 
 import pytest
@@ -118,6 +119,11 @@ async def run_do(self, action):
         catch = action.pop("catch", None)
         warn = action.pop("warnings", ())
         allowed_warnings = action.pop("allowed_warnings", ())
+        if isinstance(allowed_warnings, str):
+            allowed_warnings = (allowed_warnings,)
+        allowed_warnings_regex = action.pop("allowed_warnings_regex", ())
+        if isinstance(allowed_warnings_regex, str):
+            allowed_warnings_regex = (allowed_warnings_regex,)
         assert len(action) == 1
 
         # Remove the x_pack_rest_user authentication
@@ -138,6 +144,9 @@ async def run_do(self, action):
 
         # locate api endpoint
         for m in method.split("."):
+            # Some deprecated APIs are prefixed with 'xpack-*'
+            if m.startswith("xpack-"):
+                m = m.replace("xpack-", "")
             assert hasattr(api, m)
             api = getattr(api, m)
 
@@ -186,7 +195,14 @@ async def run_do(self, action):
             str(w.message)
             for w in caught_warnings
             if w.category == ElasticsearchWarning
-            and str(w.message) not in allowed_warnings
+            and (not allowed_warnings or str(w.message) not in allowed_warnings)
+            and (
+                not allowed_warnings_regex
+                or all(
+                    re.search(pattern, str(w.message)) is None
+                    for pattern in allowed_warnings_regex
+                )
+            )
         ]
 
         # This warning can show up in many places but isn't accounted for

test_elasticsearch/test_server/test_rest_api_spec.py

@@ -59,6 +59,7 @@
     "default_shards",
     "warnings",
     "allowed_warnings",
+    "allowed_warnings_regex",
     "contains",
     "arbitrary_key",
     "transform_and_set",
@@ -223,6 +224,11 @@ def run_do(self, action):
         catch = action.pop("catch", None)
         warn = action.pop("warnings", ())
         allowed_warnings = action.pop("allowed_warnings", ())
+        if isinstance(allowed_warnings, str):
+            allowed_warnings = (allowed_warnings,)
+        allowed_warnings_regex = action.pop("allowed_warnings_regex", ())
+        if isinstance(allowed_warnings_regex, str):
+            allowed_warnings_regex = (allowed_warnings_regex,)
         assert len(action) == 1
 
         # Remove the x_pack_rest_user authentication
@@ -243,6 +249,9 @@ def run_do(self, action):
 
         # locate api endpoint
         for m in method.split("."):
+            # Some deprecated APIs are prefixed with 'xpack-*'
+            if m.startswith("xpack-"):
+                m = m.replace("xpack-", "")
             assert hasattr(api, m)
             api = getattr(api, m)
 
@@ -291,7 +300,14 @@ def run_do(self, action):
             str(w.message)
             for w in caught_warnings
             if w.category == ElasticsearchWarning
-            and str(w.message) not in allowed_warnings
+            and (not allowed_warnings or str(w.message) not in allowed_warnings)
+            and (
+                not allowed_warnings_regex
+                or all(
+                    re.search(pattern, str(w.message)) is None
+                    for pattern in allowed_warnings_regex
+                )
+            )
         ]
 
         # This warning can show up in many places but isn't accounted for
@@ -579,7 +595,10 @@ def sync_runner(sync_client):
     # Download the zip and start reading YAML from the files in memory
     package_zip = zipfile.ZipFile(io.BytesIO(http.request("GET", package_url).data))
     for yaml_file in package_zip.namelist():
-        if not re.match(r"^rest-api-spec/test/.*\.ya?ml$", yaml_file):
+        yaml_filter_pattern = r"^rest-api-spec/%s/.*\.ya?ml$" % (
+            "compatTest" if COMPATIBILITY_MODE_ENABLED else "test"
+        )
+        if not re.match(yaml_filter_pattern, yaml_file):
             continue
         yaml_tests = list(yaml.safe_load_all(package_zip.read(yaml_file)))
 
@@ -608,7 +627,13 @@ def sync_runner(sync_client):
             # is to remove most of the file path prefixes and
             # the .yml suffix.
             pytest_test_name = yaml_file.rpartition(".")[0].replace(".", "/")
-            for prefix in ("rest-api-spec/", "test/", "free/", "platinum/"):
+            for prefix in (
+                "rest-api-spec/",
+                "compatTest/",
+                "test/",
+                "free/",
+                "platinum/",
+            ):
                 if pytest_test_name.startswith(prefix):
                     pytest_test_name = pytest_test_name[len(prefix) :]
             pytest_param_id = "%s[%d]" % (pytest_test_name, test_number)