From 7a03b0e18fc4252679fd1f3e1c3730584a631f5e Mon Sep 17 00:00:00 2001 From: Jib Date: Wed, 11 Dec 2024 09:58:30 -0500 Subject: [PATCH 1/7] first stack --- first_stack.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 first_stack.txt diff --git a/first_stack.txt b/first_stack.txt new file mode 100644 index 0000000..e69de29 From 28e4f7d1db237c2f9b5a8c518fe954a515355448 Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 1 Jul 2025 10:54:56 -0400 Subject: [PATCH 2/7] add language tags to each buildvariant --- .evergreen/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c13402a..eff31cb 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -264,6 +264,7 @@ tasks: buildvariants: - name: test-semantic-kernel-python-rhel display_name: Semantic-Kernel RHEL Python + tags: [python] expansions: DIR: semantic-kernel-python run_on: @@ -276,6 +277,7 @@ buildvariants: - name: test-semantic-kernel-csharp-rhel display_name: Semantic-Kernel RHEL CSharp + tags: [csharp] expansions: DIR: semantic-kernel-csharp run_on: @@ -286,6 +288,7 @@ buildvariants: - name: test-langchain-python-rhel display_name: Langchain RHEL Python + tags: [python] expansions: DIR: langchain-python run_on: @@ -297,6 +300,7 @@ buildvariants: - name: test-langgraph-python-rhel display_name: Langgraph RHEL Python + tags: [python] expansions: DIR: langgraph-python run_on: @@ -308,6 +312,7 @@ buildvariants: - name: test-chatgpt-retrieval-plugin-rhel display_name: ChatGPT Retrieval Plugin + tags: [python] expansions: DIR: chatgpt-retrieval-plugin run_on: @@ -319,6 +324,7 @@ buildvariants: - name: test-llama-index-vectorstore-rhel display_name: LlamaIndex RHEL Vector Store + tags: [python] expansions: DIR: llama-index-python-vectorstore run_on: @@ -331,6 +337,7 @@ buildvariants: - name: test-docarray-rhel display_name: DocArray RHEL + tags: [python] expansions: DIR: docarray run_on: @@ -341,6 +348,7 @@ buildvariants: - name: test-pymongo-voyageai-rhel display_name: PyMongo-VoyageAI RHEL + tags: [python] expansions: DIR: pymongo-voyageai run_on: @@ -351,6 +359,7 @@ buildvariants: - name: test-haystack-embeddings-rhel display_name: Haystack Embeddings RHEL + tags: [python] expansions: DIR: haystack-embeddings run_on: @@ -362,6 +371,7 @@ buildvariants: - name: test-haystack-fulltext-rhel display_name: Haystack FullText RHEL + tags: [python] expansions: DIR: haystack-fulltext run_on: @@ -373,6 +383,7 @@ buildvariants: - name: test-langchaingo-ubuntu display_name: LangchainGo Ubuntu2204 + tags: [golang] expansions: DIR: langchaingo-golang run_on: @@ -382,6 +393,7 @@ buildvariants: - name: test-langchain-js-ubuntu display_name: LangchainJS Ubuntu2204 + tags: [javascript] expansions: DIR: langchain-js run_on: From fabe044ba37b38ce46934d8d1cde45465c99cffe Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 1 Jul 2025 11:05:54 -0400 Subject: [PATCH 3/7] add new policy for handling failing tests across different repos --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7962edd..778aab4 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ Test execution flow is defined in `.evergreen/config.yml`. The test pipeline's c - `run_on` -- Specified platform to run on. `rhel87-small` or `ubuntu2204-small` should be used by default. Any other distro may fail Atlas CLI setup. - `tasks` -- Tasks to run. See below for more details - `cron` -- The tests are run via a cron job on a nightly cadence. This can be modified by setting a different cadence. Cron jobs can be scheduled using [cron syntax](https://crontab.guru/#0_0_*_*_*) +- `tags` -- This should include the language where the AI/ML is run. i.e. `[python, csharp, golang, javascript]` Any tagged language will populate the +appropriate language-specific slack channel. **[Tasks](https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#tasks)** -- These are the "building blocks" of our runs. Here is where we consolidate the specific set of functions. The basic parameters to add are shown below @@ -185,8 +187,10 @@ evergreen patch -p ai-ml-pipeline-testing --param REPO_ORG=caseyclements --param ### Handling Failing Tests +Tests are run periodically (nightly) and any failures will propagate into both the `dbx-ai-ml-testing-pipline-notifications` and `dbx-ai-ml-testing-pipeline-notifications-{language}` channel. Repo owners of this `ai-ml-testing-pipeline` library are required to join the `dbx-ai-ml-testing-pipeline-notifications`. Pipeline specific implementers must **at least** join `dbx-ai-ml-testing-pipline-notifications-{language}` (e.g. whomever implemented `langchain-js` must at least be a member of `dbx-ai-ml-testing-pipeline-notifications-js`). + If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket, and disable the relevant tests in the `config.yml` file, with a comment about the JIRA ticket that will address it. -This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` that would mask other -failures. +This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` or `dbx-ai-ml-testing-pipeline-notifications-{language}` that would mask other +failures. From 088abf77cb8ba8e2495e19223abdfe843eac52d6 Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 1 Jul 2025 11:08:49 -0400 Subject: [PATCH 4/7] fix lint --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 778aab4..4df442a 100644 --- a/README.md +++ b/README.md @@ -193,4 +193,4 @@ If tests are found to be failing, and cannot be addressed quickly, the responsib in the `config.yml` file, with a comment about the JIRA ticket that will address it. This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` or `dbx-ai-ml-testing-pipeline-notifications-{language}` that would mask other -failures. +failures. From 3d467f318d62b3c0d7acb3c2b5aa11c64424115f Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 1 Jul 2025 14:20:32 -0400 Subject: [PATCH 5/7] Delete first_stack.txt --- first_stack.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 first_stack.txt diff --git a/first_stack.txt b/first_stack.txt deleted file mode 100644 index e69de29..0000000 From bc1f49eaea49f98614f6355d2eb88e32060baddf Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 15 Jul 2025 14:07:40 -0400 Subject: [PATCH 6/7] install pyyaml in workflow, update configs to include language, add check in lint_config.py --- .evergreen/config.yml | 17 +++++++++-------- .evergreen/lint_config.py | 28 +++++++++++++++++++++------- .github/workflows/lint.yml | 2 +- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 59e8522..dfcb252 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -325,7 +325,7 @@ buildvariants: batchtime: 10080 # 1 week # TODO: INTPYTHON-668 - # - name: test-chatgpt-retrieval-plugin-rhel + # - name: test-chatgpt-retrieval-plugin-python-rhel # display_name: ChatGPT Retrieval Plugin # tags: [python] # expansions: @@ -338,7 +338,7 @@ buildvariants: # batchtime: 10080 # 1 week # TODO: INTPYTHON-669 - # - name: test-llama-index-vectorstore-rhel + # - name: test-llama-index-vectorstore-python-rhel # display_name: LlamaIndex RHEL Vector Store # tags: [python] # expansions: @@ -351,7 +351,7 @@ buildvariants: # # - name: test-llama-index-remote # # batchtime: 10080 # 1 week - - name: test-docarray-rhel + - name: test-docarray-python-rhel display_name: DocArray RHEL tags: [python] expansions: @@ -362,7 +362,7 @@ buildvariants: - name: test-docarray-local - name: test-docarray-remote - - name: test-pymongo-voyageai-rhel + - name: test-pymongo-voyageai-python-rhel display_name: PyMongo-VoyageAI RHEL tags: [python] expansions: @@ -373,8 +373,9 @@ buildvariants: - name: test-pymongo-voyageai-local - name: test-pymongo-voyageai-remote - - name: test-crewai-tools-rhel + - name: test-crewai-tools-python-rhel display_name: CrewAI-Tools Ubuntu + tags: [python] expansions: DIR: crewai-tools run_on: @@ -383,7 +384,7 @@ buildvariants: - name: test-crewai-tools-local - name: test-crewai-tools-remote - - name: test-haystack-embeddings-rhel + - name: test-haystack-embeddings-python-rhel display_name: Haystack Embeddings RHEL tags: [python] expansions: @@ -395,7 +396,7 @@ buildvariants: # TODO: INTPYTHON-465 # - name: test-haystack-embeddings-remote - - name: test-haystack-fulltext-rhel + - name: test-haystack-fulltext-python-rhel display_name: Haystack FullText RHEL tags: [python] expansions: @@ -407,7 +408,7 @@ buildvariants: # TODO: INTPYTHON-465 # - name: test-haystack-fulltext-remote - - name: test-langchaingo-ubuntu + - name: test-langchaingo-golang-ubuntu display_name: LangchainGo Ubuntu2204 tags: [golang] expansions: diff --git a/.evergreen/lint_config.py b/.evergreen/lint_config.py index b2e0d34..2d57617 100644 --- a/.evergreen/lint_config.py +++ b/.evergreen/lint_config.py @@ -25,9 +25,10 @@ def load_yaml_file(file_path: str) -> Dict[Any, Any]: return yaml.safe_load(file) or {} -def check_buildvariant_tags(data: Dict[Any, Any]) -> List[str]: +def check_buildvariants(data: Dict[Any, Any]) -> List[str]: """ - Check if buildvariant tasks contain at least one required language tag. + Check if buildvariant tasks contain at least one required language tag + as well as the language within the buildvariant name. Example Buildvariant structure in YAML: buildvariants: @@ -57,21 +58,34 @@ def check_buildvariant_tags(data: Dict[Any, Any]) -> List[str]: for i, buildvariant in enumerate(buildvariants): if not isinstance(buildvariant, dict): - errors.append(f"buildvariants[{i}] should be a dictionary") + errors.append(f"buildvariants[{i}] should contain sub-fields") continue - buildvariant_name = buildvariant.get("display_name", f"buildvariant_{i}") + buildvariant_name = buildvariant.get("name", "") + if not buildvariant_name: + errors.append(f"buildvariants[{i}] is missing 'name'") + continue + else: + if all([f"-{lang}-" not in buildvariant_name for lang in VALID_LANGUAGES]): + errors.append( + f"buildvariant '{buildvariant_name}' should contain one" + f" '-[{', '.join(VALID_LANGUAGES)}]-' in its name" + f"got: {buildvariant_name}", + ) + + buildvariant_display_name = buildvariant.get("display_name", buildvariant_name) + tags = buildvariant.get("tags", []) if not isinstance(tags, list) or len(tags) != 1: errors.append( - f"'tags' in buildvariant '{buildvariant_name}' should be a list of size 1" + f"'tags' in buildvariant '{buildvariant_display_name}' should be a list of size 1" ) continue if tags[0] not in VALID_LANGUAGES: errors.append( - f"buildvariant '{buildvariant_name}' has invalid tag '{tags[0]}'. " + f"buildvariant '{buildvariant_display_name}' has invalid tag '{tags[0]}'. " f"Valid tags are: {', '.join(VALID_LANGUAGES)}" ) return errors @@ -85,7 +99,7 @@ def main(): if not data: raise FileNotFoundError(f"Failed to load or parse {CONFIG_YML}") - errors = check_buildvariant_tags(data) + errors = check_buildvariants(data) if errors: logger.error("❌ Errors found in %s:", CONFIG_YML) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4d4c4f1..8f145fb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: python-version: '3.10' - name: Install Python dependencies run: | - python -m pip install -U pip pre-commit + python -m pip install -U pip pre-commit pyyaml - name: Run linters run: | pre-commit run --hook-stage=manual --all-files From 3e16bb73aed01a05cb51c59355ab6d03cb1f62cc Mon Sep 17 00:00:00 2001 From: Jib Date: Tue, 22 Jul 2025 18:55:22 -0400 Subject: [PATCH 7/7] add python tag to mem0 --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index afa5cb8..1778b0c 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -446,6 +446,7 @@ buildvariants: - name: test-mem0-python-rhel display_name: mem0 RHEL Python + tags: [python] expansions: DIR: mem0-python run_on: