ci: bifurcate nemo2.0 and nemo-rl build profiles, use ci-success aggregate check in build (#91)

Sets up different dependency profiles in `pyproject.toml` for NeMo/RL
and NeMo 2.0, since they require different python versions and have
heavy dependencies that can easily conflict.

The GH actions build will run them in parallel. For now, the `nemo-rl`
extra is just an alias for the nemo one, until it is added. Now,
`dev-nemo` and `dev-nemo-rl` are the target build extras, but `dev` is
retained as an alias for `dev-nemo` for backwards compatibility.

Additionally, a `ci-success` build job has been added, which depends on
all other required jobs, and is now the sole "required" check from the
build-and-test workflow.

Author: g-husam

.github/workflows/build-and-test.yml

@@ -25,8 +25,13 @@ jobs:
     # Use larger machine with more disk space
     runs-on: ubuntu-24.04-32core
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.10", "3.12"]
+        include:
+          - python-version: "3.10"
+            profile: "dev-nemo"
+          - python-version: "3.12"
+            profile: "dev-nemo-rl"
     env:
       PYTHON_FAIL_UNDER: 90
       CPP_FAIL_UNDER: 80
@@ -51,7 +56,7 @@ jobs:
           df -h
           # Install python dependencies (with coverage enabled)
           echo -e "\n##### Running pip install #####"
-          pip install -e '.[dev]' --config-settings=cmake.args="-DENABLE_COVERAGE=ON"
+          pip install -e '.[${{ matrix.profile }}]' --config-settings=cmake.args="-DENABLE_COVERAGE=ON"
 
       - run: df -h
 
@@ -214,3 +219,20 @@ jobs:
         shell: 'bash'
         run: |
           go run github.com/google/addlicense@v1.1.1 -check -s=only .
+
+  # Add a gatekeeper job to act as the (main) required check, which in turn depends on all other required jobs.
+  # This makes the required check management in GitHub settings tractable, as we don't have to tweak the settings everytime
+  # a job name changes or a matrix option is modified.
+  ci-success:
+    runs-on: ubuntu-latest
+    if: always()
+    # Specify all top-level jobs that are required to pass.
+    needs: [build, lint-code, lint-license]
+    steps:
+      - name: Check overall status
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more dependent jobs failed or were cancelled."
+            exit 1
+          fi
+          echo "All required jobs passed!"

pyproject.toml

@@ -79,13 +79,10 @@ docs = [
     #"zensical==0.0.11",
 ]
 
-# Defines a "dev" extra for setting up a development environment.
-# Installed via: `pip install -e .[dev]`
-dev = [
+# Defines a base set of development dependencies shared across all profiles.
+dev-base = [
     # A more advanced assertion framework.
     "assertpy==1.1",
-    # Adds the optional pytorch dependency set to the development environment.
-    "ml-flashpoint[pytorch,megatron,nemo,docs]",
     # Our testing framework.
     "pytest==8.4.1",
     # A plugin for pytest that generates code coverage reports for Python.
@@ -100,6 +97,27 @@ dev = [
     "ruff==0.12.11",
 ]
 
+# Defines a "dev-nemo" extra for NeMo 2.0 development (typically uses Python 3.10+).
+# Installed via: `pip install -e .[dev-nemo]`
+dev-nemo = [
+    "ml-flashpoint[dev-base,pytorch,megatron,nemo,docs]",
+]
+
+# Defines a "dev-nemo-rl" extra for NeMo RL development (typically uses Python 3.12+).
+# Installed via: `pip install -e .[dev-nemo-rl]`
+dev-nemo-rl = [
+    "ml-flashpoint[dev-nemo]",
+    # TODO: uncomment below and remove line above when nemo-rl profile is added
+    #"ml-flashpoint[dev-base,nemo-rl]",
+]
+
+# Defines a "dev" extra for setting up a development environment.
+# Aliased to dev-nemo to retain existing behavior.
+# Installed via: `pip install -e .[dev]`
+dev = [
+    "ml-flashpoint[dev-nemo]",
+]
+
 
 # ===================================================================
 # PEP 518: Build System Definition