Merge pull request #23 from heroku/fix_ci_workflow_yaml

Improved github CI

Author: hillarysanders

.github/workflows/test.yml

@@ -1,29 +1,30 @@
 name: MCP Tests
 
 on:
-  push:
-    branches:
   pull_request:
-
 jobs:
   ###########################################################################
   # 1 - Local integration tests (always run)
   ###########################################################################
   local:
     runs-on: ubuntu-latest
-
     # Dummy key lets the clients authenticate against the local servers.
     env:
       API_KEY: ci-test-key
 
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Python version from .python-version
+        id: python-version
+        run: |
+          PY_VERSION=$(cat .python-version)
+          echo "version=$PY_VERSION" >> $GITHUB_OUTPUT
+
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ steps.python-version.outputs.version }}
 
-      # Optional: cache wheels to speed up numpy / scipy installs
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
@@ -33,47 +34,106 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-          pip install -r requirements-dev.txt
 
       - name: Run pytest (local transports)
         run: pytest -q
 
-
   ###########################################################################
-  # 2 - Remote smoke-test (only when secrets are set)
-  #
-  #    • Put MCP_SERVER_URL  → “https://<app>.herokuapp.com”
-  #    • Put API_KEY         → same key you set as Heroku config var
-  #
-  #    The fixture auto-skips the remote case if these are missing, so
-  #    the job is conditionally *created* only when both secrets exist.
+  # 2 - Deploy this PR to a temp Heroku app and run tests against deployed app (in addition to 'local')
   ###########################################################################
   remote:
-    if: ${{ secrets.MCP_SERVER_URL != '' && secrets.API_KEY != '' }}
     runs-on: ubuntu-latest
-
     env:
-      MCP_SERVER_URL: ${{ secrets.MCP_SERVER_URL }}
-      API_KEY:       ${{ secrets.API_KEY }}
+      HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+      API_KEY: ci-test-key
+      # also note that github CI doesn't have access to your app's config vars, so here we're setting the remote
+      # server type to streamable HTTP. Folks using SSE would need to change this line for their e2e remote integration
+      # tests to test SSE instead of streamable HTTP.
+      REMOTE_SERVER_TRANSPORT_MODULE: streamable_http_server
+      # $APP_NAME is set below because we need to shorten the repo owner's name, as a precaution
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # <-- disables shallow clone, which heroku is upset by when running git push heroku later on
+
+      # Setting a short $APP_NAME that will be unique even if folks choose to fork this repo --> avoids clashes.
+      # Needs to be shortened if the github repo owner has a long name (max 30 char app name heroku limit).
+      - name: Generate short APP_NAME
+        id: appname
+        run: |
+          OWNER_SHORT=${GITHUB_REPOSITORY_OWNER:0:5}
+          REPO_NAME=$(basename "$GITHUB_REPOSITORY")
+          PR_NUMBER=$(jq .number "$GITHUB_EVENT_PATH")
+          APP_NAME="${OWNER_SHORT}-${REPO_NAME}-${PR_NUMBER}"
+          echo "APP_NAME=$APP_NAME"
+          echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV
+
+      - name: Read Python version from .python-version
+        id: python-version
+        run: |
+          PY_VERSION=$(cat .python-version)
+          echo "version=$PY_VERSION" >> $GITHUB_OUTPUT
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ steps.python-version.outputs.version }}
+
+      - name: Install Heroku CLI
+        run: |
+          curl https://cli-assets.heroku.com/install.sh | sh
+
+      - name: Log in to Heroku
+        run: |
+          echo "$HEROKU_API_KEY" | heroku auth:token
+
+      - name: Pre-cleanup (destroy app if it exists)
+        continue-on-error: true
+        run: |
+          heroku apps:destroy --app $APP_NAME --confirm $APP_NAME
+
+      # github CI can't use our app.json, so the config etc bits must be set manually.
+      # note WEB_CONCURRENCY is important! You get non-deterministic errors w/out it.
+      - name: Create temp Heroku app for this PR
+        run: |
+          heroku create $APP_NAME
+          heroku buildpacks:set heroku/python -a $APP_NAME
+          heroku config:set API_KEY=$API_KEY --app $APP_NAME
+          heroku config:set STDIO_MODE_ONLY=false
+          heroku config:set REMOTE_SERVER_TRANSPORT_MODULE=$REMOTE_SERVER_TRANSPORT_MODULE --app $APP_NAME
+          heroku config:set WEB_CONCURRENCY=1 --app $APP_NAME
+
+      - name: Deploy this branch to Heroku
+        run: |
+          git push https://heroku:[email protected]/$APP_NAME.git HEAD:refs/heads/main --force
 
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
 
-      - name: Install dependencies
+      - name: Install test dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      # We reuse the *same* test-suite; the fixture detects MCP_SERVER_URL
-      # and adds the “remote” parameter automatically.
-      - name: Run pytest (remote smoke)
-        run: pytest -q
+      - name: Get Heroku env vars
+        id: heroku_env
+        run: |
+          url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n')
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
+      - name: Run pytest against deployed app
+        env:
+            MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }}
+        run: |
+          echo "APP_NAME = $APP_NAME"
+          echo "MCP_SERVER_URL = $MCP_SERVER_URL"
+          echo "REMOTE_SERVER_TRANSPORT_MODULE = $REMOTE_SERVER_TRANSPORT_MODULE"
+          echo "API_KEY is ${API_KEY:+set}"  # won't print the key, just confirms it's non-empty
+          pytest -q
+
+      - name: Destroy Heroku app after test
+        if: always()
+        run: |
+          heroku apps:destroy --app $APP_NAME --confirm $APP_NAME

README.md

@@ -87,6 +87,7 @@ uvicorn src.streamable_http_server:app --reload
 *Running with `--reload` is optional, but great for local development*
 
 Next, in a new pane, you can try running some queries against your server:
+
 #### Local Streamable HTTP, SSE - Example Requests
 First run:
 ```bash

tests/README.md

@@ -31,12 +31,20 @@ pip install -r requirements.txt
 
 ## 2 · Run local transports only
 ```bash
+git push heroku <your-branch>:main
+```
+
+## 2 · Run local & one-off-dyno (STDIO) deployed transports
+```bash
 pytest tests -q
 ```
 
-## 3 - Run local & remote transports
+## 3 - Run local & all deployed transports
 ```bash
+REMOTE_SERVER_TYPE=$(heroku config:get REMOTE_SERVER_TYPE) \
 MCP_SERVER_URL=$(heroku info -s -a "$APP_NAME" | grep web_url | cut -d= -f2 | tr -d '\n') \
 API_KEY=$(heroku config:get API_KEY -a "$APP_NAME") \
 pytest tests -q
-```
+```
+
+*NOTE: if your `REMOTE_SERVER_TYPE` is set to `sse_server` and not the default `streamable_http_server`, you'll need to change the `REMOTE_SERVER_TRANSPORT_MODULE` declaration line in `.github/workflows/test.yml` to make sure that the end to end integration tests against the temporary deployed remote server are using the appropriate client code.*

tests/conftest.py

@@ -73,11 +73,14 @@ async def _ctx_stdio_local() -> AsyncGenerator[Dict, None]:
 
 # ---------------------------------------------------------------- remote HTTP / SSE
 async def _ctx_remote() -> AsyncGenerator[Dict, None]:
-    url = os.getenv("MCP_SERVER_URL"); key = os.getenv("API_KEY")
+    url = os.getenv("MCP_SERVER_URL")
+    key = os.getenv("API_KEY")
+    server_type = os.getenv("REMOTE_SERVER_TRANSPORT_MODULE")
+
     if not url or not key:
         pytest.skip("remote env-vars missing")
-    yield {"client": "streamable_http_client",
-           "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url.rstrip("/")}}
+    yield {"client": server_type.replace("server", "client"),
+           "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url, "REMOTE_SERVER_TRANSPORT_MODULE": server_type}}
 
 # ---------------------------------------------------------------- remote STDIO ctx
 async def _ctx_remote_stdio() -> AsyncGenerator[Dict, None]:

tests/test_mcp_e2e.py

@@ -54,7 +54,6 @@ async def test_list_tools(ctx):
 
 
 async def test_code_exec(ctx):
-    payload = json.dumps({"name": "code_exec_python",
-                          "arguments": {"code": "print(2+2)"}})
+    payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}})
     data = await _safe_call(ctx, "call_tool", "--args", payload)
     assert _extract_stdout(data) == "4"