From 8108558dd20bf2704019db6b175975d357d4dcfc Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 15:52:17 -0700 Subject: [PATCH 01/61] Fixed some github CI yaml errors --- .github/workflows/test.yml | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d901878..9bc1cdb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,6 @@ name: MCP Tests on: - push: - branches: pull_request: jobs: @@ -11,8 +9,6 @@ jobs: ########################################################################### local: runs-on: ubuntu-latest - - # Dummy key lets the clients authenticate against the local servers. env: API_KEY: ci-test-key @@ -23,7 +19,6 @@ jobs: with: python-version: "3.11" - # Optional: cache wheels to speed up numpy / scipy installs - uses: actions/cache@v4 with: path: ~/.cache/pip @@ -33,47 +28,45 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install -r requirements-dev.txt - name: Run pytest (local transports) run: pytest -q - ########################################################################### - # 2 - Remote smoke-test (only when secrets are set) - # - # • Put MCP_SERVER_URL → “https://.herokuapp.com” - # • Put API_KEY → same key you set as Heroku config var - # - # The fixture auto-skips the remote case if these are missing, so - # the job is conditionally *created* only when both secrets exist. + # 2 - Remote smoke-test (skipped if ENV vars are missing) ########################################################################### remote: - if: ${{ secrets.MCP_SERVER_URL != '' && secrets.API_KEY != '' }} runs-on: ubuntu-latest - env: MCP_SERVER_URL: ${{ secrets.MCP_SERVER_URL }} - API_KEY: ${{ secrets.API_KEY }} + API_KEY: ${{ secrets.API_KEY }} + SHOULD_RUN_REMOTE: ${{ secrets.MCP_SERVER_URL != '' && secrets.API_KEY != '' }} steps: + - name: Skip if secrets are missing + if: env.SHOULD_RUN_REMOTE != 'true' + run: echo "Skipping remote tests due to missing secrets." + - uses: actions/checkout@v4 + if: env.SHOULD_RUN_REMOTE == 'true' - uses: actions/setup-python@v5 + if: env.SHOULD_RUN_REMOTE == 'true' with: python-version: "3.11" - uses: actions/cache@v4 + if: env.SHOULD_RUN_REMOTE == 'true' with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} - name: Install dependencies + if: env.SHOULD_RUN_REMOTE == 'true' run: | python -m pip install --upgrade pip pip install -r requirements.txt - # We reuse the *same* test-suite; the fixture detects MCP_SERVER_URL - # and adds the “remote” parameter automatically. - name: Run pytest (remote smoke) + if: env.SHOULD_RUN_REMOTE == 'true' run: pytest -q From f69491dd73b3941fe2cbf1ea4515d06aa814367d Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 15:53:12 -0700 Subject: [PATCH 02/61] comments back in --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bc1cdb..4c59f4f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,11 @@ jobs: ########################################################################### # 2 - Remote smoke-test (skipped if ENV vars are missing) + # + # • Put MCP_SERVER_URL → “https://.herokuapp.com” + # • Put API_KEY → same key you set as Heroku config var + # + # The fixture auto-skips the remote case if these are missing, so ########################################################################### remote: runs-on: ubuntu-latest From b67c472949aff6b3432ac9fd74b73e3088991bcb Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 15:56:41 -0700 Subject: [PATCH 03/61] comment updates --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c59f4f..92550f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,10 +9,12 @@ jobs: ########################################################################### local: runs-on: ubuntu-latest + # Dummy key lets the clients authenticate against the local servers. env: API_KEY: ci-test-key steps: + # Optional: cache wheels to speed up numpy / scipy installs - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -72,6 +74,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + # We reuse the *same* test-suite; the fixture detects MCP_SERVER_URL - name: Run pytest (remote smoke) if: env.SHOULD_RUN_REMOTE == 'true' run: pytest -q From de10b68bcc2a0735d840ea387647166cf6d8f7ac Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:17:21 -0700 Subject: [PATCH 04/61] Testing auto-deploy-test-app --- .github/workflows/test.yml | 52 ++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92550f9..744d319 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,6 @@ name: MCP Tests on: pull_request: - jobs: ########################################################################### # 1 - Local integration tests (always run) @@ -35,46 +34,55 @@ jobs: run: pytest -q ########################################################################### - # 2 - Remote smoke-test (skipped if ENV vars are missing) - # - # • Put MCP_SERVER_URL → “https://.herokuapp.com” - # • Put API_KEY → same key you set as Heroku config var - # - # The fixture auto-skips the remote case if these are missing, so + # 2 - Deploy this PR to a temp Heroku app and run remote tests ########################################################################### remote: runs-on: ubuntu-latest env: - MCP_SERVER_URL: ${{ secrets.MCP_SERVER_URL }} - API_KEY: ${{ secrets.API_KEY }} - SHOULD_RUN_REMOTE: ${{ secrets.MCP_SERVER_URL != '' && secrets.API_KEY != '' }} + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + API_KEY: ci-test-key + APP_NAME: mcp-toolkit-pr-${{ github.event.number }} steps: - - name: Skip if secrets are missing - if: env.SHOULD_RUN_REMOTE != 'true' - run: echo "Skipping remote tests due to missing secrets." - - uses: actions/checkout@v4 - if: env.SHOULD_RUN_REMOTE == 'true' + + - name: Install Heroku CLI + run: | + curl https://cli-assets.heroku.com/install.sh | sh + + - name: Log in to Heroku + run: | + echo "$HEROKU_API_KEY" | heroku auth:token + + - name: Create temp Heroku app for this PR + run: | + heroku create $APP_NAME + heroku config:set API_KEY=$API_KEY --app $APP_NAME + + - name: Deploy this branch to Heroku + run: | + git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$APP_NAME.git HEAD:main --force - uses: actions/setup-python@v5 - if: env.SHOULD_RUN_REMOTE == 'true' with: python-version: "3.11" - uses: actions/cache@v4 - if: env.SHOULD_RUN_REMOTE == 'true' with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} - - name: Install dependencies - if: env.SHOULD_RUN_REMOTE == 'true' + - name: Install test dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - # We reuse the *same* test-suite; the fixture detects MCP_SERVER_URL - - name: Run pytest (remote smoke) - if: env.SHOULD_RUN_REMOTE == 'true' + - name: Run pytest against deployed app + env: + MCP_SERVER_URL: https://${{ env.APP_NAME }}.herokuapp.com run: pytest -q + + - name: Destroy Heroku app after test + if: always() + run: | + heroku apps:destroy --app $APP_NAME --confirm $APP_NAME \ No newline at end of file From 9af210624f77149ec4a0cecd72b47ddce1c979de Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:21:38 -0700 Subject: [PATCH 05/61] python version --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 744d319..88fd00f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,7 +65,8 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.11" + # should align with .python-version + python-version: "3.12" - uses: actions/cache@v4 with: From 66f0c961c75f3cccf12071e890c5b068ec520e7a Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:25:01 -0700 Subject: [PATCH 06/61] refs/heads/main fix? --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88fd00f..abf68e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: - name: Deploy this branch to Heroku run: | - git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$APP_NAME.git HEAD:main --force + git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$APP_NAME.git HEAD:refs/heads/main --force - uses: actions/setup-python@v5 with: From 70a21e56afe391fd8b6f8b7282165e8fa3fd56fc Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:28:54 -0700 Subject: [PATCH 07/61] shallow clone fix --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abf68e9..a2124f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by? - name: Install Heroku CLI run: | From 71bc4500243f0b5d6df00ef546b25be8a107a4f9 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:40:53 -0700 Subject: [PATCH 08/61] Trigger CI From 9c9bf99f5bbe2a0af8ac1d077354dbaf45c2bf2f Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:47:48 -0700 Subject: [PATCH 09/61] set API key --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2124f6..12a75d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,7 @@ jobs: runs-on: ubuntu-latest env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + # TODO: make this into a github secret, just in case API_KEY: ci-test-key APP_NAME: mcp-toolkit-pr-${{ github.event.number }} @@ -83,6 +84,7 @@ jobs: - name: Run pytest against deployed app env: MCP_SERVER_URL: https://${{ env.APP_NAME }}.herokuapp.com + API_KEY: ${{ env.API_KEY }} run: pytest -q - name: Destroy Heroku app after test From 2df8b7c4af8e02270f9755a9f3281f5fda35fafb Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 16:58:35 -0700 Subject: [PATCH 10/61] Correct MCP_SERVER_URL --- .github/workflows/test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12a75d9..c0d211e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,9 +81,15 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Get Heroku app URL + id: heroku_url + run: | + url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') + echo "url=$url" >> "$GITHUB_OUTPUT" + - name: Run pytest against deployed app env: - MCP_SERVER_URL: https://${{ env.APP_NAME }}.herokuapp.com + MCP_SERVER_URL: ${{ steps.heroku_url.outputs.url }} API_KEY: ${{ env.API_KEY }} run: pytest -q From 237b5063c340fd3c2ec9a25bd2b33b80da2836fd Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:00:45 -0700 Subject: [PATCH 11/61] Trigger CI From a55ca1d8a974647c3455372963a8cbe20ba7b106 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:03:12 -0700 Subject: [PATCH 12/61] Trigger CI From 0adf81cc83894b7aaa269236823c9b981be96d5e Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:11:34 -0700 Subject: [PATCH 13/61] safe call maybe fix --- tests/test_mcp_e2e.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_mcp_e2e.py b/tests/test_mcp_e2e.py index 4238e67..57a9671 100644 --- a/tests/test_mcp_e2e.py +++ b/tests/test_mcp_e2e.py @@ -54,7 +54,7 @@ async def test_list_tools(ctx): async def test_code_exec(ctx): - payload = json.dumps({"name": "code_exec_python", - "arguments": {"code": "print(2+2)"}}) - data = await _safe_call(ctx, "call_tool", "--args", payload) + # payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) + # data = await _safe_call(ctx, "call_tool", "--args", payload) + data = await _safe_call(ctx, "call_tool", "--args", '{"code": "print(2+2)"}', "--name", "code_exec_python") assert _extract_stdout(data) == "4" From ed6fd9523d6e58ab20a04dd1310f95b6b3ea23c3 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:30:21 -0700 Subject: [PATCH 14/61] undo --- tests/test_mcp_e2e.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_mcp_e2e.py b/tests/test_mcp_e2e.py index 57a9671..4ba9539 100644 --- a/tests/test_mcp_e2e.py +++ b/tests/test_mcp_e2e.py @@ -54,7 +54,6 @@ async def test_list_tools(ctx): async def test_code_exec(ctx): - # payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) - # data = await _safe_call(ctx, "call_tool", "--args", payload) - data = await _safe_call(ctx, "call_tool", "--args", '{"code": "print(2+2)"}', "--name", "code_exec_python") + payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) + data = await _safe_call(ctx, "call_tool", "--args", payload) assert _extract_stdout(data) == "4" From eb0d22181cda6c72fa56800dfb0e86e6fb53e07c Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:31:21 -0700 Subject: [PATCH 15/61] test --- tests/test_mcp_e2e.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_mcp_e2e.py b/tests/test_mcp_e2e.py index 4ba9539..d7a86b4 100644 --- a/tests/test_mcp_e2e.py +++ b/tests/test_mcp_e2e.py @@ -54,6 +54,7 @@ async def test_list_tools(ctx): async def test_code_exec(ctx): - payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) - data = await _safe_call(ctx, "call_tool", "--args", payload) + # payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) + data = await _safe_call(ctx, "call_tool", "--args", '{"name":"code_exec_python","arguments":{"code":"print(2+2)"}}') + # data = await _safe_call(ctx, "call_tool", "--args", payload) assert _extract_stdout(data) == "4" From a332d068fbf3d502a732e560c337518d58112660 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:32:53 -0700 Subject: [PATCH 16/61] Trigger CI From 2558d5ef31a20176cf04023b5205241616b0965b Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:37:34 -0700 Subject: [PATCH 17/61] Trigger CI From da91c0501c090100185cfed2485045e670863347 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:40:20 -0700 Subject: [PATCH 18/61] Trigger CI From 6ff4f8a53bce472eb61cc68327725437c654ecd0 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 17:42:08 -0700 Subject: [PATCH 19/61] Trigger CI From 191f59d37f403b49cb8fb9c6b99cf81f78cff881 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 20:22:30 -0700 Subject: [PATCH 20/61] more elegant sleep instead of echo --- .github/workflows/test.yml | 3 +++ tests/test_mcp_e2e.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0d211e..5e058f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,6 +87,9 @@ jobs: url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" + - name: Wait briefly for MCP server to fully boot + run: sleep 3 + - name: Run pytest against deployed app env: MCP_SERVER_URL: ${{ steps.heroku_url.outputs.url }} diff --git a/tests/test_mcp_e2e.py b/tests/test_mcp_e2e.py index d7a86b4..4ba9539 100644 --- a/tests/test_mcp_e2e.py +++ b/tests/test_mcp_e2e.py @@ -54,7 +54,6 @@ async def test_list_tools(ctx): async def test_code_exec(ctx): - # payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) - data = await _safe_call(ctx, "call_tool", "--args", '{"name":"code_exec_python","arguments":{"code":"print(2+2)"}}') - # data = await _safe_call(ctx, "call_tool", "--args", payload) + payload = json.dumps({"name": "code_exec_python", "arguments": {"code": "print(2+2)"}}) + data = await _safe_call(ctx, "call_tool", "--args", payload) assert _extract_stdout(data) == "4" From d3574a3cb0e4886fad0d1f97997c89414adf6244 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Mon, 9 Jun 2025 20:26:12 -0700 Subject: [PATCH 21/61] Back to debug env --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e058f9..a680696 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,8 +87,10 @@ jobs: url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" - - name: Wait briefly for MCP server to fully boot - run: sleep 3 + - name: Debug env + run: | + echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty + echo "MCP_SERVER_URL = $MCP_SERVER_URL" - name: Run pytest against deployed app env: From a17ade0f6fb8d0879679d5ceba97d77ac3c33ecf Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Tue, 10 Jun 2025 08:09:58 -0700 Subject: [PATCH 22/61] Back to debug env --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a680696..e9a733b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,6 +87,9 @@ jobs: url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" + - name: Wait briefly for MCP server to fully boot + run: sleep 3 + - name: Debug env run: | echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty From 5df001fe0fb2fa3a517037c615e307ee9362db04 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Tue, 10 Jun 2025 08:45:40 -0700 Subject: [PATCH 23/61] Trigger CI From 09d4fc13aba8853023427a66c4cac99f56bae5d6 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Tue, 10 Jun 2025 13:32:57 -0700 Subject: [PATCH 24/61] Trigger CI From 938c9b48b1b0ed2fc7b45a55d32740d890514b96 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Tue, 10 Jun 2025 16:19:58 -0700 Subject: [PATCH 25/61] Trigger CI From f279a6073c53514365f3fd2b7223edb4338775ea Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Tue, 10 Jun 2025 16:45:49 -0700 Subject: [PATCH 26/61] slight README update --- README.md | 1 + tests/README.md | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 60fa8fb..4956408 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ uvicorn src.streamable_http_server:app --reload *Running with `--reload` is optional, but great for local development* Next, in a new pane, you can try running some queries against your server: + #### Local Streamable HTTP, SSE - Example Requests First run: ```bash diff --git a/tests/README.md b/tests/README.md index bc05d2f..5461a94 100644 --- a/tests/README.md +++ b/tests/README.md @@ -31,10 +31,15 @@ pip install -r requirements.txt ## 2 · Run local transports only ```bash +git push heroku :main +``` + +## 2 · Run local & one-off-dyno (STDIO) deployed transports +```bash pytest tests -q ``` -## 3 - Run local & remote transports +## 3 - Run local & all deployed transports ```bash MCP_SERVER_URL=$(heroku info -s -a "$APP_NAME" | grep web_url | cut -d= -f2 | tr -d '\n') \ API_KEY=$(heroku config:get API_KEY -a "$APP_NAME") \ From 5c4637c54245c47882b6b85687786ad467c2742a Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 09:06:08 -0700 Subject: [PATCH 27/61] Trigger CI From 0077b3cc219f1d65b82838e9e9ed1f5898bcd157 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 13:14:09 -0700 Subject: [PATCH 28/61] Longer sleep to see if this helps integration tests --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9a733b..3d2245c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,15 +81,15 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Wait briefly for MCP server to fully boot + run: sleep 10 + - name: Get Heroku app URL id: heroku_url run: | url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" - - name: Wait briefly for MCP server to fully boot - run: sleep 3 - - name: Debug env run: | echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty From eb5a09131d531b980d22a14ea1892e49509ef72f Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 15:31:22 -0700 Subject: [PATCH 29/61] 30s wait test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d2245c..84383ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,7 +82,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 10 + run: sleep 30 - name: Get Heroku app URL id: heroku_url From f19620ec6539b297443f4eb82e039e9d5a5b87db Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 15:40:49 -0700 Subject: [PATCH 30/61] test --- tests/conftest.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 192ae8a..fbe5624 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ import sys from pathlib import Path from typing import AsyncGenerator, Dict - +import httpx import pytest ROOT = Path(__file__).resolve().parents[1] @@ -71,14 +71,34 @@ async def _ctx_stdio_local() -> AsyncGenerator[Dict, None]: yield {"client": "stdio_client", "extra_env": {}} +async def _wait_until_remote_ready(url: str, timeout: int = 30): + for _ in range(timeout * 4): + try: + async with httpx.AsyncClient() as client: + resp = await client.get(url) + if resp.status_code == 200: + return + except Exception: + pass + await asyncio.sleep(0.25) + raise RuntimeError(f"Timed out waiting for remote MCP server at {url}") + + # ---------------------------------------------------------------- remote HTTP / SSE +# TODO: unhard-code client async def _ctx_remote() -> AsyncGenerator[Dict, None]: - url = os.getenv("MCP_SERVER_URL"); key = os.getenv("API_KEY") + url = os.getenv("MCP_SERVER_URL") + key = os.getenv("API_KEY") if not url or not key: pytest.skip("remote env-vars missing") - yield {"client": "streamable_http_client", - "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url.rstrip("/")}} - + await _wait_until_remote_ready(url) # <-- ADD THIS + yield { + "client": "streamable_http_client", + "extra_env": { + "API_KEY": key, + "MCP_SERVER_URL": url.rstrip("/") + }, + } # ---------------------------------------------------------------- remote STDIO ctx async def _ctx_remote_stdio() -> AsyncGenerator[Dict, None]: app = os.getenv("APP_NAME"); key = os.getenv("API_KEY") From 689081e246d248cf132e69146bb58e3f740e240c Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 15:46:51 -0700 Subject: [PATCH 31/61] My fix --- tests/conftest.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fbe5624..88adf9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ import sys from pathlib import Path from typing import AsyncGenerator, Dict -import httpx + import pytest ROOT = Path(__file__).resolve().parents[1] @@ -71,34 +71,17 @@ async def _ctx_stdio_local() -> AsyncGenerator[Dict, None]: yield {"client": "stdio_client", "extra_env": {}} -async def _wait_until_remote_ready(url: str, timeout: int = 30): - for _ in range(timeout * 4): - try: - async with httpx.AsyncClient() as client: - resp = await client.get(url) - if resp.status_code == 200: - return - except Exception: - pass - await asyncio.sleep(0.25) - raise RuntimeError(f"Timed out waiting for remote MCP server at {url}") - - # ---------------------------------------------------------------- remote HTTP / SSE -# TODO: unhard-code client async def _ctx_remote() -> AsyncGenerator[Dict, None]: url = os.getenv("MCP_SERVER_URL") key = os.getenv("API_KEY") + server_type = os.getenv("REMOTE_SERVER_TYPE") + if not url or not key: pytest.skip("remote env-vars missing") - await _wait_until_remote_ready(url) # <-- ADD THIS - yield { - "client": "streamable_http_client", - "extra_env": { - "API_KEY": key, - "MCP_SERVER_URL": url.rstrip("/") - }, - } + yield {"client": server_type.replace("server", "client"), + "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url, "REMOTE_SERVER_TYPE": server_type}} + # ---------------------------------------------------------------- remote STDIO ctx async def _ctx_remote_stdio() -> AsyncGenerator[Dict, None]: app = os.getenv("APP_NAME"); key = os.getenv("API_KEY") From 2900a5b16d6fb21c5628cf0cb0797100ab4c6394 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 15:55:02 -0700 Subject: [PATCH 32/61] My fix --- .github/workflows/test.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84383ff..c500fc8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,22 +82,20 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 30 + run: sleep 3 - - name: Get Heroku app URL - id: heroku_url + - name: Get Heroku env vars + id: heroku_env run: | url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" - - - name: Debug env - run: | - echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty - echo "MCP_SERVER_URL = $MCP_SERVER_URL" + server_type=$(heroku config:get REMOTE_SERVER_TYPE -a $APP_NAME) + echo "server_type=$server_type" >> "$GITHUB_OUTPUT" - name: Run pytest against deployed app env: - MCP_SERVER_URL: ${{ steps.heroku_url.outputs.url }} + MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} + REMOTE_SERVER_TYPE: ${{ steps.heroku_env.outputs.server_type }} API_KEY: ${{ env.API_KEY }} run: pytest -q From 3e9672d5a389b9b44e0e2f588871012a87706799 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:06:30 -0700 Subject: [PATCH 33/61] My fix --- .github/workflows/test.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c500fc8..8fde3fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,7 @@ jobs: # TODO: make this into a github secret, just in case API_KEY: ci-test-key APP_NAME: mcp-toolkit-pr-${{ github.event.number }} + REMOTE_SERVER_TYPE: streamable_http_server steps: - uses: actions/checkout@v4 @@ -61,6 +62,7 @@ jobs: run: | heroku create $APP_NAME heroku config:set API_KEY=$API_KEY --app $APP_NAME + heroku config:set REMOTE_SERVER_TYPE=$REMOTE_SERVER_TYPE --app $APP_NAME - name: Deploy this branch to Heroku run: | @@ -89,15 +91,18 @@ jobs: run: | url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n') echo "url=$url" >> "$GITHUB_OUTPUT" - server_type=$(heroku config:get REMOTE_SERVER_TYPE -a $APP_NAME) - echo "server_type=$server_type" >> "$GITHUB_OUTPUT" - name: Run pytest against deployed app env: + APP_NAME: ${{ env.APP_NAME }} MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} - REMOTE_SERVER_TYPE: ${{ steps.heroku_env.outputs.server_type }} + REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} API_KEY: ${{ env.API_KEY }} - run: pytest -q + run: | + pytest -q + echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty + echo "MCP_SERVER_URL = $MCP_SERVER_URL" + echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" - name: Destroy Heroku app after test if: always() From 80d62a6ab354d347a7f64b41ad824c7edbcf82bb Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:12:57 -0700 Subject: [PATCH 34/61] test --- .github/workflows/test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8fde3fa..10d70cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 3 + run: sleep 30 - name: Get Heroku env vars id: heroku_env @@ -99,10 +99,12 @@ jobs: REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} API_KEY: ${{ env.API_KEY }} run: | - pytest -q - echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty + echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = $MCP_SERVER_URL" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" + echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty + pytest -q + - name: Destroy Heroku app after test if: always() From e35967ffefac6f0e2c70a61a24d776ef8f3dc3da Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:17:06 -0700 Subject: [PATCH 35/61] test --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10d70cd..c6fef74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -93,17 +93,17 @@ jobs: echo "url=$url" >> "$GITHUB_OUTPUT" - name: Run pytest against deployed app - env: - APP_NAME: ${{ env.APP_NAME }} - MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} - REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} - API_KEY: ${{ env.API_KEY }} + # env: + # APP_NAME: ${{ env.APP_NAME }} + # MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} + # REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} + # API_KEY: ${{ env.API_KEY }} run: | echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = $MCP_SERVER_URL" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty - pytest -q + MCP_SERVER_URL=${{ steps.heroku_env.outputs.url }} API_KEY=${API_KEY} REMOTE_SERVER_TYPE=${REMOTE_SERVER_TYPE} pytest -q - name: Destroy Heroku app after test From 51b67ed6938792e715a56559b83aae5d24d875be Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:35:40 -0700 Subject: [PATCH 36/61] Trigger CI From 071d2e7fdabcaebd9841d4815abe48377b6556d2 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:42:32 -0700 Subject: [PATCH 37/61] test --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6fef74..f076494 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,6 +99,7 @@ jobs: # REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} # API_KEY: ${{ env.API_KEY }} run: | + curl -i -H "Authorization: Bearer $API_KEY" ${{ steps.heroku_env.outputs.url }}/mcp/ echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = $MCP_SERVER_URL" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" @@ -109,4 +110,4 @@ jobs: - name: Destroy Heroku app after test if: always() run: | - heroku apps:destroy --app $APP_NAME --confirm $APP_NAME \ No newline at end of file + heroku apps:destroy --app $APP_NAME --confirm $APP_NAME mm \ No newline at end of file From 67118f077893202f90748b1b5c444a13e6cdc31c Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 16:50:38 -0700 Subject: [PATCH 38/61] test --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f076494..1ed810d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,7 @@ jobs: # REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} # API_KEY: ${{ env.API_KEY }} run: | - curl -i -H "Authorization: Bearer $API_KEY" ${{ steps.heroku_env.outputs.url }}/mcp/ + curl -i -H "Authorization: Bearer $API_KEY" ${{ steps.heroku_env.outputs.url }}mcp/ echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = $MCP_SERVER_URL" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" @@ -110,4 +110,4 @@ jobs: - name: Destroy Heroku app after test if: always() run: | - heroku apps:destroy --app $APP_NAME --confirm $APP_NAME mm \ No newline at end of file + heroku apps:destroy --app $APP_NAME --confirm $APP_NAME \ No newline at end of file From 306fbf5cec7f740210551079c1d18674f20a5bf0 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:07:32 -0700 Subject: [PATCH 39/61] test --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1ed810d..7a3f2e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 30 + run: sleep 10 - name: Get Heroku env vars id: heroku_env @@ -101,7 +101,7 @@ jobs: run: | curl -i -H "Authorization: Bearer $API_KEY" ${{ steps.heroku_env.outputs.url }}mcp/ echo "APP_NAME = $APP_NAME" - echo "MCP_SERVER_URL = $MCP_SERVER_URL" + echo "MCP_SERVER_URL = ${{ steps.heroku_env.outputs.url }}" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty MCP_SERVER_URL=${{ steps.heroku_env.outputs.url }} API_KEY=${API_KEY} REMOTE_SERVER_TYPE=${REMOTE_SERVER_TYPE} pytest -q From a6cff215be5e3e6f7d52e8a4f89a03373a561f30 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:19:53 -0700 Subject: [PATCH 40/61] test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a3f2e6..bc89389 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 10 + run: sleep 30 - name: Get Heroku env vars id: heroku_env From c32d984699f445ee58e7ac5eae377623f0568788 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:23:27 -0700 Subject: [PATCH 41/61] test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc89389..5460555 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 30 + run: sleep 60 - name: Get Heroku env vars id: heroku_env From 35fb837fc89295eaa5acb66088e46d402e5c55b6 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:32:48 -0700 Subject: [PATCH 42/61] test --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5460555..be05500 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,6 @@ jobs: # REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} # API_KEY: ${{ env.API_KEY }} run: | - curl -i -H "Authorization: Bearer $API_KEY" ${{ steps.heroku_env.outputs.url }}mcp/ echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = ${{ steps.heroku_env.outputs.url }}" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" From 1267d0d6b0083bb69c902171fb7be90a7b8a9698 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:37:34 -0700 Subject: [PATCH 43/61] test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be05500..6520640 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: pip install -r requirements.txt - name: Wait briefly for MCP server to fully boot - run: sleep 60 + run: sleep 90 - name: Get Heroku env vars id: heroku_env From de4978d48ac548060fd2a79af2ae7bd3145766d3 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 17:41:16 -0700 Subject: [PATCH 44/61] test --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6520640..23cae96 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,6 +83,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + # sometimes this is totally uneccessary, and other times servers fail w/out this... - name: Wait briefly for MCP server to fully boot run: sleep 90 @@ -93,11 +94,6 @@ jobs: echo "url=$url" >> "$GITHUB_OUTPUT" - name: Run pytest against deployed app - # env: - # APP_NAME: ${{ env.APP_NAME }} - # MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} - # REMOTE_SERVER_TYPE: ${{ env.REMOTE_SERVER_TYPE }} - # API_KEY: ${{ env.API_KEY }} run: | echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = ${{ steps.heroku_env.outputs.url }}" From 7cbfdc1cb16dc822570189f91d3550d46f7bc710 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 20:04:48 -0700 Subject: [PATCH 45/61] test --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23cae96..cb7c6c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -94,12 +94,14 @@ jobs: echo "url=$url" >> "$GITHUB_OUTPUT" - name: Run pytest against deployed app + env: + MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} run: | echo "APP_NAME = $APP_NAME" - echo "MCP_SERVER_URL = ${{ steps.heroku_env.outputs.url }}" + echo "MCP_SERVER_URL = $MCP_SERVER_URL" echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty - MCP_SERVER_URL=${{ steps.heroku_env.outputs.url }} API_KEY=${API_KEY} REMOTE_SERVER_TYPE=${REMOTE_SERVER_TYPE} pytest -q + pytest -q - name: Destroy Heroku app after test From 0c0adf51c978914e74be65fb1aeb596183d1cc90 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Wed, 11 Jun 2025 20:30:49 -0700 Subject: [PATCH 46/61] test --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb7c6c2..b45d023 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,6 +102,8 @@ jobs: echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q + pytest -q + pytest -q - name: Destroy Heroku app after test From 3bf3e26ac318aa6531101b55602919f93e280d1f Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 07:44:51 -0700 Subject: [PATCH 47/61] test --- .github/workflows/test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b45d023..86bdf95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,10 +102,29 @@ jobs: echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q + + - name: Run pytest against deployed app 2 + env: + MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} + run: | + echo "APP_NAME = $APP_NAME" + echo "MCP_SERVER_URL = $MCP_SERVER_URL" + echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" + echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q + + - name: Run pytest against deployed app 3 + env: + MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} + run: | + echo "APP_NAME = $APP_NAME" + echo "MCP_SERVER_URL = $MCP_SERVER_URL" + echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" + echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q + - name: Destroy Heroku app after test if: always() run: | From f89c148af5fcf7731a4638aba2bae0cdd299b587 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 07:50:42 -0700 Subject: [PATCH 48/61] test --- .github/workflows/test.yml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86bdf95..dcfbd6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,7 +85,7 @@ jobs: # sometimes this is totally uneccessary, and other times servers fail w/out this... - name: Wait briefly for MCP server to fully boot - run: sleep 90 + run: sleep 3 - name: Get Heroku env vars id: heroku_env @@ -103,28 +103,6 @@ jobs: echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q - - name: Run pytest against deployed app 2 - env: - MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} - run: | - echo "APP_NAME = $APP_NAME" - echo "MCP_SERVER_URL = $MCP_SERVER_URL" - echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" - echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty - pytest -q - - - name: Run pytest against deployed app 3 - env: - MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }} - run: | - echo "APP_NAME = $APP_NAME" - echo "MCP_SERVER_URL = $MCP_SERVER_URL" - echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" - echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty - pytest -q - - - - name: Destroy Heroku app after test if: always() run: | From ba6c0d6a2a5b41a5e4be96d1c75d9b9de2fed0f5 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 07:58:22 -0700 Subject: [PATCH 49/61] test --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcfbd6c..f13cc9d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,8 +61,12 @@ jobs: - name: Create temp Heroku app for this PR run: | heroku create $APP_NAME + heroku buildpacks:set heroku/python -a $APP_NAME heroku config:set API_KEY=$API_KEY --app $APP_NAME + heroku config:set STDIO_MODE_ONLY=false heroku config:set REMOTE_SERVER_TYPE=$REMOTE_SERVER_TYPE --app $APP_NAME + heroku config:set WEB_CONCURRENCY=1 --app $APP_NAME + heroku ps:scale web=1 -a $APP_NAME - name: Deploy this branch to Heroku run: | From 1c540b1fc1ebc68534a70fd861a95320ce604d21 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 08:03:57 -0700 Subject: [PATCH 50/61] test --- .github/workflows/test.yml | 7 +++---- tests/conftest.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f13cc9d..cbc5415 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: # TODO: make this into a github secret, just in case API_KEY: ci-test-key APP_NAME: mcp-toolkit-pr-${{ github.event.number }} - REMOTE_SERVER_TYPE: streamable_http_server + REMOTE_SERVER_TRANSPORT_MODULE: streamable_http_server steps: - uses: actions/checkout@v4 @@ -64,9 +64,8 @@ jobs: heroku buildpacks:set heroku/python -a $APP_NAME heroku config:set API_KEY=$API_KEY --app $APP_NAME heroku config:set STDIO_MODE_ONLY=false - heroku config:set REMOTE_SERVER_TYPE=$REMOTE_SERVER_TYPE --app $APP_NAME + heroku config:set REMOTE_SERVER_TRANSPORT_MODULE=$REMOTE_SERVER_TRANSPORT_MODULE --app $APP_NAME heroku config:set WEB_CONCURRENCY=1 --app $APP_NAME - heroku ps:scale web=1 -a $APP_NAME - name: Deploy this branch to Heroku run: | @@ -103,7 +102,7 @@ jobs: run: | echo "APP_NAME = $APP_NAME" echo "MCP_SERVER_URL = $MCP_SERVER_URL" - echo "REMOTE_SERVER_TYPE = $REMOTE_SERVER_TYPE" + echo "REMOTE_SERVER_TRANSPORT_MODULE = $REMOTE_SERVER_TRANSPORT_MODULE" echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty pytest -q diff --git a/tests/conftest.py b/tests/conftest.py index 88adf9e..20f2911 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,12 +75,12 @@ async def _ctx_stdio_local() -> AsyncGenerator[Dict, None]: async def _ctx_remote() -> AsyncGenerator[Dict, None]: url = os.getenv("MCP_SERVER_URL") key = os.getenv("API_KEY") - server_type = os.getenv("REMOTE_SERVER_TYPE") + server_type = os.getenv("REMOTE_SERVER_TRANSPORT_MODULE") if not url or not key: pytest.skip("remote env-vars missing") yield {"client": server_type.replace("server", "client"), - "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url, "REMOTE_SERVER_TYPE": server_type}} + "extra_env": {"API_KEY": key, "MCP_SERVER_URL": url, "REMOTE_SERVER_TRANSPORT_MODULE": server_type}} # ---------------------------------------------------------------- remote STDIO ctx async def _ctx_remote_stdio() -> AsyncGenerator[Dict, None]: From b790f001435e64d3f22cc85c05915ea679df74b4 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 08:07:50 -0700 Subject: [PATCH 51/61] Fiiiinally --- .github/workflows/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbc5415..5e06dcc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,6 +58,8 @@ jobs: run: | echo "$HEROKU_API_KEY" | heroku auth:token + # github CI can't use our app.json, so the config etc bits must be set manually. + # note WEB_CONCURRENCY is important! You get non-deterministic errors w/out it. - name: Create temp Heroku app for this PR run: | heroku create $APP_NAME @@ -86,10 +88,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - # sometimes this is totally uneccessary, and other times servers fail w/out this... - - name: Wait briefly for MCP server to fully boot - run: sleep 3 - - name: Get Heroku env vars id: heroku_env run: | From 7598ccf3460eedfec00087ff40f9ff168a072113 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 08:16:51 -0700 Subject: [PATCH 52/61] comments --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e06dcc..c4dfba8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,9 +40,11 @@ jobs: runs-on: ubuntu-latest env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} - # TODO: make this into a github secret, just in case API_KEY: ci-test-key APP_NAME: mcp-toolkit-pr-${{ github.event.number }} + # also note that github CI doesn't have access to your app's config vars, so here we're setting the remote + # server type to streamable HTTP. Folks using SSE would need to change this line for their e2e remote integration + # tests to test SSE instead of streamable HTTP. REMOTE_SERVER_TRANSPORT_MODULE: streamable_http_server steps: From 082b49722e7f413afe64f9c6a2c975444e8568d7 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 08:17:55 -0700 Subject: [PATCH 53/61] comments --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4dfba8..c95af3c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: run: pytest -q ########################################################################### - # 2 - Deploy this PR to a temp Heroku app and run remote tests + # 2 - Deploy this PR to a temp Heroku app and run tests against deployed app (in addition to 'local') ########################################################################### remote: runs-on: ubuntu-latest From 2f295a6867dac598a31e217b81e97527593aa7bc Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:02:29 -0700 Subject: [PATCH 54/61] precautionary pre-cleanup step --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c95af3c..86f565a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,11 @@ jobs: run: | echo "$HEROKU_API_KEY" | heroku auth:token + - name: Pre-cleanup (destroy app if it exists) + continue-on-error: true + run: | + heroku apps:destroy --app $APP_NAME --confirm $APP_NAME + # github CI can't use our app.json, so the config etc bits must be set manually. # note WEB_CONCURRENCY is important! You get non-deterministic errors w/out it. - name: Create temp Heroku app for this PR From 61946c166fc45b8437094c4eebb304f400e4188e Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:10:03 -0700 Subject: [PATCH 55/61] README edit --- tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 5461a94..0a33b16 100644 --- a/tests/README.md +++ b/tests/README.md @@ -41,7 +41,10 @@ pytest tests -q ## 3 - Run local & all deployed transports ```bash +REMOTE_SERVER_TYPE=$(heroku config:get REMOTE_SERVER_TYPE) \ MCP_SERVER_URL=$(heroku info -s -a "$APP_NAME" | grep web_url | cut -d= -f2 | tr -d '\n') \ API_KEY=$(heroku config:get API_KEY -a "$APP_NAME") \ pytest tests -q -``` \ No newline at end of file +``` + +*NOTE: if your `REMOTE_SERVER_TYPE` is set to `sse_server` and not the default `streamable_http_server`, you'll need to change the `REMOTE_SERVER_TRANSPORT_MODULE` declaration line in `.github/workflows/test.yml` to make sure that the end to end integration tests against the temporary deployed remote server are using the appropriate client code.* \ No newline at end of file From e5f1250c9bbf88b4b9032f6e369b4bc3367722e2 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:13:53 -0700 Subject: [PATCH 56/61] avoid name clashing in app name --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86f565a..ba7f530 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,8 @@ jobs: env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} API_KEY: ci-test-key - APP_NAME: mcp-toolkit-pr-${{ github.event.number }} + # setting an $APP_NAME that will be unique even if folks choose to fork this repo --> avoids clashes. + APP_NAME: mcp-${{ github.repository_owner }}-${{ github.event.repository.name }}-pr-${{ github.event.number }} # also note that github CI doesn't have access to your app's config vars, so here we're setting the remote # server type to streamable HTTP. Folks using SSE would need to change this line for their e2e remote integration # tests to test SSE instead of streamable HTTP. From 3311a92c723736c6ce7f92df94f850341610904a Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:26:45 -0700 Subject: [PATCH 57/61] Moved python install location in yml --- .github/workflows/test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba7f530..e7bfb8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,10 @@ jobs: with: fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by? + - uses: actions/setup-python@v5 + with: + python-version: "3.12" # this should align with this repo's .python-version file + - name: Install Heroku CLI run: | curl https://cli-assets.heroku.com/install.sh | sh @@ -81,11 +85,6 @@ jobs: run: | git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$APP_NAME.git HEAD:refs/heads/main --force - - uses: actions/setup-python@v5 - with: - # should align with .python-version - python-version: "3.12" - - uses: actions/cache@v4 with: path: ~/.cache/pip From 44809e063c99369b131d1a1e9e9e590c05213990 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:28:20 -0700 Subject: [PATCH 58/61] yml comment --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7bfb8c..7e6fe34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by? + fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by when running git push heroku later on - uses: actions/setup-python@v5 with: From 76c55dfbdcaf80c4f3fd374bda53c8ea4ffa7550 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:43:02 -0700 Subject: [PATCH 59/61] Short app name precaution --- .github/workflows/test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e6fe34..7665d04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,18 +41,27 @@ jobs: env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} API_KEY: ci-test-key - # setting an $APP_NAME that will be unique even if folks choose to fork this repo --> avoids clashes. - APP_NAME: mcp-${{ github.repository_owner }}-${{ github.event.repository.name }}-pr-${{ github.event.number }} # also note that github CI doesn't have access to your app's config vars, so here we're setting the remote # server type to streamable HTTP. Folks using SSE would need to change this line for their e2e remote integration # tests to test SSE instead of streamable HTTP. REMOTE_SERVER_TRANSPORT_MODULE: streamable_http_server + # $APP_NAME is set below because we need to shorten the repo owner's name, as a precaution steps: - uses: actions/checkout@v4 with: fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by when running git push heroku later on + # Setting a short $APP_NAME that will be unique even if folks choose to fork this repo --> avoids clashes. + # Needs to be shortened if the github repo owner has a long name (max 30 char app name heroku limit). + - name: Generate short APP_NAME + id: appname + run: | + OWNER_SHORT=${GITHUB_REPOSITORY_OWNER:0:5} + APP_NAME="${OWNER_SHORT}-${GITHUB_REPOSITORY}-${GITHUB_EVENT_NUMBER}" + echo "APP_NAME=$APP_NAME" + echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV + - uses: actions/setup-python@v5 with: python-version: "3.12" # this should align with this repo's .python-version file From aa02a1a66a0e1a004976655e3b4950610e99393b Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 10:47:50 -0700 Subject: [PATCH 60/61] Short app name precaution --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7665d04..ba7bc1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,9 @@ jobs: id: appname run: | OWNER_SHORT=${GITHUB_REPOSITORY_OWNER:0:5} - APP_NAME="${OWNER_SHORT}-${GITHUB_REPOSITORY}-${GITHUB_EVENT_NUMBER}" + REPO_NAME=$(basename "$GITHUB_REPOSITORY") + PR_NUMBER=$(jq .number "$GITHUB_EVENT_PATH") + APP_NAME="${OWNER_SHORT}-${REPO_NAME}-${PR_NUMBER}" echo "APP_NAME=$APP_NAME" echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV From df406c66b0574ce8bbe1c9857857eeb96cd2e219 Mon Sep 17 00:00:00 2001 From: Hillary Sanders Date: Thu, 12 Jun 2025 11:01:05 -0700 Subject: [PATCH 61/61] automated python version --- .github/workflows/test.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba7bc1b..be91f08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,12 +13,17 @@ jobs: API_KEY: ci-test-key steps: - # Optional: cache wheels to speed up numpy / scipy installs - uses: actions/checkout@v4 + - name: Read Python version from .python-version + id: python-version + run: | + PY_VERSION=$(cat .python-version) + echo "version=$PY_VERSION" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.python-version.outputs.version }} - uses: actions/cache@v4 with: @@ -64,9 +69,15 @@ jobs: echo "APP_NAME=$APP_NAME" echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV + - name: Read Python version from .python-version + id: python-version + run: | + PY_VERSION=$(cat .python-version) + echo "version=$PY_VERSION" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v5 with: - python-version: "3.12" # this should align with this repo's .python-version file + python-version: ${{ steps.python-version.outputs.version }} - name: Install Heroku CLI run: |