Smoke v2

.github/workflows/js.yaml

@@ -614,3 +614,90 @@ jobs:
         working-directory: js/smoke
         run: |
           ./run-tests.sh ${{ matrix.mode }}
+
+  # Smoke v2: Auto-discovery of scenarios
+  smoke-v2-discover:
+    runs-on: ubuntu-latest
+    outputs:
+      scenarios: ${{ steps.discover.outputs.scenarios }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Discover scenarios
+        id: discover
+        working-directory: js/smoke-v2
+        shell: bash
+        run: |
+          # Use the same auto-discovery logic as the Makefile
+          SCENARIOS=$(command find scenarios -mindepth 1 -maxdepth 1 -type d -exec test -f {}/Makefile \; -print | sed 's|scenarios/||' | sort | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "scenarios=$SCENARIOS" >> "$GITHUB_OUTPUT"
+          echo "Discovered scenarios: $SCENARIOS"
+
+  smoke-v2-test:
+    needs: [build, smoke-v2-discover]
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        scenario: ${{ fromJson(needs.smoke-v2-discover.outputs.scenarios) }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup mise
+        uses: jdx/mise-action@v2
+        with:
+          working_directory: js/smoke-v2
+
+      - uses: pnpm/action-setup@v4
+
+      - name: Download build artifact (node 20)
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build.outputs.artifact-name }}-20-dist
+          path: js/artifacts
+          run-id: ${{ github.run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Prepare artifacts at well-known paths
+        working-directory: js/artifacts
+        shell: bash
+        run: |
+          # Copy braintrust tarball to well-known path
+          for f in braintrust-[0-9]*.tgz; do
+            if [ -f "$f" ]; then
+              cp "$f" braintrust-latest.tgz
+              echo "Copied $f to braintrust-latest.tgz"
+              break
+            fi
+          done
+
+          # Copy otel tarball to well-known path
+          for f in braintrust-otel-[0-9]*.tgz; do
+            if [ -f "$f" ]; then
+              cp "$f" braintrust-otel-latest.tgz
+              echo "Copied $f to braintrust-otel-latest.tgz"
+              break
+            fi
+          done
+
+          ls -la
+
+      - name: Build shared test package (once for all scenarios)
+        working-directory: js/smoke-v2/shared
+        shell: bash
+        run: |
+          npm ci
+          npm run build
+
+      - name: Run scenario ${{ matrix.scenario }}
+        working-directory: js/smoke-v2
+        shell: bash
+        env:
+          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+          CI: true
+          BRAINTRUST_TAR: ../artifacts/braintrust-latest.tgz
+          BRAINTRUST_OTEL_TAR: ../artifacts/braintrust-otel-latest.tgz
+          SMOKE_V2_SHARED_DIST: shared/dist
+        run: |
+          make test ${{ matrix.scenario }}

js/package.json

@@ -56,7 +56,7 @@
     "watch": "tsup --watch",
     "clean": "rm -r dist/* && rm -r dev/dist/*",
     "docs": "npx typedoc --options typedoc.json src/index.ts",
-    "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"",
+    "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"smoke-v2/**/*.test.*\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"",
     "test:api-compat": "vitest run tests/api-compatibility/api-compatibility.test.ts",
     "test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
     "test:openai": "vitest run src/wrappers/oai.test.ts",

js/smoke-v2/Makefile

@@ -0,0 +1,129 @@
+# Auto-discover scenarios (any folder in scenarios/ with a Makefile)
+SCENARIOS := $(shell find scenarios -mindepth 1 -maxdepth 1 -type d -exec test -f {}/Makefile \; -print | sed 's|scenarios/||' | sort)
+
+.PHONY: help setup test list clean
+
+# =============================================================================
+# Help
+# =============================================================================
+
+help:
+	@echo "Smoke-v2 Test Infrastructure"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  make test            - Run all scenarios (builds SDK once if needed)"
+	@echo "  make test <scenario> - Run specific scenario (e.g., make test otel-v1)"
+	@echo "  make setup           - Ensure SDK artifacts + shared package are ready"
+	@echo "  make clean           - Remove all build artifacts (force rebuild)"
+	@echo "  make list            - List discovered scenarios"
+	@echo ""
+	@echo "Environment variables:"
+	@echo "  BRAINTRUST_TAR           - Path to braintrust tarball (auto-set by parent when running locally)"
+	@echo "  BRAINTRUST_OTEL_TAR      - Path to otel tarball (auto-set by parent when running locally)"
+	@echo "  SMOKE_V2_SHARED_DIST     - Path to shared package dist (auto-set by parent when running locally)"
+	@echo ""
+	@echo "Running individual scenarios:"
+	@echo "  cd scenarios/otel-v1 && make test   - Runs standalone, builds if needed"
+	@echo ""
+	@echo "Discovered scenarios:"
+	@for scenario in $(SCENARIOS); do echo "  - $$scenario"; done
+
+# =============================================================================
+# Setup - Build SDK + shared package
+# =============================================================================
+
+setup:
+	@echo "==> Setting up smoke-v2"
+	@mkdir -p ../artifacts
+
+	@# Build SDK and create tarball if not provided
+	@if [ -n "$(BRAINTRUST_TAR)" ] && [ -f "$(BRAINTRUST_TAR)" ]; then \
+		echo "==> Using provided BRAINTRUST_TAR: $(BRAINTRUST_TAR)"; \
+	else \
+		echo "==> Building SDK and creating tarball"; \
+		cd ../../.. && pnpm exec turbo build --filter=braintrust && pnpm --filter=braintrust pack --pack-destination sdk/js/artifacts; \
+		\
+		for f in sdk/js/artifacts/braintrust-*.tgz; do \
+			if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \
+			   [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ]; then \
+				cp "$$f" sdk/js/artifacts/braintrust-latest.tgz; \
+				break; \
+			fi; \
+		done; \
+	fi
+
+	@# Build shared package if not provided
+	@if [ -n "$(SMOKE_V2_SHARED_DIST)" ] && [ -d "$(SMOKE_V2_SHARED_DIST)" ]; then \
+		echo "==> Using provided SMOKE_V2_SHARED_DIST: $(SMOKE_V2_SHARED_DIST)"; \
+	else \
+		if [ ! -d shared/dist ]; then \
+			echo "==> Building shared package"; \
+			cd shared && npm ci && npm run build; \
+		else \
+			echo "==> Shared package already built"; \
+		fi; \
+	fi
+
+# =============================================================================
+# Clean - Remove all build artifacts
+# =============================================================================
+
+clean:
+	@echo "==> Cleaning smoke-v2 artifacts"
+	rm -rf ../artifacts/*.tgz
+	rm -rf shared/dist shared/node_modules
+
+# =============================================================================
+# Test - Run scenarios
+# =============================================================================
+
+test:
+	@REQUESTED="$(filter-out test,$(MAKECMDGOALS))"; \
+	\
+	if [ -z "$$REQUESTED" ]; then \
+		SCENARIOS_TO_RUN="$(SCENARIOS)"; \
+		echo "==> Running all scenarios"; \
+	else \
+		SCENARIOS_TO_RUN=""; \
+		for scenario in $$REQUESTED; do \
+			if [ ! -d "scenarios/$$scenario" ]; then \
+				echo "Error: Scenario '$$scenario' not found"; \
+				echo "Available scenarios:"; \
+				for s in $(SCENARIOS); do echo "  - $$s"; done; \
+				exit 1; \
+			fi; \
+			SCENARIOS_TO_RUN="$$SCENARIOS_TO_RUN $$scenario"; \
+		done; \
+		echo "==> Running scenarios:$$SCENARIOS_TO_RUN"; \
+	fi; \
+	\
+	$(MAKE) setup; \
+	\
+	: $${BRAINTRUST_TAR:=../artifacts/braintrust-latest.tgz}; \
+	: $${BRAINTRUST_OTEL_TAR:=../artifacts/braintrust-otel-latest.tgz}; \
+	: $${SMOKE_V2_SHARED_DIST:=shared/dist}; \
+	export BRAINTRUST_TAR BRAINTRUST_OTEL_TAR SMOKE_V2_SHARED_DIST; \
+	\
+	for scenario in $$SCENARIOS_TO_RUN; do \
+		echo ""; \
+		echo "=== Testing $$scenario ==="; \
+		$(MAKE) -C scenarios/$$scenario test || exit 1; \
+	done; \
+	\
+	echo ""; \
+	echo "✓ All requested scenarios passed"
+
+# =============================================================================
+# List - Show discovered scenarios
+# =============================================================================
+
+list:
+	@echo "Discovered scenarios:"
+	@for scenario in $(SCENARIOS); do echo "  - $$scenario"; done
+
+# =============================================================================
+# Make scenario names valid targets (no-ops to support "make test otel-v1")
+# =============================================================================
+
+$(SCENARIOS):
+	@: