support prompt caching token tracking in langchain (#1250) #4929
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: js | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| actions: write | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| node-version: [20, 22] | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY}} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY}} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY}} | |
| outputs: | |
| artifact-name: ${{ steps.artifact.outputs.name }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| registry-url: "https://registry.npmjs.org" | |
| - uses: pnpm/action-setup@v4 | |
| - name: "verify ci" | |
| shell: bash | |
| run: | | |
| make js-verify-ci | |
| - name: Determine artifact name | |
| id: artifact | |
| env: | |
| ARTIFACT_NAME: javascript-sdk-dev-${{ github.run_id }} | |
| run: | | |
| echo "Artifact: $ARTIFACT_NAME" | |
| echo "name=$ARTIFACT_NAME" >> "$GITHUB_OUTPUT" | |
| - name: Prepare artifact | |
| id: prepare_artifact | |
| working-directory: js | |
| shell: bash | |
| run: | | |
| mkdir -p artifacts | |
| PACKED_TARBALL=$(npm pack --pack-destination artifacts) | |
| echo "packed_tarball=$PACKED_TARBALL" >> "$GITHUB_OUTPUT" | |
| - name: Build and pack @braintrust/otel | |
| id: prepare_otel_artifact | |
| shell: bash | |
| run: | | |
| # Install the built braintrust package in otel-js to satisfy peer dependency | |
| BRAINTRUST_TARBALL=$(ls js/artifacts/braintrust-*.tgz | head -n 1) | |
| if [ -z "$BRAINTRUST_TARBALL" ]; then | |
| echo "Error: braintrust tarball not found" | |
| exit 1 | |
| fi | |
| echo "Using braintrust tarball: $BRAINTRUST_TARBALL" | |
| # Install braintrust and OpenTelemetry peer dependencies in otel-js directory for build | |
| # Use pnpm to match the workspace, with no-save flags to avoid modifying lockfile | |
| cd integrations/otel-js | |
| if ! npm_config_save=false npm_config_lockfile=false pnpm add \ | |
| file:../../$BRAINTRUST_TARBALL \ | |
| @opentelemetry/api@^1.9.0 \ | |
| @opentelemetry/core@^1.9.0 \ | |
| @opentelemetry/exporter-trace-otlp-http@^0.35.0 \ | |
| @opentelemetry/sdk-trace-base@^1.9.0; then | |
| echo "Error: Failed to install dependencies" | |
| exit 1 | |
| fi | |
| # Build the otel package | |
| pnpm run build | |
| # Pack the otel package | |
| PACKED_OTEL_TARBALL=$(npm pack --pack-destination ../../js/artifacts) | |
| echo "packed_otel_tarball=$PACKED_OTEL_TARBALL" >> "$GITHUB_OUTPUT" | |
| - name: List artifacts before upload | |
| shell: bash | |
| run: | | |
| echo "Braintrust tarball: ${{ steps.prepare_artifact.outputs.packed_tarball }}" | |
| echo "Otel tarball: ${{ steps.prepare_otel_artifact.outputs.packed_otel_tarball }}" | |
| ls -la js/artifacts/ | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.artifact.outputs.name }}-${{ matrix.node-version }}-dist | |
| path: | | |
| js/artifacts/${{ steps.prepare_artifact.outputs.packed_tarball }} | |
| js/artifacts/${{ steps.prepare_otel_artifact.outputs.packed_otel_tarball }} | |
| retention-days: 1 | |
| api-compatibility: | |
| runs-on: ubuntu-latest | |
| # Run in parallel with build job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node-version: [20] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Fetch full history to access main branch for baseline | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| registry-url: "https://registry.npmjs.org" | |
| - uses: pnpm/action-setup@v4 | |
| - name: Get published braintrust version for cache key | |
| id: published-version | |
| shell: bash | |
| run: | | |
| PUBLISHED_VERSION=$(npm view braintrust version 2>/dev/null || echo "none") | |
| echo "version=$PUBLISHED_VERSION" >> "$GITHUB_OUTPUT" | |
| echo "Published version: $PUBLISHED_VERSION" | |
| - name: Cache API compatibility test tarball | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.npm/_cacache | |
| key: braintrust-tarball-${{ steps.published-version.outputs.version }}-${{ runner.os }} | |
| restore-keys: | | |
| braintrust-tarball-${{ steps.published-version.outputs.version }}- | |
| braintrust-tarball- | |
| - name: Build baseline from main branch for API compatibility | |
| shell: bash | |
| run: | | |
| # Save current state | |
| CURRENT_REF=$(git rev-parse HEAD) | |
| BASELINE_DIR="$HOME/braintrust-baseline" | |
| mkdir -p "$BASELINE_DIR" | |
| # Checkout main and build | |
| echo "Building baseline from main branch..." | |
| git checkout origin/main || { echo "Could not checkout main"; exit 0; } | |
| # Install and build main | |
| pnpm install --frozen-lockfile || { echo "Baseline install failed"; git checkout "$CURRENT_REF"; exit 0; } | |
| pnpm run build || { echo "Baseline build failed"; git checkout "$CURRENT_REF"; exit 0; } | |
| # Copy built .d.ts files to baseline directory | |
| if [ -d "js/dist" ]; then | |
| mkdir -p "$BASELINE_DIR/js/dist" | |
| cp js/dist/*.d.ts "$BASELINE_DIR/js/dist/" 2>/dev/null || true | |
| cp js/dist/*.d.mts "$BASELINE_DIR/js/dist/" 2>/dev/null || true | |
| echo "Copied main/browser dist files" | |
| fi | |
| if [ -d "js/dev/dist" ]; then | |
| mkdir -p "$BASELINE_DIR/js/dev/dist" | |
| cp js/dev/dist/*.d.ts "$BASELINE_DIR/js/dev/dist/" 2>/dev/null || true | |
| cp js/dev/dist/*.d.mts "$BASELINE_DIR/js/dev/dist/" 2>/dev/null || true | |
| echo "Copied dev dist files" | |
| fi | |
| if [ -d "js/util/dist" ]; then | |
| mkdir -p "$BASELINE_DIR/js/util/dist" | |
| cp js/util/dist/*.d.ts "$BASELINE_DIR/js/util/dist/" 2>/dev/null || true | |
| cp js/util/dist/*.d.mts "$BASELINE_DIR/js/util/dist/" 2>/dev/null || true | |
| echo "Copied util dist files" | |
| fi | |
| # Return to original commit | |
| git checkout "$CURRENT_REF" | |
| # Set environment variable for test to find baseline | |
| echo "BASELINE_DIR=$BASELINE_DIR" >> $GITHUB_ENV | |
| echo "Baseline created at $BASELINE_DIR" | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Build current branch | |
| run: pnpm run build | |
| - name: Run API compatibility test | |
| env: | |
| BASELINE_DIR: ${{ env.BASELINE_DIR }} | |
| run: | | |
| cd js && pnpm test:api-compat | |
| smoke-tests-node: | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| integration: [otel-v1, span] | |
| os: [ubuntu-latest, windows-latest] | |
| node-version: [20, 22] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Backup original integration package manifests | |
| working-directory: js/smoke/tests/${{ matrix.integration }} | |
| run: | | |
| npm run backup | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install SDK artifacts for ${{ matrix.integration }} | |
| working-directory: js/smoke/tests/${{ matrix.integration }} | |
| shell: bash | |
| run: | | |
| # Remove package-lock.json to avoid version mismatch with downloaded artifacts | |
| rm -f package-lock.json | |
| if [ "${{ matrix.integration }}" != "otel-v1" ]; then | |
| npm install | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| else | |
| npm install --legacy-peer-deps | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| npx tsx ../../install-build.ts ../../../artifacts otel | |
| fi | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run smoke test for ${{ matrix.integration }} | |
| working-directory: js/smoke | |
| shell: bash | |
| env: | |
| BRAINTRUST_API_KEY: ${{ matrix.integration == 'otel-v1' && secrets.BRAINTRUST_API_KEY || '' }} | |
| BRAINTRUST_OTEL_COMPAT: ${{ matrix.integration == 'otel-v1' && secrets.BRAINTRUST_API_KEY || '' }} | |
| run: | | |
| ./run-tests.sh ${{ matrix.integration }} | |
| - name: Restore original package.json files | |
| if: always() | |
| working-directory: js/smoke/tests/${{ matrix.integration }} | |
| run: | | |
| npm run restore | |
| smoke-tests-jest: | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| node-version: [20, 22] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Backup original span-jest package manifests | |
| working-directory: js/smoke/tests/span-jest | |
| run: | | |
| npm run backup | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies and local build for Jest span smoke test | |
| working-directory: js/smoke/tests/span-jest | |
| shell: bash | |
| run: | | |
| # Remove package-lock.json to avoid version mismatch with downloaded artifacts | |
| rm -f package-lock.json | |
| npm install | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run Jest span smoke test | |
| working-directory: js/smoke | |
| shell: bash | |
| run: | | |
| ./run-tests.sh span-jest | |
| - name: Restore original span-jest package.json files | |
| if: always() | |
| working-directory: js/smoke/tests/span-jest | |
| run: | | |
| npm run restore | |
| smoke-tests-nextjs: | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| node-version: [20, 22] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Backup original package manifests | |
| working-directory: js/smoke/tests/nextjs-instrumentation | |
| run: | | |
| npm run backup | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies and local builds | |
| working-directory: js/smoke/tests/nextjs-instrumentation | |
| shell: bash | |
| run: | | |
| # Remove package-lock.json to avoid version mismatch with downloaded artifacts | |
| rm -f package-lock.json | |
| npm install --legacy-peer-deps | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| npx tsx ../../install-build.ts ../../../artifacts otel | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run Next.js smoke test | |
| working-directory: js/smoke | |
| shell: bash | |
| env: | |
| BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} | |
| run: | | |
| ./run-tests.sh nextjs-instrumentation | |
| - name: Restore original package.json files | |
| if: always() | |
| working-directory: js/smoke/tests/nextjs-instrumentation | |
| run: | | |
| npm run restore | |
| smoke-tests-cloudflare: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node-version: [20, 22] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Backup original package manifests | |
| working-directory: js/smoke/tests/cloudflare-worker | |
| run: | | |
| npm run backup | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies and local build | |
| working-directory: js/smoke/tests/cloudflare-worker | |
| shell: bash | |
| run: | | |
| # Remove package-lock.json to avoid version mismatch with downloaded artifacts | |
| rm -f package-lock.json | |
| npm install | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run Cloudflare Worker smoke test | |
| working-directory: js/smoke | |
| shell: bash | |
| run: | | |
| ./run-tests.sh cloudflare-worker | |
| - name: Restore original package.json files | |
| if: always() | |
| working-directory: js/smoke/tests/cloudflare-worker | |
| run: | | |
| npm run restore | |
| smoke-tests-browser: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node-version: [20, 22] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Backup original package manifests | |
| working-directory: js/smoke/tests/browser | |
| run: | | |
| npm run backup | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install dependencies and local build | |
| working-directory: js/smoke/tests/browser | |
| shell: bash | |
| run: | | |
| # Remove package-lock.json to avoid version mismatch with downloaded artifacts | |
| rm -f package-lock.json | |
| npm install | |
| npx tsx ../../install-build.ts ../../../artifacts braintrust | |
| - name: Install Playwright Chromium (with OS deps) | |
| working-directory: js/smoke/tests/browser | |
| shell: bash | |
| run: | | |
| npx playwright install --with-deps chromium | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run Browser (Playwright) smoke test | |
| working-directory: js/smoke | |
| shell: bash | |
| run: | | |
| ./run-tests.sh browser | |
| - name: Restore original package.json files | |
| if: always() | |
| working-directory: js/smoke/tests/browser | |
| run: | | |
| npm run restore | |
| smoke-tests-deno: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| node-version: [20, 22] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Deno | |
| uses: denoland/setup-deno@v2 | |
| with: | |
| deno-version: v2.x | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.artifact-name }}-${{ matrix.node-version }}-dist | |
| path: js/artifacts | |
| run-id: ${{ github.run_id }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract build artifact | |
| working-directory: js/smoke/tests/deno | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| ARTIFACT_DIR=../../../artifacts | |
| TARBALL=$(ls "$ARTIFACT_DIR"/braintrust-*.tgz | head -n 1 || true) | |
| if [ -z "$TARBALL" ]; then | |
| echo "No braintrust build found" | |
| exit 1 | |
| fi | |
| rm -rf build | |
| mkdir -p build | |
| tar -xzf "$TARBALL" -C build | |
| [ -d build/package ] && mv build/package build/braintrust | |
| - name: Install Deno dependencies | |
| working-directory: js/smoke/tests/deno | |
| run: | | |
| deno install | |
| - name: Build shared test package | |
| working-directory: js/smoke/shared | |
| run: | | |
| npm ci | |
| npm run build | |
| - name: Run Deno smoke test | |
| working-directory: js/smoke | |
| env: | |
| BRAINTRUST_BUILD_DIR: ${{ github.workspace }}/js/smoke/tests/deno/build/braintrust/dist/browser.mjs | |
| run: | | |
| ./run-tests.sh deno |