Performance Benchmarks #74

Workflow file for this run

.github/workflows/benchmark.yml at 6841ac3

	name: Performance Benchmarks

	on:
	schedule:
	# Run benchmarks daily at 2 AM UTC
	- cron: "0 2 * * *"
	push:
	branches: [main]
	paths:
	- "src/**"
	- "package.json"
	workflow_dispatch:

	env:
	NODE_VERSION: "20.x"

	jobs:
	benchmark:
	name: Run Performance Benchmarks
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	node: ["18.x", "20.x"]

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Node.js ${{ matrix.node }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node }}

	- name: Setup pnpm
	uses: pnpm/action-setup@v2
	with:
	version: 8

	- name: Install dependencies
	run: pnpm install

	- name: Run TPS benchmark
	run: pnpm run benchmark:tps

	- name: Run Latency benchmark
	run: pnpm run benchmark:latency

	- name: Run Concurrent Load benchmark
	run: pnpm run benchmark:concurrent

	- name: Run Full benchmark suite
	run: pnpm run benchmark

	- name: Upload benchmark results
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-${{ matrix.os }}-node${{ matrix.node }}
	path: benchmark-reports/
	retention-days: 90

	- name: Check benchmark thresholds
	run: \|
	node -e "
	const fs = require('fs');
	const report = JSON.parse(fs.readFileSync('benchmark-reports/latest.json', 'utf8'));

	console.log('\\n📊 Benchmark Results:');
	console.log('=====================\\n');

	let passed = true;

	report.results.forEach(result => {
	console.log(\`\${result.testName}:\`);
	console.log(\` TPS: \${result.tps.toFixed(2)}\`);
	console.log(\` P95 Latency: \${result.latencies.p95.toFixed(2)}ms\`);
	console.log(\` Success Rate: \${result.successRate.toFixed(2)}%\\n\`);

	// More lenient thresholds for CI (especially for Windows)
	if (result.testName === 'TPS Benchmark' && result.tps < 1000) {
	console.error('❌ FAILED: TPS below 1,000');
	passed = false;
	}

	if (result.testName === 'Latency Benchmark' && result.latencies.p95 > 500) {
	console.error('❌ FAILED: P95 latency above 500ms');
	passed = false;
	}

	if (result.successRate < 95) {
	console.error('❌ FAILED: Success rate below 95%');
	passed = false;
	}
	});

	if (!passed) {
	console.error('\\n❌ Benchmark thresholds not met');
	process.exit(1);
	}

	console.log('\\n✅ All benchmark thresholds met!');
	"

	compare-benchmarks:
	name: Compare with Baseline
	runs-on: ubuntu-latest
	needs: benchmark
	if: github.event_name == 'push'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: ${{ env.NODE_VERSION }}

	- name: Setup pnpm
	uses: pnpm/action-setup@v2
	with:
	version: 8

	- name: Download current benchmark
	uses: actions/download-artifact@v4
	with:
	name: benchmark-ubuntu-latest-node20.x
	path: current-benchmark/

	- name: Download baseline benchmark
	uses: dawidd6/action-download-artifact@v3
	with:
	workflow: benchmark.yml
	name: benchmark-ubuntu-latest-node20.x
	path: baseline-benchmark/
	if_no_artifact_found: warn

	- name: Compare benchmarks
	run: \|
	node -e "
	const fs = require('fs');

	if (!fs.existsSync('baseline-benchmark/latest.json')) {
	console.log('No baseline found, setting current as baseline');
	process.exit(0);
	}

	const current = JSON.parse(fs.readFileSync('current-benchmark/latest.json', 'utf8'));
	const baseline = JSON.parse(fs.readFileSync('baseline-benchmark/latest.json', 'utf8'));

	console.log('\\n📊 Benchmark Comparison:');
	console.log('=========================\\n');

	current.results.forEach((curr, idx) => {
	const base = baseline.results[idx];

	const tpsDiff = ((curr.tps - base.tps) / base.tps * 100).toFixed(2);
	const latencyDiff = ((curr.latencies.p95 - base.latencies.p95) / base.latencies.p95 * 100).toFixed(2);

	console.log(\`\${curr.testName}:\`);
	console.log(\` TPS: \${curr.tps.toFixed(2)} (\${tpsDiff > 0 ? '+' : ''}\${tpsDiff}%)\`);
	console.log(\` P95: \${curr.latencies.p95.toFixed(2)}ms (\${latencyDiff > 0 ? '+' : ''}\${latencyDiff}%)\\n\`);
	});
	"

	publish-results:
	name: Publish Benchmark Results
	runs-on: ubuntu-latest
	needs: [benchmark, compare-benchmarks]
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: ${{ env.NODE_VERSION }}

	- name: Setup pnpm
	uses: pnpm/action-setup@v2
	with:
	version: 8

	- name: Download benchmark results
	uses: actions/download-artifact@v4
	with:
	name: benchmark-ubuntu-latest-node20.x
	path: benchmark-reports/

	- name: Deploy to GitHub Pages
	uses: peaceiris/actions-gh-pages@v3
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	publish_dir: ./benchmark-reports
	destination_dir: benchmarks
	keep_files: true

	- name: Create benchmark badge
	run: \|
	npm install -g badge-maker

	TPS=$(node -e "
	const report = require('./benchmark-reports/latest.json');
	const tpsBenchmark = report.results.find(r => r.testName === 'TPS Benchmark');
	console.log(Math.round(tpsBenchmark.tps));
	")

	P95=$(node -e "
	const report = require('./benchmark-reports/latest.json');
	const latencyBenchmark = report.results.find(r => r.testName === 'Latency Benchmark');
	console.log(Math.round(latencyBenchmark.latencies.p95));
	")

	badge "TPS" "${TPS}+" "brightgreen" > tps-badge.svg
	badge "P95 Latency" "${P95}ms" "blue" > latency-badge.svg

	- name: Commit badges
	run: \|
	git config --local user.email "action@github.com"
	git config --local user.name "GitHub Action"
	git add *-badge.svg
	git diff --quiet && git diff --staged --quiet \|\| git commit -m "Update benchmark badges [skip ci]"
	git push

Navigation Menu

Performance Benchmarks #74

Workflow file

Performance Benchmarks #74

Uh oh!

Workflow file for this run