ci: introduce benchmark comparison workflow

This is a workflow which can be triggered by commenting "\benchmark" on PRs (or through manual dispatch).

It uses the `benchmark_compare.py` to run and compare benchmarks. The comparison result is posted as a comment on the PR.

Author: ajay-mk

.github/workflows/benchmark_compare.yml

@@ -0,0 +1,195 @@
+name: Compare Benchmarks
+
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to run benchmarks for'
+        required: true
+        type: number
+
+jobs:
+  benchmark:
+    if: >
+      (github.event_name == 'issue_comment' 
+        && github.event.issue.pull_request 
+        && contains(github.event.comment.body, '/benchmark') 
+        && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+      ) || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+      statuses: write
+
+    steps:
+      - name: Get PR details
+        id: pr
+        uses: actions/github-script@v7
+        with:
+          script: |
+            let pullRequestNumber;
+            
+            if (context.eventName === 'workflow_dispatch') {
+              // Manual dispatch - use provided PR number
+              pullRequestNumber = parseInt('${{ github.event.inputs.pr_number }}');
+            } else {
+              // Issue comment trigger - use issue number
+              pullRequestNumber = context.issue.number;
+            }
+            
+            // Fetch PR details
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: pullRequestNumber,
+            });
+            
+            const is_fork = pullRequest.head.repo.full_name !== pullRequest.base.repo.full_name;    
+            return {
+              base_sha: pullRequest.base.sha,
+              head_sha: pullRequest.head.sha,
+              base_ref: pullRequest.base.ref,
+              head_ref: pullRequest.head.ref,
+              head_repo_full_name: pullRequest.head.repo.full_name,
+              base_repo_full_name: pullRequest.base.repo.full_name,
+              is_fork: is_fork,
+              pr_number: pullRequestNumber
+            };
+
+      - name: Create status check
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'pending',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Running benchmark comparison...',
+              context: 'comparing benchmarks'
+            });
+
+      - name: Checkout base repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Add fork as remote and fetch (if needed)
+        if: fromJson(steps.pr.outputs.result).is_fork
+        run: |
+          echo "PR is from fork: ${{ fromJson(steps.pr.outputs.result).head_repo_full_name }}"
+          git remote add fork https://github.com/${{ fromJson(steps.pr.outputs.result).head_repo_full_name }}.git
+          git fetch fork
+
+      - name: Verify commits
+        run: |
+          echo "=== Git Remotes ==="
+          git remote -v
+          echo "=== Base commit ==="
+          echo "Base SHA: ${{ fromJson(steps.pr.outputs.result).base_sha }}"
+          git log --oneline -1 ${{ fromJson(steps.pr.outputs.result).base_sha }} 2>/dev/null || echo "Base commit not found locally"
+          echo "=== Head commit ==="
+          echo "Head SHA: ${{ fromJson(steps.pr.outputs.result).head_sha }}"
+          git log --oneline -1 ${{ fromJson(steps.pr.outputs.result).head_sha }} 2>/dev/null || echo "Head commit not found locally"
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake ninja-build libboost-dev libboost-locale-dev libboost-random-dev libboost-regex-dev libeigen3-dev libmimalloc-dev
+
+      - name: Run Benchmark Script
+        working-directory: ${{ github.workspace }}
+        run: python3 bin/admin/benchmark_compare.py ${{ fromJson(steps.pr.outputs.result).base_sha }} ${{ fromJson(steps.pr.outputs.result).head_sha }}
+
+      - name: Upload Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-data
+          path: |
+            ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).base_sha }}-results.json
+            ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).head_sha }}-results.json
+            ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).base_sha }}-${{ fromJson(steps.pr.outputs.result).head_sha }}-comparison.txt
+          retention-days: 30
+
+      - name: Post Results
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const prData = ${{ steps.pr.outputs.result }};
+            const comparisonFile = `${prData.base_sha}-${prData.head_sha}-comparison.txt`;
+            const comparisonPath = path.join('${{ github.workspace }}', comparisonFile);
+
+            let comparisonResults = '';
+            try {
+              comparisonResults = fs.readFileSync(comparisonPath, 'utf8');
+            } catch (error) {
+              comparisonResults = 'Error reading comparison results: ' + error.message;
+            }
+
+            const commentBody = `## Benchmark Comparison Results
+
+            \`\`\`
+            ${comparisonResults}
+            \`\`\`
+
+            #### [Full benchmark data](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            `;
+
+            github.rest.issues.createComment({
+              issue_number: prData.pr_number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: commentBody
+            });
+
+      - name: Update status check - Success
+        if: success()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'success',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Benchmark comparison completed successfully',
+              context: 'comparing benchmarks'
+            });
+
+      - name: Notify on Failure
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+            
+            // Update status check to failed
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'failure',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Benchmark comparison failed',
+              context: 'comparing benchmarks'
+            });
+            
+            // Post failure comment
+            github.rest.issues.createComment({
+              issue_number: prData.pr_number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Benchmark comparison failed. Check the workflow logs for details.'
+            });