Benchmark Comparison Workflow

.github/workflows/benchmark_compare.yml

@@ -0,0 +1,197 @@
+name: Compare Benchmarks
+
+on:
+  issue_comment:
+    types: [created]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'PR number to run benchmarks for'
+        required: true
+        type: number
+
+concurrency:
+  group: benchmark-${{ github.event.issue.number || github.event.inputs.pr_number }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    if: >
+      (github.event_name == 'issue_comment' 
+        && github.event.issue.pull_request 
+        && contains(github.event.comment.body, '/benchmark') 
+        && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+      ) || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+      statuses: write
+
+    steps:
+      - name: Get PR details
+        id: pr
+        uses: actions/github-script@v7
+        with:
+          script: |
+            let pullRequestNumber;
+
+            if (context.eventName === 'workflow_dispatch') {
+              // Manual dispatch - use provided PR number
+              pullRequestNumber = parseInt('${{ github.event.inputs.pr_number }}');
+            } else {
+              // PR comment trigger - use PR number
+              pullRequestNumber = context.issue.number;
+            }
+
+            // Fetch PR details
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: pullRequestNumber,
+            });
+
+            const is_fork = pullRequest.head.repo.full_name !== pullRequest.base.repo.full_name;    
+            return {
+              base_sha: pullRequest.base.sha,
+              head_sha: pullRequest.head.sha,
+              base_ref: pullRequest.base.ref,
+              head_ref: pullRequest.head.ref,
+              head_repo_full_name: pullRequest.head.repo.full_name,
+              base_repo_full_name: pullRequest.base.repo.full_name,
+              is_fork: is_fork,
+              pr_number: pullRequestNumber
+            };
+
+      - name: Create status check
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'pending',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Running benchmark comparison...',
+              context: 'comparing benchmarks'
+            });
+
+      - name: Checkout base repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Add fork as remote and fetch (if needed)
+        if: fromJson(steps.pr.outputs.result).is_fork
+        run: |
+          echo "PR is from fork: ${{ fromJson(steps.pr.outputs.result).head_repo_full_name }}"
+          git remote add fork https://github.com/${{ fromJson(steps.pr.outputs.result).head_repo_full_name }}.git
+          git fetch fork
+
+      - name: Verify commits
+        run: |
+          echo "=== Git Remotes ==="
+          git remote -v
+          echo "=== Base commit ==="
+          echo "${{ fromJson(steps.pr.outputs.result).base_sha }}"
+          echo "=== Head commit ==="
+          echo "${{ fromJson(steps.pr.outputs.result).head_sha }}"
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake ninja-build libboost-dev libboost-locale-dev libboost-random-dev libboost-regex-dev libeigen3-dev libmimalloc-dev
+
+      - name: Run Benchmark Script
+        working-directory: ${{ github.workspace }}
+        run: python3 bin/admin/benchmark_compare.py ${{ fromJson(steps.pr.outputs.result).base_sha }} ${{ fromJson(steps.pr.outputs.result).head_sha }}
+
+      - name: Upload Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-data
+          path: |
+            ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).base_sha }}-results.json
+            ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).head_sha }}-results.json
+            ${{ github.workspace }}/benchmark-comparison.txt
+          retention-days: 30
+
+      - name: Post Results
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const prData = ${{ steps.pr.outputs.result }};
+            const comparisonFile = `benchmark-comparison.txt`;
+            const comparisonPath = path.join('${{ github.workspace }}', comparisonFile);
+
+            let comparisonResults = '';
+            try {
+              comparisonResults = fs.readFileSync(comparisonPath, 'utf8');
+            } catch (error) {
+              comparisonResults = 'Error reading comparison results: ' + error.message;
+            }
+
+            const commentBody = `## Benchmark Comparison Results
+
+            \`\`\`
+            ${comparisonResults}
+            \`\`\`
+
+            #### [Full benchmark data](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+            `;
+
+            github.rest.issues.createComment({
+              issue_number: prData.pr_number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: commentBody
+            });
+
+      - name: Update status check - Success
+        if: success()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'success',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Benchmark comparison completed successfully',
+              context: 'comparing benchmarks'
+            });
+
+      - name: Notify on Failure
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const prData = ${{ steps.pr.outputs.result }};
+
+            // Update status check to failed
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: prData.head_sha,
+              state: 'failure',
+              target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+              description: 'Benchmark comparison failed',
+              context: 'comparing benchmarks'
+            });
+
+            // Post failure comment
+            github.rest.issues.createComment({
+              issue_number: prData.pr_number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'Benchmark comparison failed. Check the workflow logs for details.'
+            });

SeQuant/core/runtime.cpp

@@ -25,4 +25,28 @@ void set_locale() {
   std::ios_base::sync_with_stdio(true);
 }
 
+namespace {
+// see https://en.cppreference.com/w/cpp/locale/numpunct/thousands_sep
+class no_thousands_separator : public std::numpunct<char> {
+ protected:
+  char do_thousands_sep() const override { return '\0'; }
+  std::string do_grouping() const override { return ""; }
+};
+
+class no_thousands_separator_w : public std::numpunct<wchar_t> {
+ protected:
+  wchar_t do_thousands_sep() const override { return L'\0'; }
+  std::string do_grouping() const override { return ""; }
+};
+}  // namespace
+
+void disable_thousands_separator() {
+  auto current_locale = std::locale();
+
+  std::locale modified_locale(current_locale, new no_thousands_separator);
+  modified_locale = std::locale(modified_locale, new no_thousands_separator_w);
+
+  std::locale::global(modified_locale);
+}
+
 }  // namespace sequant

SeQuant/core/runtime.hpp

@@ -175,8 +175,13 @@ T transform_reduce(SizedRange&& rng, T init, const BinaryReductionOp& reduce,
 #endif
 }
 
+/// Set up the global locale settings
 void set_locale();
 
+/// @brief disables thousands separators in the current locale
+/// @note Need this in case of JSON/CSV output
+void disable_thousands_separator();
+
 }  // namespace sequant
 
 #endif  // SEQUANT_RUNTIME_HPP

benchmarks/CMakeLists.txt

@@ -17,7 +17,13 @@ add_executable(sequant_benchmarks
 set_target_properties(sequant_benchmarks PROPERTIES UNITY_BUILD OFF)
 
 target_link_libraries(sequant_benchmarks
-	PRIVATE
-		benchmark::benchmark
-		SeQuant::SeQuant
+        PRIVATE
+        benchmark::benchmark
+        SeQuant::SeQuant
+)
+
+# benchmark target to just run cc
+add_custom_target(sequant_benchmark_cc
+        COMMAND sequant_benchmarks --benchmark_filter="cc_full"
+        DEPENDS sequant_benchmarks
 )

benchmarks/main.cpp

@@ -12,6 +12,7 @@ int main(int argc, char *argv[]) {
   // Disable multithreading
   set_num_threads(1);
   set_locale();
+  disable_thousands_separator();
   auto idxreg = mbpt::make_sr_spaces();
   Context fermi_ctx = Context({.index_space_registry_shared_ptr = idxreg,
                                .vacuum = Vacuum::SingleProduct,