Parallelize AST extraction and rewrite Louvain clustering

src/Graphify.Cli/PipelineRunner.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using Graphify.Export;
 using Graphify.Graph;
 using Graphify.Models;
@@ -62,35 +63,47 @@ public PipelineRunner(TextWriter output, bool verbose = false, IChatClient? chat
             // Stage 2: Extract nodes and edges
             await WriteLineAsync("[2/6] Extracting code structure...");
             var extractor = new Extractor();
-            var extractionResults = new List<ExtractionResult>();
+            var extractionBag = new ConcurrentBag<ExtractionResult>();
             int processed = 0;
             int skipped = 0;
+            var verboseWarnings = new ConcurrentQueue<string>();
 
-            foreach (var file in detectedFiles)
-            {
-                cancellationToken.ThrowIfCancellationRequested();
-
-                try
+            await Parallel.ForEachAsync(
+                detectedFiles,
+                new ParallelOptions
                 {
-                    var result = await extractor.ExecuteAsync(file, cancellationToken);
-                    if (result.Nodes.Count > 0 || result.Edges.Count > 0)
-                    {
-                        extractionResults.Add(result);
-                        processed++;
-                    }
-                    else
+                    MaxDegreeOfParallelism = Environment.ProcessorCount,
+                    CancellationToken = cancellationToken
+                },
+                async (file, ct) =>
+                {
+                    try
                     {
-                        skipped++;
+                        var result = await extractor.ExecuteAsync(file, ct);
+                        if (result.Nodes.Count > 0 || result.Edges.Count > 0)
+                        {
+                            extractionBag.Add(result);
+                            Interlocked.Increment(ref processed);
+                        }
+                        else
+                        {
+                            Interlocked.Increment(ref skipped);
+                        }
                     }
-                }
-                catch (Exception ex)
-                {
-                    if (_verbose)
+                    catch (Exception ex)
                     {
-                        await WriteLineAsync($"      Warning: Failed to extract {file.RelativePath}: {ex.Message}");
+                        Interlocked.Increment(ref skipped);
+                        if (_verbose)
+                        {
+                            verboseWarnings.Enqueue($"      Warning: Failed to extract {file.RelativePath}: {ex.Message}");
+                        }
                     }
-                    skipped++;
-                }
+                });
+
+            var extractionResults = new List<ExtractionResult>(extractionBag);
+            foreach (var warning in verboseWarnings)
+            {
+                await WriteLineAsync(warning);
             }
 
             await WriteLineAsync($"      Processed {processed} files, skipped {skipped}");