Merge pull request #38 from pinchtab/chore/benchmark-improvements

chore: expand benchmark corpus and tuning tools

Author: luigi-agosti

cmd/semantic/main.go

@@ -56,6 +56,8 @@ Flags (find/match):
   --threshold <n>     Minimum score (default: 0.3)
   --top-k <n>         Max results (default: 3)
   --strategy <name>   lexical, embedding, or combined (default: combined)
+  --lexical-weight <n>   Combined strategy lexical weight override
+  --embedding-weight <n> Combined strategy embedding weight override
   --format <fmt>      json, table, or refs (default: table)
 `)
 }
@@ -209,6 +211,8 @@ func runFind(args []string) {
 	threshold := fs.Float64("threshold", 0.3, "minimum score")
 	topK := fs.Int("top-k", 3, "max results")
 	strategy := fs.String("strategy", "combined", "matching strategy")
+	lexicalWeight := fs.Float64("lexical-weight", 0, "combined strategy lexical weight override")
+	embeddingWeight := fs.Float64("embedding-weight", 0, "combined strategy embedding weight override")
 	format := fs.String("format", "table", "output format: json, table, refs")
 	_ = fs.Parse(args)
 
@@ -226,8 +230,10 @@ func runFind(args []string) {
 
 	matcher := newMatcher(*strategy)
 	result, err := matcher.Find(context.Background(), query, elements, semantic.FindOptions{
-		Threshold: *threshold,
-		TopK:      *topK,
+		Threshold:       *threshold,
+		TopK:            *topK,
+		LexicalWeight:   *lexicalWeight,
+		EmbeddingWeight: *embeddingWeight,
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)

docs/reference/cli.md

@@ -16,6 +16,8 @@ semantic find <query> [flags]
 | `--threshold` | 0.3 | Minimum score |
 | `--top-k` | 3 | Maximum results |
 | `--strategy` | combined | `combined`, `lexical`, or `embedding` |
+| `--lexical-weight` | 0 | Combined strategy lexical weight override |
+| `--embedding-weight` | 0 | Combined strategy embedding weight override |
 | `--format` | table | `table`, `json`, or `refs` |
 
 **Examples:**
@@ -31,6 +33,9 @@ curl -s localhost:9999/snapshot | semantic find "search box"
 # Machine-readable
 semantic find "login" --snapshot page.json --format json
 
+# Tune combined scoring
+semantic find "login" --snapshot page.json --lexical-weight 0.7 --embedding-weight 0.3
+
 # Just refs (for piping)
 semantic find "submit" --snapshot page.json --format refs
 

tests/benchmark/README.md

@@ -14,6 +14,9 @@ cd tests/benchmark
 ./scripts/run-corpus-benchmark.sh --strategy lexical
 ./scripts/run-corpus-benchmark.sh --strategy embedding
 ./scripts/run-corpus-benchmark.sh --strategy combined
+
+# Sweep combined lexical/embedding weights
+./scripts/tune-weights.sh
 ```
 
 ## Metrics
@@ -55,32 +58,35 @@ corpus/
 ## Current Results (combined strategy)
 
 ```
-Queries:     50
-MRR:         0.88
-P@1:         0.87
-P@3:         0.34
-Latency P50: 31 ms
-Latency P95: 52 ms
+Queries:     105
+MRR:         0.8897
+P@1:         0.8762
+P@3:         0.3412
+Latency P50: 23 ms
+Latency P95: 28 ms
 
 By Difficulty:
-  easy:   34 queries, P@1 = 0.95
-  medium: 14 queries, P@1 = 0.78
-  hard:    2 queries, P@1 = 0.00
+  easy:   76 queries, P@1 = 0.94
+  medium: 25 queries, P@1 = 0.74
+  hard:    4 queries, P@1 = 0.50
 ```
 
 ## Optimization Targets
 
-The 6 current misses are "hard" cases requiring:
+The current misses cluster around:
 - Synonym expansion (save for later → wishlist)
 - Implicit actions (clone → Code button)
 - Domain knowledge (CI status → Actions tab)
+- Form/input intent (type new query → search box)
+- Accessibility/navigation shortcuts (skip to content, homepage)
 
 ## Scripts
 
 | Script | Purpose |
 |--------|---------|
 | `run-corpus-benchmark.sh` | Main benchmark with MRR/P@K metrics |
 | `run-benchmark.sh` | Simple pass/fail test runner |
+| `tune-weights.sh` | Grid search combined matcher lexical/embedding weights |
 
 ## Adding to Corpus
 
@@ -95,6 +101,13 @@ The 6 current misses are "hard" cases requiring:
 
 3. Run benchmark to establish baseline
 
+4. Add several related queries for the same behavior, not one isolated case.
+   Include easy, medium, hard, and at least one near-miss or partial match where
+   ambiguity matters.
+
+5. Re-run `./scripts/tune-weights.sh` after larger corpus changes to see whether
+   the best combined weights moved.
+
 ## CI Integration
 
 ```yaml

tests/benchmark/cases/negative-threshold.json

@@ -0,0 +1,117 @@
+[
+  {
+    "id": "neg-001",
+    "query": "xyzzy plugh qwerty",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.3,
+    "tags": ["no-match", "nonsense"]
+  },
+  {
+    "id": "neg-002",
+    "query": "upload spreadsheet to cloud",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control"]
+  },
+  {
+    "id": "neg-003",
+    "query": "open video player",
+    "snapshot": "dashboard.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control"]
+  },
+  {
+    "id": "neg-004",
+    "query": "print receipt",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control"]
+  },
+  {
+    "id": "neg-005",
+    "query": "submit button",
+    "snapshot": "multi-form.json",
+    "expect_ref": "e11",
+    "expect_ref_alt": ["e3", "e7"],
+    "threshold": 0.3,
+    "min_score": 0.5,
+    "tags": ["threshold", "duplicate-labels"]
+  },
+  {
+    "id": "neg-006",
+    "query": "enter",
+    "snapshot": "login-page.json",
+    "expect_has_matches": true,
+    "threshold": 0.1,
+    "min_score": 0.15,
+    "tags": ["threshold", "weak-match"]
+  },
+  {
+    "id": "neg-007",
+    "query": "click",
+    "snapshot": "ecommerce-product.json",
+    "expect_has_matches": true,
+    "threshold": 0.1,
+    "tags": ["threshold", "generic-verb"]
+  },
+  {
+    "id": "neg-008",
+    "query": "the thing",
+    "snapshot": "dashboard.json",
+    "expect_has_matches": true,
+    "threshold": 0.05,
+    "tags": ["threshold", "vague-query"]
+  },
+  {
+    "id": "neg-009",
+    "query": "asdfghjkl",
+    "snapshot": "multi-form.json",
+    "expect_no_match": true,
+    "threshold": 0.3,
+    "tags": ["no-match", "keyboard-mash"]
+  },
+  {
+    "id": "neg-010",
+    "query": "stale element e999",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.3,
+    "tags": ["no-match", "stale-ref"]
+  },
+  {
+    "id": "neg-011",
+    "query": "a b c d e f",
+    "snapshot": "dashboard.json",
+    "expect_no_crash": true,
+    "threshold": 0.3,
+    "tags": ["threshold", "noise-tokens"]
+  },
+  {
+    "id": "neg-012",
+    "query": "configure webhook endpoint",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control", "domain-intent"]
+  },
+  {
+    "id": "neg-013",
+    "query": "invoice download",
+    "snapshot": "ecommerce-product.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control"]
+  },
+  {
+    "id": "neg-014",
+    "query": "share on twitter",
+    "snapshot": "login-page.json",
+    "expect_no_match": true,
+    "threshold": 0.4,
+    "tags": ["no-match", "absent-control"]
+  }
+]

tests/benchmark/cases/visual.json

@@ -1,46 +1,46 @@
 [
   {
-    "id": "visual-001",
+    "id": "vcase-001",
     "query": "button in top right",
     "snapshot": "visual-layout.json",
     "expect_ref": "e1",
     "min_score": 0.5,
     "tags": ["visual", "position", "directional"]
   },
   {
-    "id": "visual-002",
+    "id": "vcase-002",
     "query": "button on the left",
     "snapshot": "visual-layout.json",
     "expect_ref": "e0",
     "min_score": 0.4,
     "tags": ["visual", "position", "directional"]
   },
   {
-    "id": "visual-003",
+    "id": "vcase-003",
     "query": "button at bottom",
     "snapshot": "visual-layout.json",
     "expect_ref": "e7",
     "min_score": 0.4,
     "tags": ["visual", "position", "directional"]
   },
   {
-    "id": "visual-004",
+    "id": "vcase-004",
     "query": "link on left side",
     "snapshot": "visual-layout.json",
     "expect_ref": "e3",
     "min_score": 0.4,
     "tags": ["visual", "position", "directional"]
   },
   {
-    "id": "visual-005",
+    "id": "vcase-005",
     "query": "top left menu button",
     "snapshot": "visual-layout.json",
     "expect_ref": "e0",
     "min_score": 0.5,
     "tags": ["visual", "position", "compound"]
   },
   {
-    "id": "visual-006",
+    "id": "vcase-006",
     "query": "settings in upper right corner",
     "snapshot": "visual-layout.json",
     "expect_ref": "e1",

tests/benchmark/corpus/README.md

@@ -38,6 +38,30 @@ Each corpus entry is a directory containing:
 - **P@3**: How many of top-3 are relevant?
 - **Margin**: Score gap between relevant and irrelevant
 
+## Expansion Groups
+
+### Expansion 1: Complex Query Patterns (2026-04)
+
+Added corpora for underrepresented query types:
+
+- **implicit-domain-intent/**: GitHub-like repo page with 56 elements. Tests implicit intents like "clone this repo", "check CI status", "switch branch", "save for later". 18 queries, 8 hard.
+
+- **form-state-controls/**: Settings page with checkboxes, radios, toggles, comboboxes. Tests stateful controls like "keep me logged in", "enable 2FA", "subscribe to newsletter". 18 queries, 8 hard.
+
+- **ambiguous-layout-context/**: Multi-section page with duplicate labels (3x Search, 2x Save, 2x Cancel, 2x Login, 2x Home, 2x Help). Tests positional and section disambiguation. 17 queries, 7 hard.
+
+Also added `tests/benchmark/cases/negative-threshold.json` with 14 no-match and threshold calibration cases.
+
+### Expansion 2: Enterprise UI Patterns (2026-04)
+
+Added corpora for complex enterprise UI scenarios:
+
+- **table-grid/**: Invoice table with 50+ elements. Tests row-level context, repeated buttons (Edit, Delete, More), ordinal references ("second invoice", "last row"), and bulk operations. 24 queries, 8 hard.
+
+- **overlays-menus-dialogs/**: Multi-layer UI with modal dialogs, dropdown menus, context menus, notifications. Tests duplicate controls across scopes ("cancel in modal", "save on page not dialog"), menu item selection, and overlay disambiguation. 24 queries, 8 hard.
+
+- **icon-aria-labels/**: Icon-only controls across toolbar, media player, navigation. Tests sparse accessible names, icon descriptions ("kebab menu", "hamburger", "pencil edit"), and section context for repeated icons. 25 queries, 6 hard.
+
 ## Sources
 
 Snapshots should be captured from real websites using pinchtab: