Skip to content

Commit 630740b

Browse files
Davin Hillsclaude
andcommitted
Fix Google provider for large specs and improve JSON robustness
- google.go: set ResponseMIMEType=application/json to suppress code fence wrapping; create genai.Client per-call using caller context with defer Close() - llm.go: handle orphaned opening fence (truncated fenced responses) in stripMarkdownFences; add fixInvalidJSONEscapes sanitizer for \d/\w/\s regex patterns LLMs embed unescaped in JSON strings - main.go: update default Google model to gemini-2.5-flash (65k output tokens vs 8k for gemini-2.0-flash, required for large spec/plan files) - README.md: update default model reference for Google provider Smoke tested all three providers against the full specs/SPEC.md + specs/PLAN.md: anthropic DRIFT_DETECTED score=78 (134s) openai VIOLATION score=60 ( 10s) google VIOLATION score=73 (117s) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 3736ecc commit 630740b

4 files changed

Lines changed: 42 additions & 8 deletions

File tree

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ realitycheck check [path] [flags]
7474
--strict No inferred intent; escalate drift severities
7575
--fail-on <verdict> Exit 2 if verdict >= level (ALIGNED|PARTIALLY_ALIGNED|DRIFT_DETECTED|VIOLATION)
7676
--severity-threshold <s> Filter output to findings at or above INFO|WARN|CRITICAL
77-
--model <id> Model ID (default: claude-opus-4-6 / gpt-4o / gemini-2.0-flash per provider)
77+
--model <id> Model ID (default: claude-opus-4-6 / gpt-4o / gemini-2.5-flash per provider)
7878
--offline Skip API key pre-flight check
7979
--verbose Print execution trace to stderr
8080
--debug Dump assembled prompt to stderr

cmd/realitycheck/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ func defaultModelForProvider(provider string) string {
416416
case "openai":
417417
return "gpt-4o"
418418
case "google":
419-
return "gemini-2.0-flash"
419+
return "gemini-2.5-flash"
420420
default:
421421
return "claude-opus-4-6"
422422
}

internal/llm/google.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ func (p *googleProvider) Complete(
4747
m.MaxOutputTokens = &maxOut
4848
temp32 := float32(temperature)
4949
m.Temperature = &temp32
50+
// Force JSON output mode to prevent the model from wrapping the response
51+
// in markdown code fences.
52+
m.ResponseMIMEType = "application/json"
5053

5154
resp, err := m.GenerateContent(ctx, genai.Text(userPrompt))
5255
if err != nil {

internal/llm/llm.go

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -126,13 +126,24 @@ func needsRepair(errs []ValidationError) bool {
126126
// `.*?` (not `.+?`) to allow empty bodies inside fences.
127127
var fenceRe = regexp.MustCompile("(?s)^(?:`{3}|~{3})[^\\n]*\\n(.*?)(?:`{3}|~{3})\\s*$")
128128

129+
// openFenceRe matches only an opening fence line (no closing fence required).
130+
// Used to strip orphaned opening fences from truncated responses.
131+
var openFenceRe = regexp.MustCompile("^(?:`{3}|~{3})[^\\n]*\\n")
132+
129133
// stripMarkdownFences removes leading/trailing markdown code fences that LLMs
130134
// sometimes wrap around JSON output (e.g., "```json\n...\n```").
135+
// If only an opening fence is present (e.g., the response was truncated before
136+
// the closing fence), the opening line is stripped so that the JSON content can
137+
// still be parsed.
131138
func stripMarkdownFences(s string) string {
132139
s = strings.TrimSpace(s)
133140
if m := fenceRe.FindStringSubmatch(s); m != nil {
134141
return strings.TrimSpace(m[1])
135142
}
143+
// Handle truncated fenced responses: strip the opening fence line only.
144+
if loc := openFenceRe.FindStringIndex(s); loc != nil {
145+
return strings.TrimSpace(s[loc[1]:])
146+
}
136147
return s
137148
}
138149

@@ -147,14 +158,21 @@ func ValidateResponse(raw string, index codeindex.Index) (*schema.PartialReport,
147158

148159
raw = stripMarkdownFences(raw)
149160

150-
// 1. JSON parse.
161+
// 1. JSON parse. If parsing fails due to invalid escape sequences (common
162+
// when LLM output includes regex patterns like \d+ inside JSON strings),
163+
// attempt a one-shot sanitization before giving up.
151164
var report schema.PartialReport
152165
if err := json.Unmarshal([]byte(raw), &report); err != nil {
153-
errs = append(errs, ValidationError{
154-
Field: "json_parse",
155-
Message: err.Error(),
156-
})
157-
return nil, errs
166+
fixed := fixInvalidJSONEscapes(raw)
167+
if err2 := json.Unmarshal([]byte(fixed), &report); err2 != nil {
168+
errs = append(errs, ValidationError{
169+
Field: "json_parse",
170+
Message: err.Error(),
171+
})
172+
return nil, errs
173+
}
174+
// Sanitized successfully; continue with the fixed payload.
175+
raw = fixed
158176
}
159177

160178
// 2. Required field check.
@@ -207,6 +225,19 @@ var (
207225
violationIDRe = regexp.MustCompile(`^VIOLATION-\d+$`)
208226
)
209227

228+
// invalidJSONEscapeRe matches a backslash followed by any character that is not
229+
// a valid JSON string escape character ("\/bfnrtu). LLMs sometimes emit regex
230+
// patterns (e.g. \d+, \w+) unescaped inside JSON strings; this sanitizer
231+
// converts them to properly double-escaped sequences (\\d, \\w, etc.) so that
232+
// the JSON parser accepts the response.
233+
var invalidJSONEscapeRe = regexp.MustCompile(`\\([^"\\/bfnrtu])`)
234+
235+
// fixInvalidJSONEscapes replaces invalid JSON escape sequences in s with their
236+
// correctly double-escaped equivalents.
237+
func fixInvalidJSONEscapes(s string) string {
238+
return invalidJSONEscapeRe.ReplaceAllString(s, `\\$1`)
239+
}
240+
210241
// validateEnums checks that all enum fields contain valid constants.
211242
func validateEnums(r *schema.PartialReport) []ValidationError {
212243
var errs []ValidationError

0 commit comments

Comments
 (0)