Added a human-in-the-loop example

crossword_companion/lib/services/gemini_service.dart

@@ -11,6 +11,7 @@ import 'package:firebase_ai/firebase_ai.dart';
 import 'package:flutter/foundation.dart';
 import 'package:http/http.dart' as http;
 import 'package:image_picker/image_picker.dart';
+import 'package:mime/mime.dart';
 
 import '../models/clue.dart';
 import '../models/clue_answer.dart';
@@ -37,6 +38,7 @@ class GeminiService {
         Tool.functionDeclarations([
           _getWordMetadataFunction,
           _returnResultFunction,
+          _resolveConflictFunction,
         ]),
       ],
     );
@@ -76,6 +78,24 @@ class GeminiService {
     },
   );
 
+  static final _resolveConflictFunction = FunctionDeclaration(
+    'resolveConflict',
+    'Asks the user to resolve a conflict between the letter pattern and the '
+        'proposed answer. Use this BEFORE calling returnResult if the answer you '
+        'want to propose does not match the letter pattern.',
+    parameters: {
+      'proposedAnswer': Schema(
+        SchemaType.string,
+        description: 'The answer the LLM wants to suggest.',
+      ),
+      'pattern': Schema(
+        SchemaType.string,
+        description: 'The current letter pattern from the grid.',
+      ),
+      'clue': Schema(SchemaType.string, description: 'The clue text.'),
+    },
+  );
+
   static String get clueSolverSystemInstruction =>
       '''
 You are an expert crossword puzzle solver.
@@ -85,8 +105,9 @@ You are an expert crossword puzzle solver.
 2.  **Match the Clue:** Ensure your answer strictly matches the clue's tense, plurality (singular vs. plural), and part of speech.
 3.  **Verify Grammatically:** If a clue implies a specific part of speech (e.g., it's a verb, adverb, or plural), it's a good idea to use the `getWordMetadata` tool to verify your candidate answer matches. However, avoid using it for every clue.
 4.  **Be Confident:** Provide a confidence score from 0.0 to 1.0 indicating your certainty.
-5.  **Trust the Clue Over the Pattern:** The provided letter pattern is only a suggestion based on other potentially incorrect answers. Your primary goal is to find the best word that fits the **clue text**. If you are confident in an answer that contradicts the provided pattern, you should use that answer.
-6.  **Format Correctly:** You must return your answer in the specified JSON format.
+5.  **Trust the Clue Over the Pattern:** The provided letter pattern is only a suggestion based on other potentially incorrect answers. Your primary goal is to find the best word that fits the **clue text**.
+6.  **Resolve Conflicts:** If the answer you are confident in conflicts with the provided `pattern`, you **MUST** use the `resolveConflict` tool to ask the user for the correct answer. Use the result of `resolveConflict` as your final answer.
+7.  **Format Correctly:** You must return your answer in the specified JSON format.
 
 ---
 
@@ -98,13 +119,13 @@ You have a tool to get grammatical information about a word.
 - This tool is most helpful as a verification step after you have a likely answer.
 - Consider using this tool when a clue contains a grammatical hint that could be ambiguous.
 - **Good candidates for verification:**
-    - Clues that seem to be verbs (e.g., "To run," "Waving").
-    - Clues that are adverbs (e.g., "Happily," "Quickly").
-    - Clues that specify a plural form.
+- Clues that seem to be verbs (e.g., "To run," "Waving").
+- Clues that are adverbs (e.g., "Happily," "Quickly").
+- Clues that specify a plural form.
 - **Try to avoid using the tool for:**
-    - Simple definitions (e.g., "A small dog").
-    - Fill-in-the-blank clues (e.g., "___ and flow").
-    - Proper nouns (e.g., "Capital of France").
+- Simple definitions (e.g., "A small dog").
+- Fill-in-the-blank clues (e.g., "___ and flow").
+- Proper nouns (e.g., "Capital of France").
 
 **Function signature:**
 ```json
@@ -117,12 +138,26 @@ You have a tool to return the final result of the clue solving process.
 
 **When to use:**
 - Use this tool when you have a final answer and confidence score to return. You
-  must use this tool exactly once, and only once, to return the final result.
+must use this tool exactly once, and only once, to return the final result.
 
 **Function signature:**
 ```json
 ${jsonEncode(_returnResultFunction.toJson())}
 ```
+
+### Tool: `resolveConflict`
+
+You have a tool to ask the user to resolve a conflict.
+
+**When to use:**
+- Use this tool **BEFORE** `returnResult` if your proposed answer conflicts with the provided letter pattern.
+- For example, if the pattern is `_ R _ Y` and you want to suggest `RENT` (which fits the clue), there is a conflict at the second letter (`R` vs `E`). You should call `resolveConflict(proposedAnswer: "RENT", pattern: "_ R _ Y", clue: "...")`.
+- The tool will return the user's decision (either your proposed answer or a new one). You should then use that result to call `returnResult`.
+
+**Function signature:**
+```json
+${jsonEncode(_resolveConflictFunction.toJson())}
+```
 ''';
 
   static final _crosswordSchema = Schema(
@@ -175,7 +210,8 @@ ${jsonEncode(_returnResultFunction.toJson())}
     final imageParts = <Part>[];
     for (final image in images) {
       final imageBytes = await image.readAsBytes();
-      imageParts.add(InlineDataPart('image/jpeg', imageBytes));
+      final mimeType = lookupMimeType(image.path, headerBytes: imageBytes)!;
+      imageParts.add(InlineDataPart(mimeType, imageBytes));
     }
 
     final content = [
@@ -186,7 +222,7 @@ representing the grid size, contents, and clues. The images may contain
 different parts of the same puzzle (e.g., the grid the across clues, the down
 clues). Combine them to form a complete puzzle.
 The JSON schema is as follows: ${jsonEncode(_crosswordSchema.toJson())}
-          '''),
+      '''),
         ...imageParts,
       ]),
     ];
@@ -242,7 +278,13 @@ The JSON schema is as follows: ${jsonEncode(_crosswordSchema.toJson())}
   // Buffer for the result of the clue solving process.
   final _returnResult = <String, dynamic>{};
 
-  Future<ClueAnswer?> solveClue(Clue clue, int length, String pattern) async {
+  Future<ClueAnswer?> solveClue(
+    Clue clue,
+    int length,
+    String pattern, {
+    Future<String> Function(String clue, String proposedAnswer, String pattern)?
+    onConflict,
+  }) async {
     // Cancel any previous, in-flight request.
     await cancelCurrentSolve();
 
@@ -257,6 +299,10 @@ The JSON schema is as follows: ${jsonEncode(_crosswordSchema.toJson())}
           functionCall.args['word'] as String,
         ),
         'returnResult' => _cacheReturnResult(functionCall.args),
+        'resolveConflict' => await _handleResolveConflict(
+          functionCall.args,
+          onConflict,
+        ),
         _ => throw Exception('Unknown function call: ${functionCall.name}'),
       },
     );
@@ -291,10 +337,24 @@ The JSON schema is as follows: ${jsonEncode(_crosswordSchema.toJson())}
     return {'status': 'success'};
   }
 
-  String getSolverPrompt(Clue clue, int length, String pattern) =>
-      buildSolverPrompt(clue, length, pattern);
+  Future<Map<String, dynamic>> _handleResolveConflict(
+    Map<String, dynamic> args,
+    Future<String> Function(String clue, String proposedAnswer, String pattern)?
+    onConflict,
+  ) async {
+    final proposedAnswer = args['proposedAnswer'] as String;
+    final pattern = args['pattern'] as String;
+    final clue = args['clue'] as String;
+
+    if (onConflict != null) {
+      final result = await onConflict(clue, proposedAnswer, pattern);
+      return {'result': result};
+    }
 
-  String buildSolverPrompt(Clue clue, int length, String pattern) =>
+    return {'result': proposedAnswer};
+  }
+
+  String getSolverPrompt(Clue clue, int length, String pattern) =>
       '''
 Your task is to solve the following crossword clue.
 

crossword_companion/lib/services/puzzle_solver.dart

@@ -15,6 +15,8 @@ class PuzzleSolver {
     PuzzleDataState dataState,
     GeminiService geminiService, {
     bool isResuming = false,
+    Future<String> Function(String clue, String proposedAnswer, String pattern)?
+    onConflict,
   }) async {
     assert(
       solverState.todos.isNotEmpty,
@@ -47,6 +49,7 @@ class PuzzleSolver {
           clue,
           expectedLength,
           pattern,
+          onConflict: onConflict,
         );
 
         if (!solverState.isSolving) break;

crossword_companion/lib/state/puzzle_solver_state.dart

@@ -94,11 +94,16 @@ class PuzzleSolverState with ChangeNotifier {
     unawaited(solvePuzzle());
   }
 
-  Future<void> solvePuzzle({bool isResuming = false}) => _puzzleSolver.solve(
+  Future<void> solvePuzzle({
+    bool isResuming = false,
+    Future<String> Function(String clue, String proposedAnswer, String pattern)?
+    onConflict,
+  }) => _puzzleSolver.solve(
     this,
     _puzzleDataState,
     _geminiService,
     isResuming: isResuming,
+    onConflict: onConflict,
   );
 
   void resetSolution() {

crossword_companion/lib/widgets/conflict_dialog.dart

@@ -0,0 +1,77 @@
+// Copyright 2025 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/material.dart';
+
+class ConflictDialog extends StatefulWidget {
+  const ConflictDialog({
+    required this.clue,
+    required this.pattern,
+    required this.proposedAnswer,
+    super.key,
+  });
+
+  final String clue;
+  final String pattern;
+  final String proposedAnswer;
+
+  @override
+  State<ConflictDialog> createState() => _ConflictDialogState();
+}
+
+class _ConflictDialogState extends State<ConflictDialog> {
+  late final TextEditingController _controller;
+
+  @override
+  void initState() {
+    super.initState();
+    _controller = TextEditingController(text: widget.proposedAnswer);
+  }
+
+  @override
+  void dispose() {
+    _controller.dispose();
+    super.dispose();
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    return AlertDialog(
+      title: const Text('Conflict Detected'),
+      content: Column(
+        mainAxisSize: MainAxisSize.min,
+        crossAxisAlignment: CrossAxisAlignment.start,
+        children: [
+          Text('Clue: ${widget.clue}'),
+          const SizedBox(height: 8),
+          Text('Pattern: ${widget.pattern}'),
+          const SizedBox(height: 16),
+          TextField(
+            controller: _controller,
+            decoration: const InputDecoration(
+              labelText: 'Answer',
+              border: OutlineInputBorder(),
+            ),
+            autofocus: true,
+          ),
+        ],
+      ),
+      actions: [
+        TextButton(
+          onPressed: () {
+            // Return the original proposed answer if canceled
+            Navigator.pop(context, widget.proposedAnswer);
+          },
+          child: const Text('Cancel'),
+        ),
+        TextButton(
+          onPressed: () {
+            Navigator.pop(context, _controller.text);
+          },
+          child: const Text('OK'),
+        ),
+      ],
+    );
+  }
+}

crossword_companion/lib/widgets/step5_solve_puzzle.dart

@@ -13,6 +13,7 @@ import '../state/app_step_state.dart';
 import '../state/puzzle_data_state.dart';
 import '../state/puzzle_solver_state.dart';
 import '../styles.dart';
+import 'conflict_dialog.dart';
 import 'grid_view.dart';
 import 'step_activation_mixin.dart';
 import 'todo_list_widget.dart';
@@ -41,7 +42,12 @@ class _StepFiveSolvePuzzleState extends State<StepFiveSolvePuzzle>
     // Start solving only if we are not already solving and there are todos.
     if (!puzzleSolverState.isSolving &&
         puzzleSolverState.todos.any((t) => t.status != TodoStatus.done)) {
-      unawaited(puzzleSolverState.solvePuzzle());
+      unawaited(
+        puzzleSolverState.solvePuzzle(
+          onConflict: (clue, proposedAnswer, pattern) =>
+              _showConflictDialog(context, clue, proposedAnswer, pattern),
+        ),
+      );
     }
   }
 
@@ -101,6 +107,24 @@ class _StepFiveSolvePuzzleState extends State<StepFiveSolvePuzzle>
     );
   }
 
+  Future<String> _showConflictDialog(
+    BuildContext context,
+    String clue,
+    String proposedAnswer,
+    String pattern,
+  ) async {
+    final result = await showDialog<String>(
+      context: context,
+      barrierDismissible: false,
+      builder: (context) => ConflictDialog(
+        clue: clue,
+        pattern: pattern,
+        proposedAnswer: proposedAnswer,
+      ),
+    );
+    return result ?? proposedAnswer;
+  }
+
   @override
   Widget build(BuildContext context) {
     final puzzleDataState = Provider.of<PuzzleDataState>(context);
@@ -134,7 +158,7 @@ class _StepFiveSolvePuzzleState extends State<StepFiveSolvePuzzle>
                         (t) => t.status != TodoStatus.done,
                       ))
                     ElevatedButton(
-                      onPressed: puzzleSolverState.resumeSolving,
+                      onPressed: () => puzzleSolverState.resumeSolving(),
                       child: const Text('Resume'),
                     ),
                   ElevatedButton(