diff --git a/packages/firebase_ai/firebase_ai/example/lib/main.dart b/packages/firebase_ai/firebase_ai/example/lib/main.dart index a7fd0363aba7..b8dbdaaec19c 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/main.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/main.dart @@ -154,10 +154,6 @@ class HomeScreen extends StatefulWidget { class _HomeScreenState extends State { void _onItemTapped(int index) { - if (index == 9 && !widget.useVertexBackend) { - // Live Stream feature only works with Vertex AI now. - return; - } widget.onSelectedIndexChanged(index); } @@ -192,12 +188,12 @@ class _HomeScreenState extends State { case 8: return VideoPage(title: 'Video Prompt', model: currentModel); case 9: - if (useVertexBackend) { - return BidiPage(title: 'Live Stream', model: currentModel); - } else { - // Fallback to the first page in case of an unexpected index - return ChatPage(title: 'Chat', model: currentModel); - } + return BidiPage( + title: 'Live Stream', + model: currentModel, + useVertexBackend: useVertexBackend, + ); + default: // Fallback to the first page in case of an unexpected index return ChatPage(title: 'Chat', model: currentModel); @@ -270,48 +266,48 @@ class _HomeScreenState extends State { unselectedItemColor: widget.useVertexBackend ? Theme.of(context).colorScheme.onSurface.withValues(alpha: 0.7) : Colors.grey, - items: [ - const BottomNavigationBarItem( + items: const [ + BottomNavigationBarItem( icon: Icon(Icons.chat), label: 'Chat', tooltip: 'Chat', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.mic), label: 'Audio', tooltip: 'Audio Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.numbers), label: 'Tokens', tooltip: 'Token Count', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.functions), label: 'Functions', tooltip: 'Function Calling', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.image), label: 'Image', tooltip: 'Image Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.image_search), label: 'Imagen', tooltip: 'Imagen Model', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.schema), label: 'Schema', tooltip: 'Schema Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.edit_document), label: 'Document', tooltip: 'Document Prompt', ), - const BottomNavigationBarItem( + BottomNavigationBarItem( icon: Icon(Icons.video_collection), label: 'Video', tooltip: 'Video Prompt', @@ -319,12 +315,9 @@ class _HomeScreenState extends State { BottomNavigationBarItem( icon: Icon( Icons.stream, - color: widget.useVertexBackend ? null : Colors.grey, ), label: 'Live', - tooltip: widget.useVertexBackend - ? 'Live Stream' - : 'Live Stream (Currently Disabled)', + tooltip: 'Live Stream', ), ], currentIndex: widget.selectedIndex, diff --git a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart index cb221329d28f..6796fb996cab 100644 --- a/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_ai/firebase_ai/example/lib/pages/bidi_page.dart @@ -22,10 +22,15 @@ import '../utils/audio_output.dart'; import '../widgets/message_widget.dart'; class BidiPage extends StatefulWidget { - const BidiPage({super.key, required this.title, required this.model}); + const BidiPage( + {super.key, + required this.title, + required this.model, + required this.useVertexBackend}); final String title; final GenerativeModel model; + final bool useVertexBackend; @override State createState() => _BidiPageState(); @@ -64,13 +69,21 @@ class _BidiPageState extends State { ); // ignore: deprecated_member_use - _liveModel = FirebaseAI.vertexAI().liveGenerativeModel( - model: 'gemini-2.0-flash-exp', - liveGenerationConfig: config, - tools: [ - Tool.functionDeclarations([lightControlTool]), - ], - ); + _liveModel = widget.useVertexBackend + ? FirebaseAI.vertexAI().liveGenerativeModel( + model: 'gemini-2.0-flash-exp', + liveGenerationConfig: config, + tools: [ + Tool.functionDeclarations([lightControlTool]), + ], + ) + : FirebaseAI.googleAI().liveGenerativeModel( + model: 'gemini-live-2.5-flash-preview', + liveGenerationConfig: config, + tools: [ + Tool.functionDeclarations([lightControlTool]), + ], + ); _initAudio(); } @@ -389,9 +402,13 @@ class _BidiPageState extends State { brightness: brightness, colorTemperature: color, ); - await _session.send( - input: Content.functionResponse(functionCall.name, functionResult), - ); + await _session.sendToolResponse([ + FunctionResponse( + functionCall.name, + functionResult, + id: functionCall.id, + ), + ]); } else { throw UnimplementedError( 'Function not declared to the model: ${functionCall.name}', diff --git a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart index 413af8ba49eb..80841778e761 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/base_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/base_model.dart @@ -56,6 +56,7 @@ enum Task { abstract interface class _ModelUri { String get baseAuthority; + String get apiVersion; Uri taskUri(Task task); ({String prefix, String name}) get model; } @@ -96,6 +97,9 @@ final class _VertexUri implements _ModelUri { @override String get baseAuthority => _baseAuthority; + @override + String get apiVersion => _apiVersion; + @override Uri taskUri(Task task) { return _projectUri.replace( @@ -135,6 +139,9 @@ final class _GoogleAIUri implements _ModelUri { @override String get baseAuthority => _baseAuthority; + @override + String get apiVersion => _apiVersion; + @override Uri taskUri(Task task) => _baseUri.replace( pathSegments: _baseUri.pathSegments diff --git a/packages/firebase_ai/firebase_ai/lib/src/content.dart b/packages/firebase_ai/firebase_ai/lib/src/content.dart index b2661d05c880..fb627a9871c1 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/content.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/content.dart @@ -48,8 +48,9 @@ final class Content { static Content model(Iterable parts) => Content('model', [...parts]); /// Return a [Content] with [FunctionResponse]. - static Content functionResponse(String name, Map response) => - Content('function', [FunctionResponse(name, response)]); + static Content functionResponse(String name, Map response, + {String? id}) => + Content('function', [FunctionResponse(name, response, id: id)]); /// Return a [Content] with multiple [FunctionResponse]. static Content functionResponses(Iterable responses) => diff --git a/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart b/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart index 232d052658d0..3739cfec6a9c 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/firebase_ai.dart @@ -175,14 +175,11 @@ class FirebaseAI extends FirebasePluginPlatform { List? tools, Content? systemInstruction, }) { - if (!_useVertexBackend) { - throw FirebaseAISdkException( - 'LiveGenerativeModel is currently only supported with the VertexAI backend.'); - } return createLiveGenerativeModel( app: app, location: location, model: model, + useVertexBackend: _useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction, diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart index 8ae67a65051f..c5358101c52d 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_api.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_api.dart @@ -234,7 +234,15 @@ class LiveClientToolResponse { final List? functionResponses; // ignore: public_member_api_docs Map toJson() => { - 'functionResponses': functionResponses?.map((e) => e.toJson()).toList(), + 'toolResponse': { + 'functionResponses': functionResponses + ?.map((e) => { + 'name': e.name, + 'response': e.response, + if (e.id != null) 'id': e.id, + }) + .toList(), + }, }; } diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart index 4787d0ea97ed..86106ad9a4ec 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_model.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_model.dart @@ -15,7 +15,8 @@ part of 'base_model.dart'; const _apiUrl = 'ws/google.firebase.vertexai'; -const _apiUrlSuffix = 'LlmBidiService/BidiGenerateContent/locations'; +const _apiUrlSuffixVertexAI = 'LlmBidiService/BidiGenerateContent/locations'; +const _apiUrlSuffixGoogleAI = 'GenerativeService/BidiGenerateContent'; /// A live, generative AI model for real-time interaction. /// @@ -32,6 +33,7 @@ final class LiveGenerativeModel extends BaseModel { {required String model, required String location, required FirebaseApp app, + required bool useVertexBackend, FirebaseAppCheck? appCheck, FirebaseAuth? auth, LiveGenerationConfig? liveGenerationConfig, @@ -39,6 +41,7 @@ final class LiveGenerativeModel extends BaseModel { Content? systemInstruction}) : _app = app, _location = location, + _useVertexBackend = useVertexBackend, _appCheck = appCheck, _auth = auth, _liveGenerationConfig = liveGenerationConfig, @@ -46,22 +49,40 @@ final class LiveGenerativeModel extends BaseModel { _systemInstruction = systemInstruction, super._( serializationStrategy: VertexSerialization(), - modelUri: _VertexUri( - model: model, - app: app, - location: location, - ), + modelUri: useVertexBackend + ? _VertexUri( + model: model, + app: app, + location: location, + ) + : _GoogleAIUri( + model: model, + app: app, + ), ); - static const _apiVersion = 'v1beta'; final FirebaseApp _app; final String _location; + final bool _useVertexBackend; final FirebaseAppCheck? _appCheck; final FirebaseAuth? _auth; final LiveGenerationConfig? _liveGenerationConfig; final List? _tools; final Content? _systemInstruction; + String _vertexAIUri() => 'wss://${_modelUri.baseAuthority}/' + '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixVertexAI/' + '$_location?key=${_app.options.apiKey}'; + + String _vertexAIModelString() => 'projects/${_app.options.projectId}/' + 'locations/$_location/publishers/google/models/${model.name}'; + + String _googleAIUri() => 'wss://${_modelUri.baseAuthority}/' + '$_apiUrl.${_modelUri.apiVersion}.$_apiUrlSuffixGoogleAI?key=${_app.options.apiKey}'; + + String _googleAIModelString() => + 'projects/${_app.options.projectId}/models/${model.name}'; + /// Establishes a connection to a live generation service. /// /// This function handles the WebSocket connection setup and returns an [LiveSession] @@ -70,11 +91,9 @@ final class LiveGenerativeModel extends BaseModel { /// Returns a [Future] that resolves to an [LiveSession] object upon successful /// connection. Future connect() async { - final uri = 'wss://${_modelUri.baseAuthority}/' - '$_apiUrl.$_apiVersion.$_apiUrlSuffix/' - '$_location?key=${_app.options.apiKey}'; - final modelString = 'projects/${_app.options.projectId}/' - 'locations/$_location/publishers/google/models/${model.name}'; + final uri = _useVertexBackend ? _vertexAIUri() : _googleAIUri(); + final modelString = + _useVertexBackend ? _vertexAIModelString() : _googleAIModelString(); final setupJson = { 'setup': { @@ -95,7 +114,11 @@ final class LiveGenerativeModel extends BaseModel { : IOWebSocketChannel.connect(Uri.parse(uri), headers: headers); await ws.ready; + print('websocket connect with uri $uri'); + ws.sink.add(request); + + print('setup request sent: $setupJson'); return LiveSession(ws); } } @@ -105,6 +128,7 @@ LiveGenerativeModel createLiveGenerativeModel({ required FirebaseApp app, required String location, required String model, + required bool useVertexBackend, FirebaseAppCheck? appCheck, FirebaseAuth? auth, LiveGenerationConfig? liveGenerationConfig, @@ -117,6 +141,7 @@ LiveGenerativeModel createLiveGenerativeModel({ appCheck: appCheck, auth: auth, location: location, + useVertexBackend: useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction, diff --git a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart index 34835ff11247..20e700bc82bf 100644 --- a/packages/firebase_ai/firebase_ai/lib/src/live_session.dart +++ b/packages/firebase_ai/firebase_ai/lib/src/live_session.dart @@ -61,7 +61,18 @@ class LiveSession { ? LiveClientContent(turns: [input], turnComplete: turnComplete) : LiveClientContent(turnComplete: turnComplete); var clientJson = jsonEncode(clientMessage.toJson()); + _ws.sink.add(clientJson); + } + /// Sends tool responses for function calling to the server. + /// + /// [functionResponses] (optional): The list of function responses. + Future sendToolResponse( + List? functionResponses) async { + final toolResponse = + LiveClientToolResponse(functionResponses: functionResponses); + _checkWsStatus(); + var clientJson = jsonEncode(toolResponse.toJson()); _ws.sink.add(clientJson); } diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart index f7b3952e649c..9edd832302f6 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/firebase_vertexai.dart @@ -176,6 +176,7 @@ class FirebaseVertexAI extends FirebasePluginPlatform { app: app, location: location, model: model, + useVertexBackend: _useVertexBackend, liveGenerationConfig: liveGenerationConfig, tools: tools, systemInstruction: systemInstruction,