Support images in Gemini responses

Author: crmne

lib/ruby_llm/attachment.rb

@@ -65,6 +65,14 @@ def encoded
       Base64.strict_encode64(content)
     end
 
+    def save(path)
+      return unless io_like?
+
+      File.open(path, 'w') do |f|
+        f.puts(@source.read)
+      end
+    end
+
     def for_llm
       case type
       when :text

lib/ruby_llm/connection.rb

@@ -65,7 +65,7 @@ def setup_logging(faraday)
                        errors: true,
                        headers: false,
                        log_level: :debug do |logger|
-        logger.filter(%r{[A-Za-z0-9+/=]{100,}}, 'data":"[BASE64 DATA]"')
+        logger.filter(%r{[A-Za-z0-9+/=]{100,}}, '[BASE64 DATA]')
         logger.filter(/[-\d.e,\s]{100,}/, '[EMBEDDINGS ARRAY]')
       end
     end

lib/ruby_llm/providers/gemini/chat.rb

@@ -50,22 +50,9 @@ def format_role(role)
 
         def format_parts(msg)
           if msg.tool_call?
-            [{
-              functionCall: {
-                name: msg.tool_calls.values.first.name,
-                args: msg.tool_calls.values.first.arguments
-              }
-            }]
+            format_tool_call(msg)
           elsif msg.tool_result?
-            [{
-              functionResponse: {
-                name: msg.tool_call_id,
-                response: {
-                  name: msg.tool_call_id,
-                  content: Media.format_content(msg.content)
-                }
-              }
-            }]
+            format_tool_result(msg)
           else
             Media.format_content(msg.content)
           end
@@ -77,7 +64,7 @@ def parse_completion_response(response)
 
           Message.new(
             role: :assistant,
-            content: extract_content(data),
+            content: parse_content(data),
             tool_calls: tool_calls,
             input_tokens: data.dig('usageMetadata', 'promptTokenCount'),
             output_tokens: calculate_output_tokens(data),
@@ -109,17 +96,16 @@ def normalize_any_of(schema)
           { type: 'string', nullable: true }
         end
 
-        def extract_content(data)
+        def parse_content(data)
           candidate = data.dig('candidates', 0)
           return '' unless candidate
 
           return '' if function_call?(candidate)
 
           parts = candidate.dig('content', 'parts')
-          text_parts = parts&.select { |p| p['text'] }
-          return '' unless text_parts&.any?
+          return '' unless parts&.any?
 
-          text_parts.map { |p| p['text'] }.join
+          build_response_content(parts)
         end
 
         def function_call?(candidate)

lib/ruby_llm/providers/gemini/media.rb

@@ -2,7 +2,7 @@
 
 module RubyLLM
   module Providers
-    class Gemini
+    class Gemini # rubocop:disable Style/Documentation
       # Media handling methods for the Gemini API integration
       module Media
         module_function
@@ -50,6 +50,63 @@ def format_text(text)
           }
         end
       end
+
+      def build_response_content(parts) # rubocop:disable Metrics/PerceivedComplexity
+        text = []
+        attachments = []
+
+        parts.each_with_index do |part, index|
+          if part['text']
+            text << part['text']
+          elsif part['inlineData']
+            attachment = build_inline_attachment(part['inlineData'], index)
+            attachments << attachment if attachment
+          elsif part['fileData']
+            attachment = build_file_attachment(part['fileData'], index)
+            attachments << attachment if attachment
+          end
+        end
+
+        text = text.join
+        text = nil if text.empty?
+        return text if attachments.empty?
+
+        Content.new(text:, attachments:)
+      end
+
+      def build_inline_attachment(inline_data, index)
+        encoded = inline_data['data']
+        return unless encoded
+
+        mime_type = inline_data['mimeType']
+        decoded = Base64.decode64(encoded)
+        io = StringIO.new(decoded)
+        io.set_encoding(Encoding::BINARY) if io.respond_to?(:set_encoding)
+
+        filename = attachment_filename(mime_type, index)
+        RubyLLM::Attachment.new(io, filename:)
+      rescue ArgumentError => e
+        RubyLLM.logger.warn "Failed to decode Gemini inline data attachment: #{e.message}"
+        nil
+      end
+
+      def build_file_attachment(file_data, index)
+        uri = file_data['fileUri']
+        return unless uri
+
+        filename = file_data['filename'] || attachment_filename(file_data['mimeType'], index)
+        RubyLLM::Attachment.new(uri, filename:)
+      end
+
+      def attachment_filename(mime_type, index)
+        return "gemini_attachment_#{index + 1}" unless mime_type
+
+        extension = mime_type.split('/').last.to_s
+        extension = 'jpg' if extension == 'jpeg'
+        extension = 'txt' if extension == 'plain'
+        extension = extension.tr('+', '.')
+        "gemini_attachment_#{index + 1}.#{extension}"
+      end
     end
   end
 end

lib/ruby_llm/providers/gemini/tools.rb

@@ -13,6 +13,27 @@ def format_tools(tools)
           }]
         end
 
+        def format_tool_call(msg)
+          [{
+            functionCall: {
+              name: msg.tool_calls.values.first.name,
+              args: msg.tool_calls.values.first.arguments
+            }
+          }]
+        end
+
+        def format_tool_result(msg)
+          [{
+            functionResponse: {
+              name: msg.tool_call_id,
+              response: {
+                name: msg.tool_call_id,
+                content: Media.format_content(msg.content)
+              }
+            }
+          }]
+        end
+
         def extract_tool_calls(data)
           return nil unless data