feat(go/plugins/compat_oai): Add media parts support

go/plugins/compat_oai/generate.go

@@ -107,9 +107,32 @@ func (g *ModelGenerator) WithMessages(messages []*ai.Message) *ModelGenerator {
 				)
 				oaiMessages = append(oaiMessages, tm)
 			}
-		default:
+		case ai.RoleUser:
 			oaiMessages = append(oaiMessages, openai.UserMessage(content))
+
+			parts := []openai.ChatCompletionContentPartUnionParam{}
+			for _, p := range msg.Content {
+				if p.IsMedia() {
+					part := openai.ImageContentPart(
+						openai.ChatCompletionContentPartImageImageURLParam{
+							URL: p.Text,
+						})
+					parts = append(parts, part)
+					continue
+				}
+			}
+			if len(parts) > 0 {
+				oaiMessages = append(oaiMessages, openai.ChatCompletionMessageParamUnion{
+					OfUser: &openai.ChatCompletionUserMessageParam{
+						Content: openai.ChatCompletionUserMessageParamContentUnion{OfArrayOfContentParts: parts},
+					},
+				})
+			}
+		default:
+			// ignore parts from not supported roles
+			continue
 		}
+
 	}
 	g.messages = oaiMessages
 	return g

go/plugins/googlegenai/googleai_live_test.go

@@ -178,7 +178,7 @@ func TestGoogleAILive(t *testing.T) {
 		}
 	})
 	t.Run("tool with thinking", func(t *testing.T) {
-		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17")
+		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash")
 		resp, err := genkit.Generate(ctx, g,
 			ai.WithConfig(&genai.GenerateContentConfig{
 				ThinkingConfig: &genai.ThinkingConfig{
@@ -331,8 +331,8 @@ func TestGoogleAILive(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !strings.Contains(strings.ToLower(resp.Text()), "donkey") {
-			t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text())
+		if !strings.Contains(strings.ToLower(resp.Text()), "cat") {
+			t.Fatalf("image detection failed, want: cat, got: %s", resp.Text())
 		}
 	})
 	t.Run("media content", func(t *testing.T) {
@@ -357,19 +357,19 @@ func TestGoogleAILive(t *testing.T) {
 			t.Fatal(err)
 		}
 		resp, err := genkit.Generate(ctx, g,
-			ai.WithSystem("You are a pirate expert in TV Shows, your response should include the name of the character in the image provided"),
+			ai.WithSystem("You are an excellent animal detector, the user will provide you a request with an image, identify which animal is in there"),
 			ai.WithMessages(
 				ai.NewUserMessage(
 					ai.NewTextPart("do you know who's in the image?"),
-					ai.NewDataPart("data:image/png;base64,"+i),
+					ai.NewDataPart("data:image/jpg;base64,"+i),
 				),
 			),
 		)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !strings.Contains(resp.Text(), "donkey") {
-			t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text())
+		if !strings.Contains(strings.ToLower(resp.Text()), "cat") {
+			t.Fatalf("image detection failed, want: cat, got: %s", resp.Text())
 		}
 	})
 	t.Run("image generation", func(t *testing.T) {
@@ -435,7 +435,7 @@ func TestGoogleAILive(t *testing.T) {
 		}
 	})
 	t.Run("thinking", func(t *testing.T) {
-		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17")
+		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash")
 		resp, err := genkit.Generate(ctx, g,
 			ai.WithConfig(genai.GenerateContentConfig{
 				Temperature: genai.Ptr[float32](0.4),
@@ -457,7 +457,7 @@ func TestGoogleAILive(t *testing.T) {
 		}
 	})
 	t.Run("thinking disabled", func(t *testing.T) {
-		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17")
+		m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash")
 		resp, err := genkit.Generate(ctx, g,
 			ai.WithConfig(genai.GenerateContentConfig{
 				Temperature: genai.Ptr[float32](0.4),
@@ -567,7 +567,7 @@ func TestCacheHelper(t *testing.T) {
 
 func fetchImgAsBase64() (string, error) {
 	// CC0 license image
-	imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg"
+	imgUrl := "https://pd.w.org/2025/07/896686fbbcd9990c9.84605288-2048x1365.jpg"
 	resp, err := http.Get(imgUrl)
 	if err != nil {
 		return "", err

go/plugins/googlegenai/models.go

@@ -28,7 +28,6 @@ const (
 	gemini20FlashPrevImageGen    = "gemini-2.0-flash-preview-image-generation"
 
 	gemini25Flash             = "gemini-2.5-flash"
-	gemini25FlashPreview0417  = "gemini-2.5-flash-preview-04-17"
 	gemini25FlashLite         = "gemini-2.5-flash-lite"
 	gemini25FlashLitePrev0617 = "gemini-2.5-flash-lite-preview-06-17"
 
@@ -66,7 +65,6 @@ var (
 		gemini25Flash,
 		gemini25FlashLite,
 		gemini25Pro,
-		gemini25FlashPreview0417,
 		gemini25FlashLitePrev0617,
 		gemini25ProExp0325,
 		gemini25ProPreview0325,
@@ -90,7 +88,6 @@ var (
 		gemini25Flash,
 		gemini25FlashLite,
 		gemini25Pro,
-		gemini25FlashPreview0417,
 		gemini25FlashLitePrev0617,
 		gemini25ProExp0325,
 		gemini25ProPreview0325,
@@ -192,12 +189,6 @@ var (
 			Supports: &Multimodal,
 			Stage:    ai.ModelStageStable,
 		},
-		gemini25FlashPreview0417: {
-			Label:    "Gemini 2.5 Flash Preview 04-17",
-			Versions: []string{},
-			Supports: &Multimodal,
-			Stage:    ai.ModelStageUnstable,
-		},
 		gemini25ProExp0325: {
 			Label:    "Gemini 2.5 Pro Exp 03-25",
 			Versions: []string{},

go/plugins/googlegenai/vertexai_live_test.go

@@ -216,16 +216,16 @@ func TestVertexAILive(t *testing.T) {
 			ai.WithSystem("You are a pirate expert in animals, your response should include the name of the animal in the provided image"),
 			ai.WithMessages(
 				ai.NewUserMessage(
-					ai.NewTextPart("do you know who's in the image?"),
-					ai.NewMediaPart("image/png", "data:image/png;base64,"+i),
+					ai.NewTextPart("do you know which animal is in the image?"),
+					ai.NewMediaPart("image/jpg", "data:image/jpg;base64,"+i),
 				),
 			),
 		)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !strings.Contains(resp.Text(), "donkey") {
-			t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text())
+		if !strings.Contains(strings.ToLower(resp.Text()), "cat") {
+			t.Fatalf("image detection failed, want: cat, got: %s", resp.Text())
 		}
 	})
 	t.Run("media content", func(t *testing.T) {
@@ -250,24 +250,24 @@ func TestVertexAILive(t *testing.T) {
 			t.Fatal(err)
 		}
 		resp, err := genkit.Generate(ctx, g,
-			ai.WithSystem("You are a pirate expert in TV Shows, your response should include the name of the character in the image provided"),
+			ai.WithSystem("You are a pirate expert in animals, your response should include the name of the animal in the image provided"),
 			ai.WithMessages(
 				ai.NewUserMessage(
-					ai.NewTextPart("do you know who's in the image?"),
-					ai.NewDataPart("data:image/png;base64,"+i),
+					ai.NewTextPart("do you know which animal is in the image?"),
+					ai.NewDataPart("data:image/jpg;base64,"+i),
 				),
 			),
 		)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if !strings.Contains(resp.Text(), "donkey") {
-			t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text())
+		if !strings.Contains(strings.ToLower(resp.Text()), "cat") {
+			t.Fatalf("image detection failed, want: cat, got: %s", resp.Text())
 		}
 	})
 	t.Run("image generation", func(t *testing.T) {
 		if location != "global" {
-			t.Skip("image generation in Vertex AI is only supported in region: global")
+			t.Skipf("image generation in Vertex AI is only supported in region: global, got: %s", location)
 		}
 		m := googlegenai.VertexAIModel(g, "gemini-2.0-flash-preview-image-generation")
 		resp, err := genkit.Generate(ctx, g,

go/plugins/vertexai/modelgarden/anthropic_live_test.go

@@ -100,14 +100,13 @@ func TestAnthropicLive(t *testing.T) {
 			ai.WithModel(m),
 			ai.WithMessages(
 				ai.NewUserMessage(
-					ai.NewTextPart("do you know who's in the image?"),
+					ai.NewTextPart("do you know which animal is in the image?"),
 					ai.NewMediaPart("", "data:image/jpeg;base64,"+i))))
 		if err != nil {
 			t.Fatal(err)
 		}
-
-		if !strings.Contains(resp.Text(), "donkey") {
-			t.Fatalf("it should've said donkey but got: %s", resp.Text())
+		if !strings.Contains(strings.ToLower(resp.Text()), "cat") {
+			t.Fatalf("want: cat, got: %s", resp.Text())
 		}
 	})
 
@@ -209,7 +208,7 @@ func TestAnthropicLive(t *testing.T) {
 
 func fetchImgAsBase64() (string, error) {
 	// CC0 license image
-	imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg"
+	imgUrl := "https://pd.w.org/2025/07/896686fbbcd9990c9.84605288-2048x1365.jpg"
 	resp, err := http.Get(imgUrl)
 	if err != nil {
 		return "", err

go/samples/compat_oai/openai/main.go

@@ -42,11 +42,11 @@ func main() {
 
 		prompt := fmt.Sprintf("tell me a joke about %s", subject)
 		config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5), MaxTokens: openai.Int(100)}
-		foo, err := genkit.Generate(ctx, g, ai.WithModel(gpt4o), ai.WithPrompt(prompt), ai.WithConfig(config))
+		resp, err := genkit.Generate(ctx, g, ai.WithModel(gpt4o), ai.WithPrompt(prompt), ai.WithConfig(config))
 		if err != nil {
 			return "", err
 		}
-		return fmt.Sprintf("foo: %s", foo.Text()), nil
+		return fmt.Sprintf("resp: %s", resp.Text()), nil
 	})
 
 	genkit.DefineFlow(g, "defined-model", func(ctx context.Context, subject string) (string, error) {
@@ -56,11 +56,32 @@ func main() {
 		}
 		prompt := fmt.Sprintf("tell me a joke about %s", subject)
 		config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5)}
-		foo, err := genkit.Generate(ctx, g, ai.WithModel(gpt4oMini), ai.WithPrompt(prompt), ai.WithConfig(config))
+		resp, err := genkit.Generate(ctx, g, ai.WithModel(gpt4oMini), ai.WithPrompt(prompt), ai.WithConfig(config))
 		if err != nil {
 			return "", err
 		}
-		return fmt.Sprintf("foo: %s", foo.Text()), nil
+		return resp.Text(), nil
+	})
+
+	genkit.DefineFlow(g, "media", func(ctx context.Context, subject string) (string, error) {
+		gpt4oMini := oai.Model(g, "gpt-4o-mini")
+		if err != nil {
+			return "", err
+		}
+		config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5)}
+		resp, err := genkit.Generate(ctx, g,
+			ai.WithModel(gpt4oMini),
+			ai.WithConfig(config),
+			ai.WithMessages(
+				ai.NewUserMessage(ai.NewTextPart("Hi, I'll provide you a quick request in the following message")),
+				ai.NewUserMessage(
+					ai.NewTextPart("can you tell me which animal is in the provided image?"),
+					ai.NewMediaPart("image/jpg", "https://pd.w.org/2025/07/58268765f177911d4.13750400-2048x1365.jpg"),
+				)))
+		if err != nil {
+			return "", err
+		}
+		return resp.Text(), nil
 	})
 
 	mux := http.NewServeMux()

go/samples/prompts/main.go

@@ -340,7 +340,7 @@ func PromptWithMediaType(ctx context.Context, g *genkit.Genkit) {
 	resp, err := prompt.Execute(ctx,
 
 		ai.WithModelName("vertexai/gemini-2.0-flash"),
-		ai.WithInput(map[string]any{"imageUrl": "data:image/png;base64," + img}),
+		ai.WithInput(map[string]any{"imageUrl": "data:image/jpg;base64," + img}),
 	)
 	if err != nil {
 		log.Fatal(err)
@@ -349,7 +349,7 @@ func PromptWithMediaType(ctx context.Context, g *genkit.Genkit) {
 }
 
 func fetchImgAsBase64() (string, error) {
-	imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg"
+	imgUrl := "https://pd.w.org/2025/07/58268765f177911d4.13750400-2048x1365.jpg"
 	resp, err := http.Get(imgUrl)
 	if err != nil {
 		return "", err