diff --git a/go/plugins/compat_oai/generate.go b/go/plugins/compat_oai/generate.go index 2b39c3bea8..48de5cbfba 100644 --- a/go/plugins/compat_oai/generate.go +++ b/go/plugins/compat_oai/generate.go @@ -107,9 +107,32 @@ func (g *ModelGenerator) WithMessages(messages []*ai.Message) *ModelGenerator { ) oaiMessages = append(oaiMessages, tm) } - default: + case ai.RoleUser: oaiMessages = append(oaiMessages, openai.UserMessage(content)) + + parts := []openai.ChatCompletionContentPartUnionParam{} + for _, p := range msg.Content { + if p.IsMedia() { + part := openai.ImageContentPart( + openai.ChatCompletionContentPartImageImageURLParam{ + URL: p.Text, + }) + parts = append(parts, part) + continue + } + } + if len(parts) > 0 { + oaiMessages = append(oaiMessages, openai.ChatCompletionMessageParamUnion{ + OfUser: &openai.ChatCompletionUserMessageParam{ + Content: openai.ChatCompletionUserMessageParamContentUnion{OfArrayOfContentParts: parts}, + }, + }) + } + default: + // ignore parts from not supported roles + continue } + } g.messages = oaiMessages return g diff --git a/go/plugins/googlegenai/googleai_live_test.go b/go/plugins/googlegenai/googleai_live_test.go index e4edb8775d..a61c384fe4 100644 --- a/go/plugins/googlegenai/googleai_live_test.go +++ b/go/plugins/googlegenai/googleai_live_test.go @@ -178,7 +178,7 @@ func TestGoogleAILive(t *testing.T) { } }) t.Run("tool with thinking", func(t *testing.T) { - m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17") + m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash") resp, err := genkit.Generate(ctx, g, ai.WithConfig(&genai.GenerateContentConfig{ ThinkingConfig: &genai.ThinkingConfig{ @@ -331,8 +331,8 @@ func TestGoogleAILive(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(strings.ToLower(resp.Text()), "donkey") { - t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text()) + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Fatalf("image detection failed, want: cat, got: %s", resp.Text()) } }) t.Run("media content", func(t *testing.T) { @@ -357,19 +357,19 @@ func TestGoogleAILive(t *testing.T) { t.Fatal(err) } resp, err := genkit.Generate(ctx, g, - ai.WithSystem("You are a pirate expert in TV Shows, your response should include the name of the character in the image provided"), + ai.WithSystem("You are an excellent animal detector, the user will provide you a request with an image, identify which animal is in there"), ai.WithMessages( ai.NewUserMessage( ai.NewTextPart("do you know who's in the image?"), - ai.NewDataPart("data:image/png;base64,"+i), + ai.NewDataPart("data:image/jpg;base64,"+i), ), ), ) if err != nil { t.Fatal(err) } - if !strings.Contains(resp.Text(), "donkey") { - t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text()) + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Fatalf("image detection failed, want: cat, got: %s", resp.Text()) } }) t.Run("image generation", func(t *testing.T) { @@ -435,7 +435,7 @@ func TestGoogleAILive(t *testing.T) { } }) t.Run("thinking", func(t *testing.T) { - m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17") + m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash") resp, err := genkit.Generate(ctx, g, ai.WithConfig(genai.GenerateContentConfig{ Temperature: genai.Ptr[float32](0.4), @@ -457,7 +457,7 @@ func TestGoogleAILive(t *testing.T) { } }) t.Run("thinking disabled", func(t *testing.T) { - m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash-preview-04-17") + m := googlegenai.GoogleAIModel(g, "gemini-2.5-flash") resp, err := genkit.Generate(ctx, g, ai.WithConfig(genai.GenerateContentConfig{ Temperature: genai.Ptr[float32](0.4), @@ -567,7 +567,7 @@ func TestCacheHelper(t *testing.T) { func fetchImgAsBase64() (string, error) { // CC0 license image - imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg" + imgUrl := "https://pd.w.org/2025/07/896686fbbcd9990c9.84605288-2048x1365.jpg" resp, err := http.Get(imgUrl) if err != nil { return "", err diff --git a/go/plugins/googlegenai/models.go b/go/plugins/googlegenai/models.go index 96166a3936..127bf9b586 100644 --- a/go/plugins/googlegenai/models.go +++ b/go/plugins/googlegenai/models.go @@ -28,7 +28,6 @@ const ( gemini20FlashPrevImageGen = "gemini-2.0-flash-preview-image-generation" gemini25Flash = "gemini-2.5-flash" - gemini25FlashPreview0417 = "gemini-2.5-flash-preview-04-17" gemini25FlashLite = "gemini-2.5-flash-lite" gemini25FlashLitePrev0617 = "gemini-2.5-flash-lite-preview-06-17" @@ -66,7 +65,6 @@ var ( gemini25Flash, gemini25FlashLite, gemini25Pro, - gemini25FlashPreview0417, gemini25FlashLitePrev0617, gemini25ProExp0325, gemini25ProPreview0325, @@ -90,7 +88,6 @@ var ( gemini25Flash, gemini25FlashLite, gemini25Pro, - gemini25FlashPreview0417, gemini25FlashLitePrev0617, gemini25ProExp0325, gemini25ProPreview0325, @@ -192,12 +189,6 @@ var ( Supports: &Multimodal, Stage: ai.ModelStageStable, }, - gemini25FlashPreview0417: { - Label: "Gemini 2.5 Flash Preview 04-17", - Versions: []string{}, - Supports: &Multimodal, - Stage: ai.ModelStageUnstable, - }, gemini25ProExp0325: { Label: "Gemini 2.5 Pro Exp 03-25", Versions: []string{}, diff --git a/go/plugins/googlegenai/vertexai_live_test.go b/go/plugins/googlegenai/vertexai_live_test.go index 170f43d58a..5b696de223 100644 --- a/go/plugins/googlegenai/vertexai_live_test.go +++ b/go/plugins/googlegenai/vertexai_live_test.go @@ -216,16 +216,16 @@ func TestVertexAILive(t *testing.T) { ai.WithSystem("You are a pirate expert in animals, your response should include the name of the animal in the provided image"), ai.WithMessages( ai.NewUserMessage( - ai.NewTextPart("do you know who's in the image?"), - ai.NewMediaPart("image/png", "data:image/png;base64,"+i), + ai.NewTextPart("do you know which animal is in the image?"), + ai.NewMediaPart("image/jpg", "data:image/jpg;base64,"+i), ), ), ) if err != nil { t.Fatal(err) } - if !strings.Contains(resp.Text(), "donkey") { - t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text()) + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Fatalf("image detection failed, want: cat, got: %s", resp.Text()) } }) t.Run("media content", func(t *testing.T) { @@ -250,24 +250,24 @@ func TestVertexAILive(t *testing.T) { t.Fatal(err) } resp, err := genkit.Generate(ctx, g, - ai.WithSystem("You are a pirate expert in TV Shows, your response should include the name of the character in the image provided"), + ai.WithSystem("You are a pirate expert in animals, your response should include the name of the animal in the image provided"), ai.WithMessages( ai.NewUserMessage( - ai.NewTextPart("do you know who's in the image?"), - ai.NewDataPart("data:image/png;base64,"+i), + ai.NewTextPart("do you know which animal is in the image?"), + ai.NewDataPart("data:image/jpg;base64,"+i), ), ), ) if err != nil { t.Fatal(err) } - if !strings.Contains(resp.Text(), "donkey") { - t.Fatalf("image detection failed, want: donkey, got: %s", resp.Text()) + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Fatalf("image detection failed, want: cat, got: %s", resp.Text()) } }) t.Run("image generation", func(t *testing.T) { if location != "global" { - t.Skip("image generation in Vertex AI is only supported in region: global") + t.Skipf("image generation in Vertex AI is only supported in region: global, got: %s", location) } m := googlegenai.VertexAIModel(g, "gemini-2.0-flash-preview-image-generation") resp, err := genkit.Generate(ctx, g, diff --git a/go/plugins/vertexai/modelgarden/anthropic_live_test.go b/go/plugins/vertexai/modelgarden/anthropic_live_test.go index 0487e92f19..847f9a4719 100644 --- a/go/plugins/vertexai/modelgarden/anthropic_live_test.go +++ b/go/plugins/vertexai/modelgarden/anthropic_live_test.go @@ -100,14 +100,13 @@ func TestAnthropicLive(t *testing.T) { ai.WithModel(m), ai.WithMessages( ai.NewUserMessage( - ai.NewTextPart("do you know who's in the image?"), + ai.NewTextPart("do you know which animal is in the image?"), ai.NewMediaPart("", "data:image/jpeg;base64,"+i)))) if err != nil { t.Fatal(err) } - - if !strings.Contains(resp.Text(), "donkey") { - t.Fatalf("it should've said donkey but got: %s", resp.Text()) + if !strings.Contains(strings.ToLower(resp.Text()), "cat") { + t.Fatalf("want: cat, got: %s", resp.Text()) } }) @@ -209,7 +208,7 @@ func TestAnthropicLive(t *testing.T) { func fetchImgAsBase64() (string, error) { // CC0 license image - imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg" + imgUrl := "https://pd.w.org/2025/07/896686fbbcd9990c9.84605288-2048x1365.jpg" resp, err := http.Get(imgUrl) if err != nil { return "", err diff --git a/go/samples/compat_oai/openai/main.go b/go/samples/compat_oai/openai/main.go index 5618690ec7..3901ff3e10 100644 --- a/go/samples/compat_oai/openai/main.go +++ b/go/samples/compat_oai/openai/main.go @@ -42,11 +42,11 @@ func main() { prompt := fmt.Sprintf("tell me a joke about %s", subject) config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5), MaxTokens: openai.Int(100)} - foo, err := genkit.Generate(ctx, g, ai.WithModel(gpt4o), ai.WithPrompt(prompt), ai.WithConfig(config)) + resp, err := genkit.Generate(ctx, g, ai.WithModel(gpt4o), ai.WithPrompt(prompt), ai.WithConfig(config)) if err != nil { return "", err } - return fmt.Sprintf("foo: %s", foo.Text()), nil + return fmt.Sprintf("resp: %s", resp.Text()), nil }) genkit.DefineFlow(g, "defined-model", func(ctx context.Context, subject string) (string, error) { @@ -56,11 +56,32 @@ func main() { } prompt := fmt.Sprintf("tell me a joke about %s", subject) config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5)} - foo, err := genkit.Generate(ctx, g, ai.WithModel(gpt4oMini), ai.WithPrompt(prompt), ai.WithConfig(config)) + resp, err := genkit.Generate(ctx, g, ai.WithModel(gpt4oMini), ai.WithPrompt(prompt), ai.WithConfig(config)) if err != nil { return "", err } - return fmt.Sprintf("foo: %s", foo.Text()), nil + return resp.Text(), nil + }) + + genkit.DefineFlow(g, "media", func(ctx context.Context, subject string) (string, error) { + gpt4oMini := oai.Model(g, "gpt-4o-mini") + if err != nil { + return "", err + } + config := &openai.ChatCompletionNewParams{Temperature: openai.Float(0.5)} + resp, err := genkit.Generate(ctx, g, + ai.WithModel(gpt4oMini), + ai.WithConfig(config), + ai.WithMessages( + ai.NewUserMessage(ai.NewTextPart("Hi, I'll provide you a quick request in the following message")), + ai.NewUserMessage( + ai.NewTextPart("can you tell me which animal is in the provided image?"), + ai.NewMediaPart("image/jpg", "https://pd.w.org/2025/07/58268765f177911d4.13750400-2048x1365.jpg"), + ))) + if err != nil { + return "", err + } + return resp.Text(), nil }) mux := http.NewServeMux() diff --git a/go/samples/prompts/main.go b/go/samples/prompts/main.go index 08e217a73f..fd2caf4fd5 100644 --- a/go/samples/prompts/main.go +++ b/go/samples/prompts/main.go @@ -340,7 +340,7 @@ func PromptWithMediaType(ctx context.Context, g *genkit.Genkit) { resp, err := prompt.Execute(ctx, ai.WithModelName("vertexai/gemini-2.0-flash"), - ai.WithInput(map[string]any{"imageUrl": "data:image/png;base64," + img}), + ai.WithInput(map[string]any{"imageUrl": "data:image/jpg;base64," + img}), ) if err != nil { log.Fatal(err) @@ -349,7 +349,7 @@ func PromptWithMediaType(ctx context.Context, g *genkit.Genkit) { } func fetchImgAsBase64() (string, error) { - imgUrl := "https://pd.w.org/2025/05/64268380a8c42af85.63713105-2048x1152.jpg" + imgUrl := "https://pd.w.org/2025/07/58268765f177911d4.13750400-2048x1365.jpg" resp, err := http.Get(imgUrl) if err != nil { return "", err