fix(tokenizer): use Base64 encoding for binary data token estimation

Signed-off-by: Hyeri1ee <[email protected]>

Author: Hyeri1ee

spring-ai-commons/src/main/java/org/springframework/ai/tokenizer/JTokkitTokenCountEstimator.java

@@ -16,6 +16,8 @@
 
 package org.springframework.ai.tokenizer;
 
+import java.util.Base64;
+
 import com.knuddels.jtokkit.Encodings;
 import com.knuddels.jtokkit.api.Encoding;
 import com.knuddels.jtokkit.api.EncodingType;
@@ -34,43 +36,52 @@
  */
 public class JTokkitTokenCountEstimator implements TokenCountEstimator {
 
+	/**
+	 * The JTokkit encoding instance used for token counting.
+	 */
 	private final Encoding estimator;
 
+	/**
+	 * Creates a new JTokkitTokenCountEstimator with default CL100K_BASE encoding.
+	 */
 	public JTokkitTokenCountEstimator() {
 		this(EncodingType.CL100K_BASE);
 	}
 
-	public JTokkitTokenCountEstimator(EncodingType tokenEncodingType) {
+	/**
+	 * Creates a new JTokkitTokenCountEstimator with the specified encoding type.
+	 * @param tokenEncodingType the encoding type to use for token counting
+	 */
+	public JTokkitTokenCountEstimator(final EncodingType tokenEncodingType) {
 		this.estimator = Encodings.newLazyEncodingRegistry().getEncoding(tokenEncodingType);
 	}
 
 	@Override
-	public int estimate(String text) {
+	public int estimate(final String text) {
 		if (text == null) {
 			return 0;
 		}
 		return this.estimator.countTokens(text);
 	}
 
 	@Override
-	public int estimate(MediaContent content) {
+	public int estimate(final MediaContent content) {
 		int tokenCount = 0;
 
 		if (content.getText() != null) {
 			tokenCount += this.estimate(content.getText());
 		}
 
 		if (!CollectionUtils.isEmpty(content.getMedia())) {
-
 			for (Media media : content.getMedia()) {
-
 				tokenCount += this.estimate(media.getMimeType().toString());
 
 				if (media.getData() instanceof String textData) {
 					tokenCount += this.estimate(textData);
 				}
 				else if (media.getData() instanceof byte[] binaryData) {
-					tokenCount += binaryData.length; // This is likely incorrect.
+					String base64 = Base64.getEncoder().encodeToString(binaryData);
+					tokenCount += this.estimate(base64);
 				}
 			}
 		}
@@ -79,7 +90,7 @@ else if (media.getData() instanceof byte[] binaryData) {
 	}
 
 	@Override
-	public int estimate(Iterable<MediaContent> contents) {
+	public int estimate(final Iterable<MediaContent> contents) {
 		int totalSize = 0;
 		for (MediaContent mediaContent : contents) {
 			totalSize += this.estimate(mediaContent);