Simplify utf8 sanitization

Author: ericmj

lib/diff_web/live/diff_live_view.ex

@@ -227,7 +227,7 @@ defmodule DiffWeb.DiffLiveView do
 
             diff_data =
               Jason.encode!(%{
-                "diff" => sanitize_utf8(raw_diff),
+                "diff" => DiffWeb.LiveView.sanitize_utf8(raw_diff),
                 "path_from" => path_from,
                 "path_to" => path_to
               })
@@ -336,45 +336,4 @@ defmodule DiffWeb.DiffLiveView do
   end
 
   defp build_url(app, from, to), do: "/diff/#{app}/#{from}..#{to}"
-
-  # UTF-8 sanitization functions
-  defp sanitize_utf8(content) when is_binary(content) do
-    case String.valid?(content) do
-      true ->
-        content
-
-      false ->
-        # Multiple fallback strategies for invalid UTF-8
-        content
-        |> sanitize_invalid_bytes()
-    end
-  end
-
-  defp sanitize_utf8(content), do: content
-
-  defp sanitize_invalid_bytes(content) do
-    # Try different encoding conversions and fallbacks
-    cond do
-      # Try converting from Latin-1/ISO-8859-1 encoding
-      latin1_result = safe_unicode_convert(content, :latin1, :utf8) ->
-        latin1_result
-
-      # Last resort: replace invalid bytes with replacement character
-      true ->
-        content
-        |> :binary.bin_to_list()
-        # Replace high bytes with '?'
-        |> Enum.map(fn byte -> if byte > 127, do: 63, else: byte end)
-        |> :binary.list_to_bin()
-    end
-  end
-
-  defp safe_unicode_convert(content, from, to) do
-    case :unicode.characters_to_binary(content, from, to) do
-      result when is_binary(result) -> result
-      _ -> nil
-    end
-  rescue
-    _ -> nil
-  end
 end

lib/diff_web/views/live_view.ex

@@ -11,7 +11,6 @@ defmodule DiffWeb.LiveView do
             DiffWeb.TooLargeComponent.render(%{file: file_path})
             |> Phoenix.HTML.Safe.to_iodata()
             |> IO.iodata_to_binary()
-            |> sanitize_utf8()
 
           {:ok, %{"diff" => raw_diff, "path_from" => path_from, "path_to" => path_to}} ->
             case GitDiff.parse_patch(raw_diff, relative_from: path_from, relative_to: path_to) do
@@ -38,42 +37,16 @@ defmodule DiffWeb.LiveView do
     end
   end
 
-  defp sanitize_utf8(content) when is_binary(content) do
-    case String.valid?(content) do
-      true ->
-        content
-
-      false ->
-        # Multiple fallback strategies for invalid UTF-8
-        sanitize_invalid_bytes(content)
-    end
-  end
-
-  defp sanitize_utf8(content), do: content
-
-  defp sanitize_invalid_bytes(content) do
-    # Try different encoding conversions and fallbacks
-    cond do
-      # Try converting from Latin-1/ISO-8859-1 encoding
-      latin1_result = safe_unicode_convert(content, :latin1, :utf8) ->
-        latin1_result
-
-      # Last resort: replace invalid bytes with replacement character
-      true ->
-        content
-        |> :binary.bin_to_list()
-        # Replace high bytes with '?'
-        |> Enum.map(fn byte -> if byte > 127, do: 63, else: byte end)
-        |> :binary.list_to_bin()
-    end
-  end
-
-  defp safe_unicode_convert(content, from, to) do
-    case :unicode.characters_to_binary(content, from, to) do
-      result when is_binary(result) -> result
-      _ -> nil
-    end
-  rescue
-    _ -> nil
+  def sanitize_utf8(content) when is_binary(content) do
+    content
+    |> String.chunk(:valid)
+    |> Enum.map(fn chunk ->
+      if String.valid?(chunk) do
+        chunk
+      else
+        String.duplicate("?", byte_size(chunk))
+      end
+    end)
+    |> Enum.join("")
   end
 end