Lazy load diffs (#105)

Author: ericmj

Dockerfile

@@ -48,7 +48,7 @@ FROM debian:${DEBIAN_VERSION} AS app
 
 RUN apt update && \
     apt upgrade -y && \
-    apt install --no-install-recommends -y bash openssl git && \
+    apt install --no-install-recommends -y bash openssl git ca-certificates && \
     apt clean -y && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /app

assets/css/app.css

@@ -382,6 +382,62 @@ table.package-list .button {
   margin-bottom: 0;
 }
 
+.diff-stats-header {
+  margin: 15px 0;
+  padding: 10px 0;
+  border-bottom: 1px solid rgba(255, 255, 255, 0.1);
+}
+
+.diff-stats-header span {
+  margin-right: 15px;
+  font-family: 'SFMono-Regular', 'Consolas', 'Liberation Mono', 'Menlo', monospace;
+  font-size: 14px;
+}
+
+.diff-stats-header .files-changed {
+  color: #ddd;
+}
+
+.diff-stats-header .additions {
+  color: #4CAF50;
+  font-weight: 600;
+}
+
+.diff-stats-header .deletions {
+  color: #f44336;
+  font-weight: 600;
+}
+
+.loading-spinner {
+  height: 60px;
+  width: 100%;
+  margin: 30px 0;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  color: #888;
+  font-size: 14px;
+  opacity: 0.8;
+}
+
+.loading-spinner::before {
+  content: '';
+  width: 20px;
+  height: 20px;
+  border: 2px solid #f3f3f3;
+  border-top: 2px solid #888;
+  border-radius: 50%;
+  animation: spin 1s linear infinite;
+  margin-right: 10px;
+}
+
+@keyframes spin {
+  0% { transform: rotate(0deg); }
+  100% { transform: rotate(360deg); }
+}
+
+
+
 @media only screen
   and (min-device-width: 320px)
   and (max-device-width: 480px) {

assets/js/app.js

@@ -14,7 +14,53 @@ import "phoenix_html"
 import { Socket } from 'phoenix'
 import { LiveSocket } from "phoenix_live_view"
 
-let liveSocket = new LiveSocket("/live", Socket, {})
+// Define hooks for LiveView
+window.Hooks = {}
+
+window.Hooks.InfiniteScroll = {
+  mounted() {
+    this.pending = false
+
+    this.observer = new IntersectionObserver((entries) => {
+      const target = entries[0]
+      if (target.isIntersecting && !this.pending) {
+        this.pending = true
+        this.pushEvent("load-more", {})
+      }
+    }, {
+      root: null,
+      rootMargin: '100px',
+      threshold: 0.1
+    })
+
+    this.observer.observe(this.el)
+  },
+
+  destroyed() {
+    if (this.observer) {
+      this.observer.disconnect()
+    }
+  },
+
+  updated() {
+    this.pending = false
+
+    // Check if we're still at the bottom after loading content
+    // Use requestAnimationFrame to ensure DOM has fully updated
+    requestAnimationFrame(() => {
+      const target = this.el
+      const rect = target.getBoundingClientRect()
+      const isIntersecting = rect.top <= (window.innerHeight || document.documentElement.clientHeight)
+
+      if (isIntersecting && !this.pending) {
+        this.pending = true
+        this.pushEvent("load-more", {})
+      }
+    })
+  }
+}
+
+let liveSocket = new LiveSocket("/live", Socket, { hooks: window.Hooks })
 liveSocket.connect()
 
 /*

assets/yarn.lock

@@ -11,4 +11,5 @@ config :logger, level: :warning
 
 config :diff,
   package_store_impl: Diff.Package.StoreMock,
-  storage_impl: Diff.StorageMock
+  storage_impl: Diff.StorageMock,
+  hex_impl: Diff.HexMock

config/test.exs

@@ -12,13 +12,9 @@ defmodule Diff.Application do
     children = [
       goth_spec(),
       {Task.Supervisor, name: Diff.Tasks},
-      # Start the PubSub system
       {Phoenix.PubSub, name: Diff.PubSub},
-      # Start the endpoint when the application starts
-      DiffWeb.Endpoint,
-      # Starts a worker by calling: Diff.Worker.start_link(arg)
-      # {Diff.Worker, arg},
-      Diff.Package.Supervisor
+      Diff.Package.Supervisor,
+      DiffWeb.Endpoint
     ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

lib/diff/application.ex

@@ -0,0 +1,4 @@
+defmodule Diff.Hex.Behaviour do
+  @callback diff(package :: String.t(), from :: String.t(), to :: String.t()) ::
+              {:ok, Enumerable.t()} | :error
+end

lib/diff/hex/behaviour.ex

@@ -1,4 +1,6 @@
 defmodule Diff.Hex do
+  @behaviour Diff.Hex.Behaviour
+
   @config %{
     :hex_core.default_config()
     | http_adapter: {Diff.Hex.Adapter, %{}},
@@ -101,9 +103,12 @@ defmodule Diff.Hex do
 
           with {_, true} <- {:file_size_old, file_size_check?(path_old)},
                {_, true} <- {:file_size_new, file_size_check?(path_new)},
-               {_, {:ok, output}} <- {:git_diff, git_diff(path_old, path_new)},
-               {_, {:ok, patches}} <- {:parse_patch, parse_patch(output, path_from, path_to)} do
-            Enum.map(patches, &{:ok, &1})
+               {_, {:ok, output}} <- {:git_diff, git_diff(path_old, path_new)} do
+            if output do
+              [{:ok, {output, path_from, path_to}}]
+            else
+              []
+            end
           else
             {:file_size_old, false} ->
               [{:too_large, Path.relative_to(path_old, path_from)}]
@@ -150,14 +155,6 @@ defmodule Diff.Hex do
     end
   end
 
-  defp parse_patch(_output = nil, _path_from, _path_to) do
-    {:ok, []}
-  end
-
-  defp parse_patch(output, path_from, path_to) do
-    GitDiff.parse_patch(output, relative_from: path_from, relative_to: path_to)
-  end
-
   defp file_size_check?(path) do
     File.stat!(path).size <= @max_file_size
   end

lib/diff/hex/hex.ex

@@ -5,12 +5,12 @@ defmodule Diff.Storage.GCS do
 
   @gs_xml_url "https://storage.googleapis.com"
 
-  def get(package, from_version, to_version) do
+  def get_diff(package, from_version, to_version, diff_id) do
     with {:ok, hash} <- combined_checksum(package, from_version, to_version),
-         url = url(key(package, from_version, to_version, hash)),
-         {:ok, 200, _headers, stream} <-
-           Diff.HTTP.retry("gs", fn -> Diff.HTTP.get_stream(url, headers()) end) do
-      {:ok, stream}
+         url = url(diff_key(package, from_version, to_version, hash, diff_id)),
+         {:ok, 200, _headers, body} <-
+           Diff.HTTP.retry("gs", fn -> Diff.HTTP.get(url, headers()) end) do
+      {:ok, body}
     else
       {:ok, 404, _headers, _body} ->
         {:error, :not_found}
@@ -25,23 +25,82 @@ defmodule Diff.Storage.GCS do
     end
   end
 
-  def put(package, from_version, to_version, stream) do
+  def put_diff(package, from_version, to_version, diff_id, diff_data) do
     with {:ok, hash} <- combined_checksum(package, from_version, to_version),
-         url = url(key(package, from_version, to_version, hash)),
+         url = url(diff_key(package, from_version, to_version, hash, diff_id)),
          {:ok, 200, _headers, _body} <-
-           Diff.HTTP.retry("gs", fn -> Diff.HTTP.put_stream(url, headers(), stream) end) do
+           Diff.HTTP.retry("gs", fn -> Diff.HTTP.put(url, headers(), diff_data) end) do
       :ok
     else
       {:ok, status, _headers, _body} ->
         Logger.error("Failed to put diff to storage. Status #{status}")
-        {:error, :not_found}
+        {:error, :storage_error}
 
       error ->
         Logger.error("Failed to put diff to storage. Reason #{inspect(error)}")
         error
     end
   end
 
+  def list_diffs(package, from_version, to_version) do
+    case get_metadata(package, from_version, to_version) do
+      {:ok, %{total_diffs: total_diffs}} ->
+        diff_ids = 0..(total_diffs - 1) |> Enum.map(&"diff-#{&1}")
+        {:ok, diff_ids}
+
+      {:error, :not_found} ->
+        {:ok, []}
+
+      error ->
+        error
+    end
+  end
+
+  def get_metadata(package, from_version, to_version) do
+    with {:ok, hash} <- combined_checksum(package, from_version, to_version),
+         url = url(metadata_key(package, from_version, to_version, hash)),
+         {:ok, 200, _headers, body} <-
+           Diff.HTTP.retry("gs", fn -> Diff.HTTP.get(url, headers()) end) do
+      case Jason.decode(body, keys: :atoms) do
+        {:ok, metadata} -> {:ok, metadata}
+        {:error, _} -> {:error, :invalid_metadata}
+      end
+    else
+      {:ok, 404, _headers, _body} ->
+        {:error, :not_found}
+
+      {:ok, status, _headers, _body} ->
+        Logger.error("Failed to get metadata from storage. Status #{status}")
+        {:error, :not_found}
+
+      {:error, reason} ->
+        Logger.error("Failed to get metadata from storage. Reason #{inspect(reason)}")
+        {:error, :not_found}
+    end
+  end
+
+  def put_metadata(package, from_version, to_version, metadata) do
+    with {:ok, hash} <- combined_checksum(package, from_version, to_version),
+         url = url(metadata_key(package, from_version, to_version, hash)),
+         {:ok, json} <- Jason.encode(metadata),
+         {:ok, 200, _headers, _body} <-
+           Diff.HTTP.retry("gs", fn -> Diff.HTTP.put(url, headers(), json) end) do
+      :ok
+    else
+      {:ok, status, _headers, _body} ->
+        Logger.error("Failed to put metadata to storage. Status #{status}")
+        {:error, :storage_error}
+
+      {:error, %Jason.EncodeError{}} ->
+        Logger.error("Failed to encode metadata as JSON")
+        {:error, :invalid_metadata}
+
+      error ->
+        Logger.error("Failed to put metadata to storage. Reason #{inspect(error)}")
+        error
+    end
+  end
+
   defp headers() do
     token = Goth.fetch!(Diff.Goth)
     [{"authorization", "#{token.type} #{token.token}"}]
@@ -53,8 +112,12 @@ defmodule Diff.Storage.GCS do
     end
   end
 
-  defp key(package, from_version, to_version, hash) do
-    "diffs/#{package}-#{from_version}-#{to_version}-#{hash}.html"
+  defp diff_key(package, from_version, to_version, hash, diff_id) do
+    "diffs/#{package}-#{from_version}-#{to_version}-#{hash}-#{diff_id}.json"
+  end
+
+  defp metadata_key(package, from_version, to_version, hash) do
+    "metadata/#{package}-#{from_version}-#{to_version}-#{hash}.json"
   end
 
   defp url(key) do