Share chat history across git worktrees of the same repo

Canonicalize linked worktrees to the main worktree root when keying the
workspace cache and fold old per-worktree cache dirs in on first open.
Since pooling makes concurrent servers on the same cache file common,
writes now read+merge under a cross process file lock instead of
overwriting, with deleted chats tombstoned so merges don't resurrect
them.

Fixes #558

Author: ericdallo

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Share chat history across git worktrees of the same repo, and merge workspace cache writes from concurrent servers instead of overwriting. #558
+
 ## 0.151.1
 
 - Fix git tool heredoc failures for messages with apostrophes on old shells (e.g. macOS bash 3.2); git tool now honors `toolCall.shellCommand` path/args config. #562

src/eca/cache.clj

@@ -40,26 +40,86 @@
                        (io/file (user-home) ".cache"))]
     (io/file cache-home "eca")))
 
+(defn ^:private linked-worktree-root*
+  "When `path` is the root of a *linked* git worktree, returns the repository's
+   main worktree root as a string; otherwise nil.
+
+   Resolved from the filesystem alone (no `git` subprocess, no PATH dependency
+   in the native image): a linked worktree has a `.git` *file* containing a
+   `gitdir:` pointer to `<main>/.git/worktrees/<name>`, whose `commondir` file
+   points back to the shared `.git` directory; the main worktree root is that
+   directory's parent. Anything else (normal clone, bare repo, submodule
+   `.git` file, broken layout) yields nil."
+  [^String path]
+  (try
+    (let [dot-git (fs/file path ".git")]
+      (when (fs/regular-file? dot-git)
+        (when-let [gitdir-str (some #(second (re-find #"^gitdir:\s*(.+?)\s*$" %))
+                                    (fs/read-all-lines dot-git))]
+          (let [gitdir (fs/normalize (if (fs/relative? (fs/path gitdir-str))
+                                       (fs/absolutize (fs/path path gitdir-str))
+                                       (fs/path gitdir-str)))
+                commondir-file (fs/file (str gitdir) "commondir")]
+            (when (fs/regular-file? commondir-file)
+              (when-let [common-str (some->> (fs/read-all-lines commondir-file)
+                                             (remove string/blank?)
+                                             (first)
+                                             (string/trim))]
+                (let [common (fs/normalize (if (fs/relative? (fs/path common-str))
+                                             (fs/absolutize (fs/path gitdir common-str))
+                                             (fs/path common-str)))]
+                  (when (and (= ".git" (fs/file-name common))
+                             (fs/directory? common))
+                    (some-> (fs/parent common) str)))))))))
+    (catch Throwable _ nil)))
+
+(def ^:private linked-worktree-root
+  "Memoized `linked-worktree-root*`: resolved on nearly every chat mutation via
+   `eca.db/update-workspaces-cache!`, and a worktree's gitdir pointer does not
+   change during a server's lifetime."
+  (memoize linked-worktree-root*))
+
+(defn ^:private canonicalize-workspace-path
+  "Canonical identity path for a workspace folder: linked git worktrees map to
+   their repository's main worktree root so chat history is shared across all
+   worktrees of a repo (#558). Anything else returns unchanged, keeping
+   existing cache keys byte-identical."
+  [path]
+  (or (linked-worktree-root path) path))
+
+(defn ^:private raw-workspace-paths
+  "Absolute workspace paths as reported by the editor, without worktree
+   canonicalization."
+  [workspaces uri->filename-fn]
+  (map #(str (fs/absolutize (fs/file (uri->filename-fn (:uri %))))) workspaces))
+
 (defn ^:private sorted-workspace-paths
-  "Absolute workspace paths, de-duplicated and sorted, so the result is stable
-   regardless of the order the editor reports its workspace folders."
+  "Absolute canonical workspace paths, de-duplicated and sorted, so the result
+   is stable regardless of the order the editor reports its workspace folders."
   [workspaces uri->filename-fn]
-  (->> workspaces
-       (map #(str (fs/absolutize (fs/file (uri->filename-fn (:uri %))))))
+  (->> (raw-workspace-paths workspaces uri->filename-fn)
+       (map canonicalize-workspace-path)
        (distinct)
        (sort)))
 
-(defn workspaces-hash
-  "Returns an 8-char base64 (URL-safe, no padding) hash key for the given workspace set.
-   Order-independent: the same set of folders always yields the same hash."
-  [workspaces uri->filename-fn]
-  (let [joined (string/join ":" (sorted-workspace-paths workspaces uri->filename-fn))
+(defn ^:private paths-hash
+  "8-char base64 (URL-safe, no padding) hash key for a seq of paths."
+  [paths]
+  (let [joined (string/join ":" paths)
         digest-bytes (digest/sha-256-bytes joined)
         encoder (-> (java.util.Base64/getUrlEncoder)
                     (.withoutPadding))
         key (.encodeToString encoder digest-bytes)]
     (subs key 0 (min 8 (count key)))))
 
+(defn workspaces-hash
+  "Returns an 8-char base64 (URL-safe, no padding) hash key for the given workspace set.
+   Order-independent: the same set of folders always yields the same hash.
+   Linked git worktrees are canonicalized to their main worktree root, so every
+   worktree of a repository shares the same hash as the repository itself."
+  [workspaces uri->filename-fn]
+  (paths-hash (sorted-workspace-paths workspaces uri->filename-fn)))
+
 (def ^:private logger-tag "[CACHE]")
 
 (def ^:private max-prefix-length 30)
@@ -93,18 +153,24 @@
 (defn redundant-workspace-cache-files
   "Returns the cache files named `filename` that live in directories belonging to
    the same workspace set as the canonical dir but under a different name - i.e.
-   legacy hash-only dirs, or dirs prefixed from a different folder order. The
-   canonical dir is excluded. Used to heal fragmented chat caches."
+   legacy hash-only dirs, dirs prefixed from a different folder order, or dirs
+   keyed by the pre-worktree-canonicalization hash (chats saved when a linked
+   worktree was its own history bucket). The canonical dir is excluded. Used to
+   heal fragmented chat caches."
   [workspaces filename uri->filename-fn]
-  (let [hash (workspaces-hash workspaces uri->filename-fn)
+  (let [canonical-hash (workspaces-hash workspaces uri->filename-fn)
+        raw-hash (paths-hash (sort (distinct (raw-workspace-paths workspaces uri->filename-fn))))
+        hashes (cond-> #{canonical-hash}
+                 (not= raw-hash canonical-hash) (conj raw-hash))
         canonical-dir-name (workspace-dir-name workspaces uri->filename-fn)
         base (global-dir)]
     (if (fs/exists? base)
       (->> (fs/list-dir base)
            (filter fs/directory?)
            (map #(str (fs/file-name %)))
-           (filter (fn [n] (or (= n hash)
-                               (string/ends-with? n (str "_" hash)))))
+           (filter (fn [n] (some (fn [h] (or (= n h)
+                                             (string/ends-with? n (str "_" h))))
+                                 hashes)))
            (remove #(= % canonical-dir-name))
            (map #(io/file base % filename))
            (filter fs/exists?)

src/eca/db.clj

@@ -10,8 +10,8 @@
   (:import
    [java.io OutputStream RandomAccessFile]
    [java.nio.channels FileChannel FileLock]
-   [java.nio.file AtomicMoveNotSupportedException CopyOption Files StandardCopyOption]
-   [java.nio.file.attribute FileAttribute]
+   [java.nio.file AtomicMoveNotSupportedException CopyOption Files LinkOption StandardCopyOption]
+   [java.nio.file.attribute BasicFileAttributes FileAttribute]
    [java.util.concurrent ConcurrentHashMap]))
 
 (set! *warn-on-reflection* true)
@@ -31,6 +31,9 @@
    :providers-config-hash :string
    :last-config-notified ::any-map
    :stopping :boolean
+   ;; chat ids deleted in this session; excluded from workspace cache writes so
+   ;; the merge-on-write never resurrects them from a shared cache file.
+   :deleted-chat-ids #{:string}
    :models {"<model-name>" {:web-search :boolean
                             :tools :boolean
                             :reason? :boolean
@@ -150,6 +153,8 @@
    ;; {tool-name {:remember-to-approve? boolean
    ;;             :remembered-command-keys #{string}}}
    :tool-calls {}
+   ;; Chat ids deleted in this session (not cached), see _db-spec.
+   :deleted-chat-ids #{}
 
    ;; cacheable, bump db `version` when changing any below
    :chats {}
@@ -305,25 +310,91 @@
           {}
           chat-maps))
 
+(defn ^:private with-os-file-lock-fn
+  "Run `f` while holding both a JVM monitor for `lock-file` and an OS advisory
+   exclusive lock on it. The JVM monitor avoids `OverlappingFileLockException`
+   when two threads in the same ECA server race; the file lock serializes
+   across `eca server` processes that share the same cache dir. Blocks until
+   both are acquired."
+  [^java.io.File lock-file f]
+  ;; `file-lock` interns the lock object in `file-locks`, so it is
+  ;; not actually local to this scope; suppress the false positive.
+  #_{:clj-kondo/ignore [:locking-suspicious-lock]}
+  (locking (file-lock lock-file)
+    (io/make-parents lock-file)
+    (let [^RandomAccessFile raf (RandomAccessFile. lock-file "rw")
+          ^FileChannel channel (.getChannel raf)
+          lock-ref (volatile! nil)]
+      (try
+        (vreset! lock-ref ^FileLock (.lock channel))
+        (f)
+        (finally
+          (when-let [^FileLock lock @lock-ref]
+            (try (.release lock)
+                 (catch Throwable e
+                   (logger/warn logger-tag "Could not release cache lock" e))))
+          (try (.close channel) (catch Throwable _))
+          (try (.close raf) (catch Throwable _)))))))
+
+(defn ^:private workspace-cache-lock-file ^java.io.File [^java.io.File cache-file]
+  (io/file (str (.getPath cache-file) ".lock")))
+
+(defonce ^:private ^ConcurrentHashMap last-workspace-write-attrs (ConcurrentHashMap.))
+
+(defn ^:private cache-file-attrs
+  "Returns [last-modified-time size] for `f`, or nil when it does not exist."
+  [^java.io.File f]
+  (try
+    (when (.exists f)
+      (let [^BasicFileAttributes attrs (Files/readAttributes
+                                        (.toPath f)
+                                        BasicFileAttributes
+                                        ^"[Ljava.nio.file.LinkOption;" (into-array LinkOption []))]
+        [(.lastModifiedTime attrs) (.size attrs)]))
+    (catch Throwable _ nil)))
+
+(defn ^:private record-workspace-write-attrs!
+  "Remembers the on-disk attributes of `f` right after this process wrote it,
+   so the next write can cheaply detect whether another process wrote in
+   between (see `workspace-cache-changed-on-disk?`)."
+  [^java.io.File f]
+  (if-let [attrs (cache-file-attrs f)]
+    (.put last-workspace-write-attrs (.getAbsolutePath f) attrs)
+    (.remove last-workspace-write-attrs (.getAbsolutePath f))))
+
+(defn ^:private workspace-cache-changed-on-disk?
+  "True when `f` exists with different attributes than the last write this
+   process made to it - i.e. another process wrote it (or this process never
+   wrote it yet), so its content must be merged instead of overwritten."
+  [^java.io.File f]
+  (let [attrs (cache-file-attrs f)]
+    (boolean (and attrs (not= attrs (.get last-workspace-write-attrs (.getAbsolutePath f)))))))
+
 (defn consolidate-workspace-cache!
   "Heals chat caches that fragmented across multiple directories for the same
-   workspace set (legacy hash-only dirs, or dirs prefixed from a different folder
-   order). Merges every matching cache into the canonical dir (newest chat wins)
-   and removes the redundant dirs. Best-effort and idempotent."
+   workspace set (legacy hash-only dirs, dirs prefixed from a different folder
+   order, or per-worktree dirs from before worktree canonicalization). Merges
+   every matching cache into the canonical dir (newest chat wins) and removes
+   the redundant dirs. Best-effort and idempotent; runs under the workspace
+   cache file lock so it cannot race writes from another live server."
   [workspaces metrics]
   (try
     (let [redundant (cache/redundant-workspace-cache-files workspaces "db.transit.json" shared/uri->filename)]
       (when (seq redundant)
-        (let [canonical (transit-global-by-workspaces-db-file workspaces)
-              caches (keep #(read-cache % metrics) (cons canonical redundant))
-              merged (merge-chats (map :chats caches))]
-          (logger/info logger-tag (str "Consolidating " (count redundant) " redundant workspace cache dir(s) into " canonical))
-          (upsert-cache! {:chats merged :version version} canonical metrics)
-          (doseq [^java.io.File f redundant]
-            (try
-              (fs/delete-tree (.getParentFile f))
-              (catch Throwable e
-                (logger/warn logger-tag (str "Could not remove redundant cache dir " (.getParentFile f)) e)))))))
+        (let [canonical (transit-global-by-workspaces-db-file workspaces)]
+          (with-os-file-lock-fn
+            (workspace-cache-lock-file canonical)
+            (fn []
+              (let [caches (keep #(read-cache % metrics) (cons canonical redundant))
+                    merged (merge-chats (map :chats caches))]
+                (logger/info logger-tag (str "Consolidating " (count redundant) " redundant workspace cache dir(s) into " canonical))
+                (upsert-cache! {:chats merged :version version} canonical metrics)
+                (record-workspace-write-attrs! canonical)
+                (doseq [^java.io.File f redundant]
+                  (try
+                    (fs/delete-tree (.getParentFile f))
+                    (catch Throwable e
+                      (logger/warn logger-tag (str "Could not remove redundant cache dir " (.getParentFile f)) e))))))))))
     (catch Throwable e
       (logger/warn logger-tag "Could not consolidate workspace cache" e))))
 
@@ -363,19 +434,42 @@
 (defn ^:private normalize-db-for-global-write [db]
   (select-keys db [:auth :mcp-auth]))
 
-(defn update-workspaces-cache! [db metrics]
-  (-> (normalize-db-for-workspace-write db)
-      (assoc :version version)
-      (upsert-cache! (transit-global-by-workspaces-db-file (or (:initial-workspace-folders db)
-                                                               (:workspace-folders db))) metrics)))
+(defn update-workspaces-cache!
+  "Persists the workspace-scoped db slice (chats) to the workspace cache file.
+
+   Safe across processes: multiple ECA servers can share one cache file (e.g. a
+   repo and its worktrees, #558), so when the file changed on disk since this
+   process last wrote it, the on-disk chats are merged in (newest wins, ties
+   keep the in-memory copy) instead of blindly overwritten. Chats deleted in
+   this session (`:deleted-chat-ids`) are never resurrected by the merge. Runs
+   under a cross-process file lock; if locking fails it falls back to a plain
+   overwrite (previous behavior)."
+  [db metrics]
+  (let [dest (transit-global-by-workspaces-db-file (or (:initial-workspace-folders db)
+                                                       (:workspace-folders db)))
+        payload (-> (normalize-db-for-workspace-write db)
+                    (assoc :version version))
+        deleted-ids (not-empty (:deleted-chat-ids db))]
+    (try
+      (with-os-file-lock-fn
+        (workspace-cache-lock-file dest)
+        (fn []
+          (let [disk-chats (when (workspace-cache-changed-on-disk? dest)
+                             (:chats (read-cache dest metrics)))
+                chats (cond-> (:chats payload)
+                        (seq disk-chats) (as-> $ (merge-chats [$ disk-chats]))
+                        deleted-ids (as-> $ (apply dissoc $ deleted-ids)))]
+            (upsert-cache! (assoc payload :chats chats) dest metrics)
+            (record-workspace-write-attrs! dest))))
+      (catch Throwable e
+        (logger/warn logger-tag (str "Workspace cache lock failed, writing without merge: " (ex-message e)))
+        (upsert-cache! payload dest metrics)))))
 
 (defn update-global-cache! [db metrics]
   (-> (normalize-db-for-global-write db)
       (assoc :version version)
       (upsert-cache! (transit-global-db-file) metrics)))
 
-(def ^:private global-cache-lock-sentinel (Object.))
-
 (defn ^:private global-cache-lock-file []
   (io/file (cache/global-dir) "db.transit.json.lock"))
 
@@ -386,22 +480,7 @@
    race a renew; the file lock serializes across `eca server` processes
    that share `~/.cache/eca/`. Blocks until both are acquired."
   [f]
-  (locking global-cache-lock-sentinel
-    (let [^java.io.File lock-file (global-cache-lock-file)
-          _ (io/make-parents lock-file)
-          ^RandomAccessFile raf (RandomAccessFile. lock-file "rw")
-          ^FileChannel channel (.getChannel raf)
-          lock-ref (volatile! nil)]
-      (try
-        (vreset! lock-ref ^FileLock (.lock channel))
-        (f)
-        (finally
-          (when-let [^FileLock lock @lock-ref]
-            (try (.release lock)
-                 (catch Throwable e
-                   (logger/warn logger-tag "Could not release global cache lock" e))))
-          (try (.close channel) (catch Throwable _))
-          (try (.close raf) (catch Throwable _)))))))
+  (with-os-file-lock-fn (global-cache-lock-file) f))
 
 (defmacro with-global-cache-lock
   "See `with-global-cache-lock-fn`. Runs `body` while holding the lock."
@@ -438,16 +517,19 @@
   (when (pos? retention-days)
     (let [retention-ms (* retention-days 24 60 60 1000)
           cutoff (- (System/currentTimeMillis) retention-ms)
-          removed (atom 0)]
+          removed-ids* (atom #{})]
       (swap! db* update :chats
              (fn [chats]
                (into {}
-                     (filter (fn [[_id chat]]
+                     (filter (fn [[id chat]]
                                (let [created-at (:created-at chat)]
                                  (if (and created-at (< created-at cutoff))
-                                   (do (swap! removed inc) false)
+                                   (do (swap! removed-ids* conj id) false)
                                    true))))
                      chats)))
-      (when (pos? @removed)
-        (logger/info logger-tag (str "Cleaned up " @removed " chat(s) older than " retention-days " days"))
+      (when-let [removed-ids (not-empty @removed-ids*)]
+        ;; Tombstone the ids so the merge-on-write in update-workspaces-cache!
+        ;; does not resurrect them from a cache file shared with another server.
+        (swap! db* update :deleted-chat-ids (fnil into #{}) removed-ids)
+        (logger/info logger-tag (str "Cleaned up " (count removed-ids) " chat(s) older than " retention-days " days"))
         (update-workspaces-cache! @db* metrics)))))

src/eca/features/chat.clj

@@ -2069,8 +2069,12 @@
                                        {}
                                        db
                                        config)))
-      ;; Delete chat from memory
-      (swap! db* update :chats dissoc chat-id)
+      ;; Delete chat from memory; tombstone the id so the cache merge-on-write
+      ;; never resurrects it from a cache file shared with another live server.
+      (swap! db* (fn [db]
+                   (-> db
+                       (update :chats dissoc chat-id)
+                       (update :deleted-chat-ids (fnil conj #{}) chat-id))))
       (messenger/chat-deleted messenger {:chat-id chat-id})
       ;; Save updated cache (without this chat)
       (db/update-workspaces-cache! @db* metrics))))