fix: report import step failures correctly (#1303)

Author: CaralHsi

apps/memos-local-openclaw/src/viewer/server.ts

@@ -18,9 +18,72 @@ import type { Logger, Chunk, PluginContext } from "../types";
 import { viewerHTML } from "./html";
 import { v4 as uuid } from "uuid";
 
-function normalizeTimestamp(ts: number): number {
-  if (ts < 1e12) return ts * 1000;
-  return ts;
+export interface MigrationStepFailureCounts {
+  summarization: number;
+  dedup: number;
+  embedding: number;
+}
+
+export interface MigrationStateSnapshot {
+  phase: string;
+  stored: number;
+  skipped: number;
+  merged: number;
+  errors: number;
+  processed: number;
+  total: number;
+  lastItem: any;
+  done: boolean;
+  stopped: boolean;
+  stepFailures: MigrationStepFailureCounts;
+  success: boolean;
+}
+
+function createInitialStepFailures(): MigrationStepFailureCounts {
+  return { summarization: 0, dedup: 0, embedding: 0 };
+}
+
+export function computeMigrationSuccess(state: Pick<MigrationStateSnapshot, "errors" | "stepFailures">): boolean {
+  const sf = state.stepFailures;
+  return state.errors === 0 && sf.summarization === 0 && sf.dedup === 0 && sf.embedding === 0;
+}
+
+export function createInitialMigrationState(): MigrationStateSnapshot {
+  const stepFailures = createInitialStepFailures();
+  return {
+    phase: "",
+    stored: 0,
+    skipped: 0,
+    merged: 0,
+    errors: 0,
+    processed: 0,
+    total: 0,
+    lastItem: null,
+    done: false,
+    stopped: false,
+    stepFailures,
+    success: computeMigrationSuccess({ errors: 0, stepFailures }),
+  };
+}
+
+export function applyMigrationItemToState(state: MigrationStateSnapshot, d: any): void {
+  if (d.status === "stored") state.stored++;
+  else if (d.status === "skipped" || d.status === "duplicate") state.skipped++;
+  else if (d.status === "merged") state.merged++;
+  else if (d.status === "error") state.errors++;
+
+  if (Array.isArray(d.stepFailures)) {
+    for (const step of d.stepFailures) {
+      if (step === "summarization") state.stepFailures.summarization++;
+      else if (step === "dedup") state.stepFailures.dedup++;
+      else if (step === "embedding") state.stepFailures.embedding++;
+    }
+  }
+
+  state.processed = d.index ?? state.processed + 1;
+  state.total = d.total ?? state.total;
+  state.lastItem = d;
+  state.success = computeMigrationSuccess(state);
 }
 
 export interface ViewerServerOptions {
@@ -60,18 +123,7 @@ export class ViewerServer {
   private resetToken: string;
   private migrationRunning = false;
   private migrationAbort = false;
-  private migrationState: {
-    phase: string;
-    stored: number;
-    skipped: number;
-    merged: number;
-    errors: number;
-    processed: number;
-    total: number;
-    lastItem: any;
-    done: boolean;
-    stopped: boolean;
-  } = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+  private migrationState: MigrationStateSnapshot = createInitialMigrationState();
   private migrationSSEClients: http.ServerResponse[] = [];
 
   private ppRunning = false;
@@ -1641,7 +1693,7 @@ export class ViewerServer {
     } else if (this.migrationState.done) {
       const evtName = this.migrationState.stopped ? "stopped" : "done";
       res.write(`event: state\ndata: ${JSON.stringify(this.migrationState)}\n\n`);
-      res.write(`event: ${evtName}\ndata: ${JSON.stringify({ ok: true })}\n\n`);
+      res.write(`event: ${evtName}\ndata: ${JSON.stringify({ ok: this.migrationState.success, ...this.migrationState })}\n\n`);
       res.end();
     } else {
       res.end();
@@ -1682,19 +1734,12 @@ export class ViewerServer {
         this.migrationSSEClients = this.migrationSSEClients.filter(c => c !== res);
       });
 
-      this.migrationAbort = false;
-      this.migrationState = { phase: "", stored: 0, skipped: 0, merged: 0, errors: 0, processed: 0, total: 0, lastItem: null, done: false, stopped: false };
+      this.migrationState = createInitialMigrationState();
 
       const send = (event: string, data: unknown) => {
         if (event === "item") {
           const d = data as any;
-          if (d.status === "stored") this.migrationState.stored++;
-          else if (d.status === "skipped" || d.status === "duplicate") this.migrationState.skipped++;
-          else if (d.status === "merged") this.migrationState.merged++;
-          else if (d.status === "error") this.migrationState.errors++;
-          this.migrationState.processed = d.index ?? this.migrationState.processed + 1;
-          this.migrationState.total = d.total ?? this.migrationState.total;
-          this.migrationState.lastItem = d;
+          applyMigrationItemToState(this.migrationState, d);
         } else if (event === "phase") {
           this.migrationState.phase = (data as any).phase;
         } else if (event === "progress") {
@@ -1707,11 +1752,13 @@ export class ViewerServer {
       this.runMigration(send, opts.sources, concurrency).finally(() => {
         this.migrationRunning = false;
         this.migrationState.done = true;
+        this.migrationState.success = computeMigrationSuccess(this.migrationState);
+        const donePayload = { ok: this.migrationState.success, ...this.migrationState };
         if (this.migrationAbort) {
           this.migrationState.stopped = true;
-          this.broadcastSSE("stopped", { ok: true, ...this.migrationState });
+          this.broadcastSSE("stopped", donePayload);
         } else {
-          this.broadcastSSE("done", { ok: true });
+          this.broadcastSSE("done", donePayload);
         }
         this.migrationAbort = false;
         const clientsToClose = [...this.migrationSSEClients];
@@ -1808,11 +1855,24 @@ export class ViewerServer {
               }
 
               try {
-                const summary = await summarizer.summarize(row.text);
+                const stepFailures: Array<"summarization" | "dedup" | "embedding"> = [];
+                let summary = "";
+                try {
+                  summary = await summarizer.summarize(row.text);
+                } catch (err) {
+                  stepFailures.push("summarization");
+                  this.log.warn(`Migration summarization failed: ${err}`);
+                }
+                if (!summary) {
+                  stepFailures.push("summarization");
+                  summary = row.text.slice(0, 200);
+                }
+
                 let embedding: number[] | null = null;
                 try {
                   [embedding] = await this.embedder.embed([summary]);
                 } catch (err) {
+                  stepFailures.push("embedding");
                   this.log.warn(`Migration embed failed: ${err}`);
                 }
 
@@ -1831,26 +1891,31 @@ export class ViewerServer {
                     }).filter(c => c.summary);
 
                     if (candidates.length > 0) {
-                      const dedupResult = await summarizer.judgeDedup(summary, candidates);
-                      if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
-                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
-                        if (targetId) {
-                          dedupStatus = "duplicate";
-                          dedupTarget = targetId;
-                          dedupReason = dedupResult.reason;
-                        }
-                      } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
-                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
-                        if (targetId) {
-                          this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text);
-                          try {
-                            const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
-                            if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
-                          } catch { /* best-effort */ }
-                          dedupStatus = "merged";
-                          dedupTarget = targetId;
-                          dedupReason = dedupResult.reason;
+                      try {
+                        const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                        if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                          const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                          if (targetId) {
+                            dedupStatus = "duplicate";
+                            dedupTarget = targetId;
+                            dedupReason = dedupResult.reason;
+                          }
+                        } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                          const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                          if (targetId) {
+                            this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, row.text);
+                            try {
+                              const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]);
+                              if (newEmb) this.store.upsertEmbedding(targetId, newEmb);
+                            } catch { /* best-effort */ }
+                            dedupStatus = "merged";
+                            dedupTarget = targetId;
+                            dedupReason = dedupResult.reason;
+                          }
                         }
+                      } catch (err) {
+                        stepFailures.push("dedup");
+                        this.log.warn(`Migration dedup judgment failed: ${err}`);
                       }
                     }
                   }
@@ -1893,7 +1958,13 @@ export class ViewerServer {
                   preview: row.text.slice(0, 120),
                   summary: summary.slice(0, 80),
                   source: file,
+                  stepFailures,
                 });
+                if (stepFailures.length > 0) {
+                  this.log.warn(`[MIGRATION] sqlite item imported with step failures: ${stepFailures.join(",")}`);
+                } else {
+                  this.log.info("[MIGRATION] sqlite item imported successfully (all steps)");
+                }
               } catch (err) {
                 totalErrors++;
                 send("item", {
@@ -2023,11 +2094,24 @@ export class ViewerServer {
               }
 
               try {
-                const summary = await summarizer.summarize(content);
+                const stepFailures: Array<"summarization" | "dedup" | "embedding"> = [];
+                let summary = "";
+                try {
+                  summary = await summarizer.summarize(content);
+                } catch (err) {
+                  stepFailures.push("summarization");
+                  this.log.warn(`Migration summarization failed: ${err}`);
+                }
+                if (!summary) {
+                  stepFailures.push("summarization");
+                  summary = content.slice(0, 200);
+                }
+
                 let embedding: number[] | null = null;
                 try {
                   [embedding] = await this.embedder.embed([summary]);
                 } catch (err) {
+                  stepFailures.push("embedding");
                   this.log.warn(`Migration embed failed: ${err}`);
                 }
 
@@ -2046,17 +2130,22 @@ export class ViewerServer {
                     }).filter(c => c.summary);
 
                     if (candidates.length > 0) {
-                      const dedupResult = await summarizer.judgeDedup(summary, candidates);
-                      if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
-                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
-                        if (targetId) { dedupStatus = "duplicate"; dedupTarget = targetId; dedupReason = dedupResult.reason; }
-                      } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
-                        const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
-                        if (targetId) {
-                          this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
-                          try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ }
-                          dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason;
+                      try {
+                        const dedupResult = await summarizer.judgeDedup(summary, candidates);
+                        if (dedupResult?.action === "DUPLICATE" && dedupResult.targetIndex) {
+                          const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                          if (targetId) { dedupStatus = "duplicate"; dedupTarget = targetId; dedupReason = dedupResult.reason; }
+                        } else if (dedupResult?.action === "UPDATE" && dedupResult.targetIndex && dedupResult.mergedSummary) {
+                          const targetId = candidates[dedupResult.targetIndex - 1]?.chunkId;
+                          if (targetId) {
+                            this.store.updateChunkSummaryAndContent(targetId, dedupResult.mergedSummary, content);
+                            try { const [newEmb] = await this.embedder.embed([dedupResult.mergedSummary]); if (newEmb) this.store.upsertEmbedding(targetId, newEmb); } catch { /* best-effort */ }
+                            dedupStatus = "merged"; dedupTarget = targetId; dedupReason = dedupResult.reason;
+                          }
                         }
+                      } catch (err) {
+                        stepFailures.push("dedup");
+                        this.log.warn(`Migration dedup judgment failed: ${err}`);
                       }
                     }
                   }
@@ -2076,7 +2165,12 @@ export class ViewerServer {
                 if (embedding && dedupStatus === "active") this.store.upsertEmbedding(chunkId, embedding);
 
                 totalStored++;
-                send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole });
+                send("item", { index: idx, total: totalMsgs, status: dedupStatus === "active" ? "stored" : dedupStatus, preview: content.slice(0, 120), summary: summary.slice(0, 80), source: file, agent: agentId, role: msgRole, stepFailures });
+                if (stepFailures.length > 0) {
+                  this.log.warn(`[MIGRATION] session item imported with step failures: ${stepFailures.join(",")}`);
+                } else {
+                  this.log.info("[MIGRATION] session item imported successfully (all steps)");
+                }
               } catch (err) {
                 totalErrors++;
                 send("item", { index: idx, total: totalMsgs, status: "error", preview: content.slice(0, 120), source: file, agent: agentId, error: String(err).slice(0, 200) });
@@ -2117,7 +2211,14 @@ export class ViewerServer {
     }
 
     send("progress", { total: totalProcessed, processed: totalProcessed, phase: "done" });
-    send("summary", { totalProcessed, totalStored, totalSkipped, totalErrors });
+    send("summary", {
+      totalProcessed,
+      totalStored,
+      totalSkipped,
+      totalErrors,
+      success: computeMigrationSuccess(this.migrationState),
+      stepFailures: this.migrationState.stepFailures,
+    });
   }
 
   // ─── Post-processing: independent task/skill generation ───

apps/memos-local-openclaw/tests/migration-status.test.ts

@@ -0,0 +1,62 @@
+import { describe, it, expect } from "vitest";
+import {
+  applyMigrationItemToState,
+  computeMigrationSuccess,
+  createInitialMigrationState,
+} from "../src/viewer/server";
+
+describe("migration status step failure reporting", () => {
+  it("reports success=false when step failures exist even if no fatal errors", () => {
+    const state = createInitialMigrationState();
+
+    applyMigrationItemToState(state, {
+      status: "stored",
+      index: 1,
+      total: 2,
+      stepFailures: ["embedding"],
+    });
+
+    expect(state.errors).toBe(0);
+    expect(state.stepFailures.embedding).toBe(1);
+    expect(state.success).toBe(false);
+    expect(computeMigrationSuccess(state)).toBe(false);
+  });
+
+  it("keeps success=true for clean imports and flips to false when dedup/summarization fail", () => {
+    const state = createInitialMigrationState();
+
+    applyMigrationItemToState(state, {
+      status: "stored",
+      index: 1,
+      total: 3,
+      stepFailures: [],
+    });
+
+    expect(state.success).toBe(true);
+
+    applyMigrationItemToState(state, {
+      status: "stored",
+      index: 2,
+      total: 3,
+      stepFailures: ["dedup", "summarization"],
+    });
+
+    expect(state.stepFailures.dedup).toBe(1);
+    expect(state.stepFailures.summarization).toBe(1);
+    expect(state.success).toBe(false);
+  });
+
+  it("counts explicit item errors and reports failure", () => {
+    const state = createInitialMigrationState();
+
+    applyMigrationItemToState(state, {
+      status: "error",
+      index: 1,
+      total: 1,
+      stepFailures: [],
+    });
+
+    expect(state.errors).toBe(1);
+    expect(state.success).toBe(false);
+  });
+});