Stop preloading cost-intelligence rows on workspace load + add gzip

The workspace page was eagerly fetching up to 5000 EffectiveCost rows
(each with a full resource + sourceObservation join) on every load,
because the always-mounted AssemblyInsertModal called
useAssemblyAuthoringContext() unconditionally. That single request took
~22s and dominated an ~8MB, 40s page load — even though the assembly
picker was never opened and the cost dropdown already does indexed
dynamic search.

Three fixes:
1. Gate useAssemblyAuthoringContext on an `enabled` flag; the modal
   passes `open`, so the heavy picker data (catalogs, rates, labor
   units, effective costs) only loads when the picker is actually opened.
2. Add @fastify/compress (global gzip/brotli, 1KB threshold) so large
   JSON payloads no longer ship uncompressed. SSE routes use
   reply.hijack() and are unaffected; already-compressed file types are
   skipped by the mime filter.
3. Add an includeObservation flag to listEffectiveCosts; the assembly
   path passes false to skip the large sourceObservation (rawText) join
   it never uses.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: 2 people

apps/api/package.json

@@ -19,6 +19,7 @@
     "@bidwright/integrations": "workspace:*",
     "@bidwright/vector": "workspace:*",
     "@bidwright/vision": "workspace:*",
+    "@fastify/compress": "^8.0.1",
     "@fastify/cors": "^10.0.2",
     "@fastify/multipart": "^9.3.0",
     "@fastify/rate-limit": "^10.2.1",

apps/api/src/routes/cost-intelligence-routes.ts

@@ -15,6 +15,8 @@ const listQuerySchema = z.object({
   vendorName: z.string().optional(),
   scope: z.enum(["aggregate", "per_vendor", "all"]).optional(),
   limit: z.coerce.number().int().positive().max(50000).optional(),
+  // String query param: "false" disables the heavy sourceObservation join.
+  includeObservation: z.enum(["true", "false"]).optional(),
 });
 
 const resourceSchema = z.object({
@@ -501,6 +503,7 @@ export async function costIntelligenceRoutes(app: FastifyInstance): Promise<void
       vendorName: parsed.data.vendorName,
       scope: parsed.data.scope,
       limit: parsed.data.limit,
+      includeObservation: parsed.data.includeObservation !== "false",
     });
   });
 

apps/api/src/server.ts

@@ -1,3 +1,4 @@
+import compress from "@fastify/compress";
 import cors from "@fastify/cors";
 import multipart from "@fastify/multipart";
 import rateLimit from "@fastify/rate-limit";
@@ -1848,6 +1849,11 @@ export function buildServer() {
     await prisma.$disconnect();
   });
 
+  // Gzip/brotli response compression. Workspace + cost/library payloads are
+  // large JSON (often several MB); without this they ship uncompressed. Only
+  // kicks in above the threshold and when the client sends Accept-Encoding.
+  app.register(compress, { global: true, threshold: 1024 });
+
   app.register(cors, {
     origin: true,
     credentials: true

apps/api/src/services/cost-intelligence-service.ts

@@ -132,6 +132,10 @@ export interface CostIntelligenceListFilters {
   vendorName?: string;
   limit?: number;
   scope?: "aggregate" | "per_vendor" | "all";
+  /** When false, skip the heavy sourceObservation join (rawText etc.). Callers
+   *  that only need cost + resource fields (e.g. the assembly pickers) pass
+   *  false to avoid pulling thousands of large observation rows. */
+  includeObservation?: boolean;
 }
 
 export interface VendorPdfIngestFile {
@@ -3026,7 +3030,9 @@ export class CostIntelligenceService {
       orderBy: [{ updatedAt: "desc" }],
       include: {
         resource: true,
-        sourceObservation: true,
+        // Only join the (potentially large) observation when the caller needs
+        // it. The assembly pickers pull thousands of rows and use none of it.
+        ...(filters.includeObservation === false ? {} : { sourceObservation: true }),
       },
       take: clampLimit(filters.limit),
     });

apps/web/components/assembly-authoring.tsx

@@ -170,14 +170,21 @@ function parseMarkupPercent(value: string): number | null | undefined {
 // Loads assembly component sources from the API. Used by both the
 // settings-page authoring view and the estimate-page quick-create flow so the
 // pickers are populated identically.
-export function useAssemblyAuthoringContext() {
+export function useAssemblyAuthoringContext(options?: { enabled?: boolean }) {
+  // Gate the fetch so this heavy data (catalogs + rate schedules + labor units +
+  // thousands of effective costs) is only pulled when actually needed — e.g.
+  // when the assembly picker opens — instead of eagerly on every page that
+  // happens to mount a (closed) consumer. Defaults to enabled for standalone
+  // pages like the assembly manager.
+  const enabled = options?.enabled ?? true;
   const [catalogItems, setCatalogItems] = useState<CatalogItemRow[]>([]);
   const [rateItems, setRateItems] = useState<RateItemRow[]>([]);
   const [laborUnits, setLaborUnits] = useState<LaborUnitRow[]>([]);
   const [effectiveCosts, setEffectiveCosts] = useState<EffectiveCostRow[]>([]);
   const [loaded, setLoaded] = useState(false);
 
   useEffect(() => {
+    if (!enabled || loaded) return;
     let cancelled = false;
     (async () => {
       try {
@@ -247,7 +254,9 @@ export function useAssemblyAuthoringContext() {
       }
 
       try {
-        const rows = await listEffectiveCosts({ limit: 5000 });
+        // includeObservation:false skips the heavy sourceObservation join —
+        // the picker only needs cost + resource fields.
+        const rows = await listEffectiveCosts({ limit: 5000, includeObservation: false });
         const flat: EffectiveCostRow[] = rows.map((cost: EffectiveCostRecord) => {
           const item = effectiveCostItem(cost);
           return {
@@ -280,7 +289,7 @@ export function useAssemblyAuthoringContext() {
     return () => {
       cancelled = true;
     };
-  }, []);
+  }, [enabled, loaded]);
 
   return { catalogItems, rateItems, laborUnits, effectiveCosts, loaded };
 }

apps/web/components/workspace/assembly-insert-modal.tsx

@@ -70,7 +70,10 @@ export function AssemblyInsertModal({
   const [editInstanceParams, setEditInstanceParams] = useState<Record<string, string>>({});
   const [instanceBusy, setInstanceBusy] = useState(false);
 
-  const { catalogItems, rateItems, laborUnits, effectiveCosts } = useAssemblyAuthoringContext();
+  // Only load the heavy picker data once the modal is actually opened — the
+  // modal stays mounted (closed) in the estimate grid, so an unconditional
+  // fetch would pull thousands of cost rows on every workspace page load.
+  const { catalogItems, rateItems, laborUnits, effectiveCosts } = useAssemblyAuthoringContext({ enabled: open });
 
   const refreshList = useCallback(async () => {
     setLoading(true);

apps/web/lib/api.ts

@@ -7361,6 +7361,7 @@ export async function listEffectiveCosts(input: {
   vendorName?: string;
   scope?: "aggregate" | "per_vendor" | "all";
   limit?: number;
+  includeObservation?: boolean;
 } = {}): Promise<EffectiveCostRecord[]> {
   const params = new URLSearchParams();
   for (const [key, value] of Object.entries(input)) {