Fix: ts upload file does not create index and document store (#422)

Author: leehuwuj

.changeset/big-turtles-own.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Ensure that the index and document store are created when uploading a file with no available index.

helpers/env-variables.ts

@@ -217,7 +217,13 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`,
         },
       ];
     default:
-      return [];
+      return [
+        {
+          name: "STORAGE_CACHE_DIR",
+          description: "The directory to store the local storage cache.",
+          value: ".cache",
+        },
+      ];
   }
 };
 

templates/components/llamaindex/typescript/documents/pipeline.ts

@@ -3,6 +3,7 @@ import {
   IngestionPipeline,
   Settings,
   SimpleNodeParser,
+  storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
 
@@ -28,11 +29,20 @@ export async function runPipeline(
     return documents.map((document) => document.id_);
   } else {
     // Initialize a new index with the documents
-    const newIndex = await VectorStoreIndex.fromDocuments(documents);
-    newIndex.storageContext.docStore.persist();
     console.log(
       "Got empty index, created new index with the uploaded documents",
     );
+    const persistDir = process.env.STORAGE_CACHE_DIR;
+    if (!persistDir) {
+      throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+    }
+    const storageContext = await storageContextFromDefaults({
+      persistDir,
+    });
+    const newIndex = await VectorStoreIndex.fromDocuments(documents, {
+      storageContext,
+    });
+    await newIndex.storageContext.docStore.persist();
     return documents.map((document) => document.id_);
   }
 }

templates/components/vectordbs/typescript/none/generate.ts

@@ -5,7 +5,6 @@ import * as dotenv from "dotenv";
 
 import { getDocuments } from "./loader";
 import { initSettings } from "./settings";
-import { STORAGE_CACHE_DIR } from "./shared";
 
 // Load environment variables from local .env file
 dotenv.config();
@@ -20,9 +19,13 @@ async function getRuntime(func: any) {
 async function generateDatasource() {
   console.log(`Generating storage context...`);
   // Split documents, create embeddings and store them in the storage context
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const ms = await getRuntime(async () => {
     const storageContext = await storageContextFromDefaults({
-      persistDir: STORAGE_CACHE_DIR,
+      persistDir,
     });
     const documents = await getDocuments();
 

templates/components/vectordbs/typescript/none/index.ts

@@ -1,10 +1,13 @@
 import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex";
 import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
-import { STORAGE_CACHE_DIR } from "./shared";
 
 export async function getDataSource(params?: any) {
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const storageContext = await storageContextFromDefaults({
-    persistDir: `${STORAGE_CACHE_DIR}`,
+    persistDir,
   });
 
   const numberOfDocs = Object.keys(