[feature branch] Search Content API (Epic EAI-956) (#786)

* stub searchContent route

* EAI-973: Persist search queries and results in mongodb (#804)

* Create MongoDbSearchResultsStore, add limit to DefaultFindContent and add test for limit

* Implement saveSearchResult, create MongoDbSearchResultsStore.test.ts

* lint, format

* Check entire returned document in MongoDbSearchResultsStore.test.ts

* Create ResultChunk type and zod check

* Correct usage of limit in makeDefaultFindContent

* PR feedback: cast badSearchResultRecord as any

* Use unknown instead of any for ResultChunk additional metadata

* PR feedback: Combine describe blocks in MongoDbSearchResultsStore.test.ts, remove zod checks where unnecessary in MongoDbSearchResultsStore

* Combine describes

* EAI-970: Expose search endpoint (#810)

* Create MongoDbSearchResultsStore, add limit to DefaultFindContent and add test for limit

* Implement saveSearchResult, create MongoDbSearchResultsStore.test.ts

* lint, format

* Check entire returned document in MongoDbSearchResultsStore.test.ts

* Create ResultChunk type and zod check

* Correct usage of limit in makeDefaultFindContent

* PR feedback: cast badSearchResultRecord as any

* Starting structure of searchContent

* Use unknown instead of any for ResultChunk additional metadata

* Add searchContent test file, broaden QueryFilter & MongoDbAtlasVectorSearchFilter types

* Work on clarity of comments in contentRouter

* PR feedback: Combine describe blocks in MongoDbSearchResultsStore.test.ts, remove zod checks where unnecessary in MongoDbSearchResultsStore

* Use generics on middleware: requireRequestOrigin, requireValidIpAddress

* Structure out contentRouter test file

* Combine describes

* Clean

* makeFindContentWithMongoDbMetadata

* config.test.ts

* Clean

* PR feedback

* Correct types

* Correct test

* Fix test return type

* lint

* Revert move of classifyMongoDbProgrammingLanguageAndProduct, jest needs function outside of file to mock

* Remove unnecessary tests and comments

* (EAI-971) Add `searchContent` to openapi spec documentation (#823)

* Add searchContent pieces to openapi spec

* PR feedback

* (EAI-1159) Implement middleware in new `contentRouter` (#816)

* Create MongoDbSearchResultsStore, add limit to DefaultFindContent and add test for limit

* Implement saveSearchResult, create MongoDbSearchResultsStore.test.ts

* lint, format

* Check entire returned document in MongoDbSearchResultsStore.test.ts

* Create ResultChunk type and zod check

* Correct usage of limit in makeDefaultFindContent

* PR feedback: cast badSearchResultRecord as any

* Starting structure of searchContent

* Use unknown instead of any for ResultChunk additional metadata

* Add searchContent test file, broaden QueryFilter & MongoDbAtlasVectorSearchFilter types

* Work on clarity of comments in contentRouter

* PR feedback: Combine describe blocks in MongoDbSearchResultsStore.test.ts, remove zod checks where unnecessary in MongoDbSearchResultsStore

* Use generics on middleware: requireRequestOrigin, requireValidIpAddress

* Structure out contentRouter test file

* Combine describes

* Clean

* makeFindContentWithMongoDbMetadata

* config.test.ts

* Clean

* PR feedback

* Correct types

* Correct test

* Fix test return type

* lint

* Revert move of classifyMongoDbProgrammingLanguageAndProduct, jest needs function outside of file to mock

* Created addCustomData.ts, generics, use in both contentRouter and conversationRouter

* Clean

* Remove unnecessary tests and comments

* Added custom middleware to contentRouter, used in searchContent route, added to tests

* Add customData to db...

* Clean: allow undefined customData value

* Alter types for createConversationsMiddlewareReq

* Rerun tests

* PR feedback

* Add Locals types to middleware invocations

* Lint, fix trace name, remove unnecessary import

* (EAI-972) Add extra braintrust tracing to searchContent route (#822)

Add extra braintrust tracing to searchContent route

* Use safely parsed req.body, handle possibly undefined dataSources

* (EAI-1173) Content sources endpoint (#833)

* Create listDataSources route, plug into searchContent, blank file created for listDatasources test file

* Clean sources endpoint and data types

* Add openapi spec documentation for new listDataSources endpoint

* listDataSources test file

* PR feedback and add isCurrent

* Clean yaml

* Rename version -> versions

* PR feedback

* ci

* Change embedding search index

* Use plain memory server

* Fix tests

* Make unused parameter obvious

* Use cache in listDataSources

* Add tests for cache

* Strong typing on atlas search filters (#860)

Co-authored-by: Ben Perlmutter <[email protected]>

---------

Co-authored-by: Ben Perlmutter <[email protected]>
Co-authored-by: mmeigs <[email protected]>

Author: 3 people

docs/docs/server/openapi.yaml

@@ -17,9 +17,104 @@ servers:
 security:
   - CustomHeaderAuth: []
 paths:
+  /content/search:
+    post:
+      operationId: searchContent
+      tags:
+        - Content
+      summary: Search content
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                query:
+                  type: string
+                  description: The search query string.
+                dataSources:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      name:
+                        type: string
+                      type:
+                        type: string
+                      versionLabel:
+                        type: string
+                    required: [name]
+                  description: An array of data sources to search. If not provided, latest version of all data sources will be searched.
+                limit:
+                  type: integer
+                  minimum: 1
+                  maximum: 100
+                  default: 5
+                  description: The maximum number of results to return.
+            required:
+              - query
+      responses:
+        200:
+          description: OK
+          headers:
+            Content-Type:
+              schema:
+                type: string
+                example: application/json
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SearchResponse"
+        400:
+          description: Bad Request
+          headers:
+            Content-Type:
+              schema:
+                type: string
+                example: application/json
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/responses/BadRequest"
+        500:
+          description: Internal Server Error
+          headers:
+            Content-Type:
+              schema:
+                type: string
+                example: application/json
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/responses/InternalServerError"
+
+  /content/sources:
+    get:
+      operationId: listDataSources
+      tags:
+        - Content
+      summary: List available data sources
+      description: Returns metadata about all available data sources.
+      responses:
+        200:
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ListDataSourcesResponse"
+        500:
+          description: Internal Server Error
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorResponse"
+
   /conversations:
     post:
       operationId: createConversation
+      tags:
+        - Conversations
       summary: Start new conversation
       description: |
         Start a new conversation.
@@ -43,6 +138,8 @@ paths:
   /conversations/{conversationId}/messages:
     post:
       operationId: addMessage
+      tags:
+        - Conversations
       summary: Add message to the conversation
       tags:
         - Conversations
@@ -133,6 +230,8 @@ paths:
   # /conversations/{conversationId}:
   #   get:
   #     operationId: getConversation
+  #     tags:
+  #       - Conversations
   #     summary: Get a conversation
   #     parameters:
   #       - $ref: "#/components/parameters/conversationId"
@@ -152,6 +251,8 @@ paths:
   /conversations/{conversationId}/messages/{messageId}/rating:
     post:
       operationId: rateMessage
+      tags:
+        - Conversations
       summary: Rate message
       tags:
         - Conversations
@@ -176,6 +277,8 @@ paths:
   /conversations/{conversationId}/messages/{messageId}/comment:
     post:
       operationId: commentMessage
+      tags:
+        - Conversations
       summary: Add comment to assistant message
       tags:
         - Conversations
@@ -972,6 +1075,68 @@ components:
       required: [type, call_id, output, status]
     ErrorResponse:
       type: object
+    SearchResponse:
+      type: object
+      properties:
+        results:
+          type: array
+          items:
+            $ref: "#/components/schemas/Chunk"
+    Chunk:
+      type: object
+      properties:
+        url:
+          type: string
+          description: The URL of the search result.
+        title:
+          type: string
+          description: Title of the search result.
+        text:
+          type: string
+          description: Chunk text
+        metadata:
+          type: object
+          properties:
+            sourceName:
+              type: string
+              description: The name of the source.
+            sourceType:
+              type: string
+            tags:
+              type: array
+              items:
+                type: string
+          additionalProperties: true
+    ListDataSourcesResponse:
+      type: object
+      properties:
+        dataSources:
+          type: array
+          items:
+            $ref: "#/components/schemas/DataSourceMetadata"
+    DataSourceMetadata:
+      type: object
+      required:
+        - id
+      properties:
+        id:
+          type: string
+          description: The name of the data source.
+        versions:
+          type: array
+          items:
+            type: object
+            properties:
+              label:
+                type: string
+                description: Version label
+              isCurrent:
+                type: boolean
+                description: Whether this version is current active version.
+          description: List of versions for this data source.
+        type:
+          type: string
+          description: The type of the data source.
   parameters:
     conversationId:
       name: conversationId
@@ -987,3 +1152,19 @@ components:
       schema:
         type: string
       description: The unique identifier for a message.
+
+tags:
+  - name: Content
+    x-displayName: Search Content
+    description: Search MongoDB content
+  - name: Conversations
+    x-displayName: Conversations
+    description: Interact with MongoDB Chatbot
+
+x-tagGroups:
+  - name: Content
+    tags:
+      - Content
+  - name: Conversations
+    tags:
+      - Conversations

packages/chatbot-server-mongodb-public/src/config.ts

@@ -17,9 +17,10 @@ import {
   AddCustomDataFunc,
   FilterPreviousMessages,
   makeDefaultFindVerifiedAnswer,
-  defaultCreateConversationCustomData,
-  defaultAddMessageToConversationCustomData,
   makeVerifiedAnswerGenerateResponse,
+  addDefaultCustomData,
+  ConversationsRouterLocals,
+  ContentRouterLocals,
   addMessageToConversationVerifiedAnswerStream,
   responsesVerifiedAnswerStream,
   type MakeVerifiedAnswerGenerateResponseParams,
@@ -34,8 +35,18 @@ import {
 import { redactConnectionUri } from "./middleware/redactConnectionUri";
 import path from "path";
 import express from "express";
-import { makeMongoDbPageStore, logger } from "mongodb-rag-core";
-import { wrapOpenAI, wrapTraced } from "mongodb-rag-core/braintrust";
+import {
+  makeMongoDbPageStore,
+  makeMongoDbSearchResultsStore,
+  logger,
+} from "mongodb-rag-core";
+import { createAzure, wrapLanguageModel } from "mongodb-rag-core/aiSdk";
+import {
+  makeBraintrustLogger,
+  BraintrustMiddleware,
+  wrapOpenAI,
+  wrapTraced,
+} from "mongodb-rag-core/braintrust";
 import { AzureOpenAI } from "mongodb-rag-core/openai";
 import { MongoClient } from "mongodb-rag-core/mongodb";
 import {
@@ -57,13 +68,9 @@ import {
   responsesApiStream,
   addMessageToConversationStream,
 } from "./processors/generateResponseWithTools";
-import {
-  makeBraintrustLogger,
-  BraintrustMiddleware,
-} from "mongodb-rag-core/braintrust";
 import { makeMongoDbScrubbedMessageStore } from "./tracing/scrubbedMessages/MongoDbScrubbedMessageStore";
 import { MessageAnalysis } from "./tracing/scrubbedMessages/analyzeMessage";
-import { createAzure, wrapLanguageModel } from "mongodb-rag-core/aiSdk";
+import { makeFindContentWithMongoDbMetadata } from "./processors/findContentWithMongoDbMetadata";
 import { makeMongoDbAssistantSystemPrompt } from "./systemPrompt";
 import { makeFetchPageTool } from "./tools/fetchPage";
 import { makeCorsOptions } from "./corsOptions";
@@ -129,6 +136,11 @@ export const embeddedContentStore = makeMongoDbEmbeddedContentStore({
   },
 });
 
+export const searchResultsStore = makeMongoDbSearchResultsStore({
+  connectionUri: MONGODB_CONNECTION_URI,
+  databaseName: MONGODB_DATABASE_NAME,
+});
+
 export const verifiedAnswerConfig = {
   embeddingModel: OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT,
   findNearestNeighborsOptions: {
@@ -306,7 +318,7 @@ export const makeGenerateResponse = (args?: MakeGenerateResponseParams) =>
 
 export const createConversationCustomDataWithAuthUser: AddCustomDataFunc =
   async (req, res) => {
-    const customData = await defaultCreateConversationCustomData(req, res);
+    const customData = await addDefaultCustomData(req, res);
     if (req.cookies.auth_user) {
       customData.authUser = req.cookies.auth_user;
     }
@@ -350,11 +362,20 @@ export async function closeDbConnections() {
 logger.info(`Segment logging is ${segmentConfig ? "enabled" : "disabled"}`);
 
 export const config: AppConfig = {
+  contentRouterConfig: {
+    findContent: makeFindContentWithMongoDbMetadata({
+      findContent,
+      classifierModel: languageModel,
+    }),
+    searchResultsStore,
+    embeddedContentStore,
+    middleware: [requireValidIpAddress<ContentRouterLocals>(), requireRequestOrigin<ContentRouterLocals>()],
+  },
   conversationsRouterConfig: {
     middleware: [
       blockGetRequests,
-      requireValidIpAddress(),
-      requireRequestOrigin(),
+      requireValidIpAddress<ConversationsRouterLocals>(),
+      requireRequestOrigin<ConversationsRouterLocals>(),
       useSegmentIds(),
       redactConnectionUri(),
       cookieParser(),
@@ -363,10 +384,7 @@ export const config: AppConfig = {
       ? createConversationCustomDataWithAuthUser
       : undefined,
     addMessageToConversationCustomData: async (req, res) => {
-      const defaultCustomData = await defaultAddMessageToConversationCustomData(
-        req,
-        res
-      );
+      const defaultCustomData = await addDefaultCustomData(req, res);
       const customData = {
         ...defaultCustomData,
       };