supermemoryai · jothipriyadharshanr · Dec 26, 2025 · Dec 26, 2025 · Prasanna721 · Dec 29, 2025
diff --git a/README.md b/README.md
@@ -20,7 +20,8 @@ A pluggable benchmarking framework for evaluating memory and context systems.
 ┌─────────────┐    ┌─────────────┐    ┌─────────────┐
 │  Benchmarks │    │  Providers  │    │   Judges    │
 │  (LoCoMo,   │    │ (Supermem,  │    │  (GPT-4o,   │
-│  LongMem..) │    │  Mem0, Zep) │    │  Claude..)  │
+│  LongMem..) │    │  Mem0, Zep, │    │  Claude..)  │
+│             │    │  LocalBM25) │    │             │
 └──────┬──────┘    └──────┬──────┘    └──────┬──────┘
        └──────────────────┼──────────────────┘
                          ▼
@@ -45,6 +46,8 @@ bun run src/index.ts run -p supermemory -b locomo
 
 ```bash
 # Providers (at least one)
+# Local provider (no keys required)
+# localbm25 requires no provider API keys
 SUPERMEMORY_API_KEY=
 MEM0_API_KEY=
 ZEP_API_KEY=
@@ -55,6 +58,8 @@ ANTHROPIC_API_KEY=
 GOOGLE_API_KEY=
 ```
 
+Note: `localbm25` runs offline and does not require any provider API keys.
+
 ## Commands
 
 | Command | Description |
@@ -73,7 +78,7 @@ GOOGLE_API_KEY=
 ## Options
 
 ```
--p, --provider         Memory provider (supermemory, mem0, zep)
+-p, --provider         Memory provider (supermemory, mem0, zep, localbm25)
 -b, --benchmark        Benchmark (locomo, longmemeval, convomem)
 -j, --judge            Judge model (gpt-4o, sonnet-4, gemini-2.5-flash, etc.)
 -r, --run-id           Run identifier (auto-generated if omitted)
@@ -98,11 +103,14 @@ bun run src/index.ts run -r my-test
 # Limited questions
 bun run src/index.ts run -p supermemory -b locomo -l 10
 
+# Offline baseline provider (no provider API keys required)
+bun run src/index.ts run -p localbm25 -b convomem -j gpt-4o -l 10
+
 # Different models
 bun run src/index.ts run -p zep -b longmemeval -j sonnet-4 -m gemini-2.5-flash
 
 # Compare multiple providers
-bun run src/index.ts compare -p supermemory,mem0,zep -b locomo -s 5
+bun run src/index.ts compare -p localbm25,supermemory,mem0,zep -b locomo -s 5
 
 # Test single question
 bun run src/index.ts test -r my-test -q question_42

diff --git a/bun.lock b/bun.lock
@@ -9,11 +9,12 @@
         "@ai-sdk/google": "^2.0.49",
         "@ai-sdk/openai": "^2.0.88",
         "@getzep/zep-cloud": "^3.13.0",
-        "@letta-ai/letta-client": "^1.6.1",
         "ai": "^5.0.115",
         "drizzle-orm": "^0.45.1",
         "mem0ai": "^2.1.38",
         "supermemory": "^4.0.0",
+        "wink-bm25-text-search": "^3.1.2",
+        "wink-nlp-utils": "^2.1.0",
         "zod": "^3.24.4",
       },
       "devDependencies": {
@@ -62,8 +63,6 @@
 
     "@langchain/core": ["@langchain/[email protected]", "", { "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", "camelcase": "6", "decamelize": "1.2.0", "js-tiktoken": "^1.0.12", "langsmith": "^0.3.67", "mustache": "^4.2.0", "p-queue": "^6.6.2", "p-retry": "4", "uuid": "^10.0.0", "zod": "^3.25.32", "zod-to-json-schema": "^3.22.3" } }, "sha512-ZLAs5YMM5N2UXN3kExMglltJrKKoW7hs3KMZFlXUnD7a5DFKBYxPFMeXA4rT+uvTxuJRZPCYX0JKI5BhyAWx4A=="],
 
-    "@letta-ai/letta-client": ["@letta-ai/[email protected]", "", {}, "sha512-kCRnEKpeTj3e1xqRd58xvoCp28p/wuJUptrIlJ8cT2GiYkrOESlKmp6lc3f246VusrowdGeB9hSXePXZgd7rAA=="],
-
     "@mistralai/mistralai": ["@mistralai/[email protected]", "", { "dependencies": { "zod": "^3.20.0", "zod-to-json-schema": "^3.24.1" } }, "sha512-6/BVj2mcaggYbpMzNSxtqtM2Tv/Jb5845XFd2CMYFO+O5VBkX70iLjtkBBTI4JFhh1l9vTCIMYXBVOjLoBVHGQ=="],
 
     "@npmcli/fs": ["@npmcli/[email protected]", "", { "dependencies": { "@gar/promisify": "^1.0.1", "semver": "^7.3.5" } }, "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ=="],
@@ -262,7 +261,7 @@
 
     "ecdsa-sig-formatter": ["[email protected]", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="],
 
-    "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
+    "emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
 
     "encoding": ["[email protected]", "", { "dependencies": { "iconv-lite": "^0.6.2" } }, "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A=="],
 
@@ -668,6 +667,24 @@
 
     "wide-align": ["[email protected]", "", { "dependencies": { "string-width": "^1.0.2 || 2 || 3 || 4" } }, "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg=="],
 
+    "wink-bm25-text-search": ["[email protected]", "", { "dependencies": { "wink-eng-lite-web-model": "^1.4.3", "wink-helpers": "^2.0.0", "wink-nlp": "^1.12.2", "wink-nlp-utils": "^2.0.4" } }, "sha512-s+xY0v/yurUhiUop/XZnf9IvO9XVuwI14X+QTW0JqlmQCg+9ZgVXTMudXKqZuQVsnm5J+RjLnqrOflnD5BLApA=="],
+
+    "wink-distance": ["[email protected]", "", { "dependencies": { "wink-helpers": "^2.0.0", "wink-jaro-distance": "^2.0.0" } }, "sha512-pyEhUB/OKFYcgOC4J6E+c+gwVA/8qg2s5n49mIcUsJZM5iDSa17uOxRQXR4rvfp+gbj55K/I08FwjFBwb6fq3g=="],
+
+    "wink-eng-lite-web-model": ["[email protected]", "", {}, "sha512-M2tSOU/rVNkDj8AS8IoKJaM7apJJjS0cN+hE8CPazfnB4A/ojyc9+7RMPk18UOiIdSyWk7MR6w8z9lWix2l5tA=="],
+
+    "wink-helpers": ["[email protected]", "", {}, "sha512-I/ZzXrHcNRXuoeFJmp2vMVqDI6UCK02Tds1WP4kSGAmx520gjL1BObVzF7d2ps24tyHIly9ngdB2jwhlFUjPvg=="],
+
+    "wink-jaro-distance": ["[email protected]", "", {}, "sha512-9bcUaXCi9N8iYpGWbFkf83OsBkg17r4hEyxusEzl+nnReLRPqxhB9YNeRn3g54SYnVRNXP029lY3HDsbdxTAuA=="],
+
+    "wink-nlp": ["[email protected]", "", {}, "sha512-lvY5iCs3T8I34F8WKS70+2P0U9dWLn3vdPf/Z+m2VK14N7OmqnPzmHfh3moHdusajoQ37Em39z0IZB9K4x/96A=="],
+
+    "wink-nlp-utils": ["[email protected]", "", { "dependencies": { "wink-distance": "^2.0.1", "wink-eng-lite-web-model": "^1.4.3", "wink-helpers": "^2.0.0", "wink-nlp": "^1.12.0", "wink-porter2-stemmer": "^2.0.1", "wink-tokenizer": "^5.2.3" } }, "sha512-b7PcRhEBNxQmsmht70jLOkwYUyie3da4/cgEXL+CumYO5b/nwV+W7fuMXToh5BtGq1RABznmc2TGTp1Qf/JUXg=="],
+
+    "wink-porter2-stemmer": ["[email protected]", "", {}, "sha512-0g+RkkqhRXFmSpJQStVXW5N/WsshWpJXsoDRW7DwVkGI2uDT6IBCoq3xdH5p6IHLaC6ygk7RWUsUx4alKxoagQ=="],
+
+    "wink-tokenizer": ["[email protected]", "", { "dependencies": { "emoji-regex": "^9.0.0" } }, "sha512-O/yAw0g3FmSgeeQuYAJJfP7fVPB4A6ay0018qASh79aWmIOyPYy4j4r9EQT8xBjicja6lCLvgRVAybmEBaATQA=="],
+
     "wrap-ansi": ["[email protected]", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="],
 
     "wrap-ansi-cjs": ["[email protected]", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="],
@@ -742,6 +759,10 @@
 
     "ssri/minipass": ["[email protected]", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="],
 
+    "string-width/emoji-regex": ["[email protected]", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
+
+    "string-width-cjs/emoji-regex": ["[email protected]", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
+
     "tar-fs/chownr": ["[email protected]", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="],
 
     "wrap-ansi/ansi-styles": ["[email protected]", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="],
@@ -754,8 +775,6 @@
 
     "@anthropic-ai/sdk/@types/node/undici-types": ["[email protected]", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
 
-    "@isaacs/cliui/string-width/emoji-regex": ["[email protected]", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
-
     "@isaacs/cliui/strip-ansi/ansi-regex": ["[email protected]", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
 
     "cloudflare/@types/node/undici-types": ["[email protected]", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
@@ -776,8 +795,6 @@
 
     "pg/pg-types/postgres-interval": ["[email protected]", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="],
 
-    "wrap-ansi/string-width/emoji-regex": ["[email protected]", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
-
     "wrap-ansi/strip-ansi/ansi-regex": ["[email protected]", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
 
     "gaxios/rimraf/glob/minimatch": ["[email protected]", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="],

diff --git a/package.json b/package.json
@@ -15,6 +15,8 @@
     "drizzle-orm": "^0.45.1",
     "mem0ai": "^2.1.38",
     "supermemory": "^4.0.0",
+    "wink-bm25-text-search": "^3.1.2",
+    "wink-nlp-utils": "^2.1.0",
     "zod": "^3.24.4"
   },
   "devDependencies": {

diff --git a/src/README.md b/src/README.md
@@ -3,7 +3,7 @@
 ```
 src/
 ├── benchmarks/      # Benchmark adapters (LoCoMo, LongMemEval, ConvoMem)
-├── providers/       # Memory provider integrations (Supermemory, Mem0, Zep)
+├── providers/       # Memory provider integrations (Supermemory, Mem0, Zep, Local BM25)
 ├── judges/          # LLM-as-judge implementations (OpenAI, Anthropic, Google)
 ├── orchestrator/    # Pipeline execution and checkpointing
 │   └── phases/      # Individual phase runners (ingest, search, answer, evaluate)

diff --git a/src/cli/index.ts b/src/cli/index.ts
@@ -66,10 +66,14 @@ Available providers for storing and retrieving memories:
   zep            Zep - Long-term memory for AI assistants
                  Requires: ZEP_API_KEY
 
+  localbm25      Local BM25 - Offline baseline provider (no API keys)
+                 Requires: none
+
 Usage:
   -p supermemory    Use Supermemory as the memory provider
   -p mem0           Use Mem0 as the memory provider
   -p zep            Use Zep as the memory provider
+  -p localbm25      Use Local BM25 as the memory provider
 `)
 }
 

diff --git a/src/providers/index.ts b/src/providers/index.ts
@@ -2,11 +2,13 @@ import type { Provider, ProviderName } from "../types/provider"
 import { SupermemoryProvider } from "./supermemory"
 import { Mem0Provider } from "./mem0"
 import { ZepProvider } from "./zep"
+import { LocalBM25Provider } from "./localbm25"
 
 const providers: Record<ProviderName, new () => Provider> = {
     supermemory: SupermemoryProvider,
     mem0: Mem0Provider,
     zep: ZepProvider,
+    localbm25: LocalBM25Provider
 }
 
 export function createProvider(name: ProviderName): Provider {