From 76164c87508064491c1944d3c7c8f2661e2a94b2 Mon Sep 17 00:00:00 2001 From: uinstinct <61635505+uinstinct@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:07:23 +0530 Subject: [PATCH 1/4] fix: ollama detect context length automatically --- core/llm/llms/Ollama.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 1d069ff711f..81e36f771ef 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -278,7 +278,8 @@ class Ollama extends BaseLLM implements ModelInstaller { top_k: options.topK, num_predict: options.maxTokens, stop: options.stop, - num_ctx: this.contextLength, + // Ollama detects the context length from model's modelfile (or uses 2048/4096 length) when not specified. We do not want to specify a DEFAULT_CONTEXT_LENGTH to avoid memory issues. + num_ctx: this._contextLength, mirostat: options.mirostat, num_thread: options.numThreads, use_mmap: options.useMmap, From d0102f80dc82f2d6f05258790dba406d6a26b063 Mon Sep 17 00:00:00 2001 From: uinstinct <61635505+uinstinct@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:26:51 +0530 Subject: [PATCH 2/4] default context length of 4096 --- core/llm/llms/Ollama.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index 81e36f771ef..a8053b3a948 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -269,6 +269,11 @@ class Ollama extends BaseLLM implements ModelInstaller { return this.modelMap[this.model] ?? this.model; } + get contextLength() { + const DEFAULT_OLLAMA_CONTEXT_LENGTH = 4096; // https://github.com/ollama/ollama/blob/29ddfc2cab7f5a83a96c3133094f67b22e4f27d1/envconfig/config.go#L185 + return this._contextLength ?? DEFAULT_OLLAMA_CONTEXT_LENGTH; + } + private _getModelFileParams( options: CompletionOptions, ): OllamaModelFileParams { @@ -278,8 +283,7 @@ class Ollama extends BaseLLM implements ModelInstaller { top_k: options.topK, num_predict: options.maxTokens, stop: options.stop, - // Ollama detects the context length from model's modelfile (or uses 2048/4096 length) when not specified. We do not want to specify a DEFAULT_CONTEXT_LENGTH to avoid memory issues. - num_ctx: this._contextLength, + num_ctx: this.contextLength, mirostat: options.mirostat, num_thread: options.numThreads, use_mmap: options.useMmap, From 9e169b8397adf9c23a6839625d5a853be7054e2e Mon Sep 17 00:00:00 2001 From: uinstinct <61635505+uinstinct@users.noreply.github.com> Date: Tue, 16 Sep 2025 08:43:06 +0530 Subject: [PATCH 3/4] make default context to 8192 --- core/llm/llms/Ollama.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/llm/llms/Ollama.ts b/core/llm/llms/Ollama.ts index a8053b3a948..430d55ccf11 100644 --- a/core/llm/llms/Ollama.ts +++ b/core/llm/llms/Ollama.ts @@ -270,7 +270,7 @@ class Ollama extends BaseLLM implements ModelInstaller { } get contextLength() { - const DEFAULT_OLLAMA_CONTEXT_LENGTH = 4096; // https://github.com/ollama/ollama/blob/29ddfc2cab7f5a83a96c3133094f67b22e4f27d1/envconfig/config.go#L185 + const DEFAULT_OLLAMA_CONTEXT_LENGTH = 8192; // twice of https://github.com/ollama/ollama/blob/29ddfc2cab7f5a83a96c3133094f67b22e4f27d1/envconfig/config.go#L185 return this._contextLength ?? DEFAULT_OLLAMA_CONTEXT_LENGTH; } From 76a4fbfd77956373597478858cfaab9b6769f2e0 Mon Sep 17 00:00:00 2001 From: uinstinct <61635505+uinstinct@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:44:17 +0530 Subject: [PATCH 4/4] fix ollama context length llminfo --- packages/llm-info/src/providers/os.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/llm-info/src/providers/os.ts b/packages/llm-info/src/providers/os.ts index 91ade3237a6..c0428ad1a02 100644 --- a/packages/llm-info/src/providers/os.ts +++ b/packages/llm-info/src/providers/os.ts @@ -4,5 +4,6 @@ export const OsLlms: Omit[] = [ { model: "starcoder2:3b", displayName: "StarCoder 2 3B", + contextLength: 8192, }, ];