Skip to content

Commit 7a2892e

Browse files
bhoustonbhouston
authored
Merge pull request #364 from drivecore/feature/configurable-context-window
Add configurable context window size
2 parents bbd9aaa + ba97bed commit 7a2892e

File tree

15 files changed

+174
-172
lines changed

15 files changed

+174
-172
lines changed

‎mycoder.config.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ export default {
3535
//provider: 'openai',
3636
//model: 'qwen2.5-coder:14b',
3737
//baseUrl: 'http://192.168.2.66:80/v1-openai',
38+
// Manual override for context window size (in tokens)
39+
// Useful for models that don't have a known context window size
40+
// contextWindow: 16384,
3841
maxTokens: 4096,
3942
temperature: 0.7,
4043

‎packages/agent/src/core/llm/providers/anthropic.ts

Lines changed: 68 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,18 @@ import {
1212
ProviderOptions,
1313
} from '../types.js';
1414

15-
// Cache for model context window sizes
16-
const modelContextWindowCache: Record<string, number> = {};
15+
const ANTHROPIC_CONTEXT_WINDOWS: Record<string, number> = {
16+
'claude-3-7-sonnet-20250219': 200000,
17+
'claude-3-7-sonnet-latest': 200000,
18+
'claude-3-5-sonnet-20241022': 200000,
19+
'claude-3-5-sonnet-latest': 200000,
20+
'claude-3-haiku-20240307': 200000,
21+
'claude-3-opus-20240229': 200000,
22+
'claude-3-sonnet-20240229': 200000,
23+
'claude-2.1': 100000,
24+
'claude-2.0': 100000,
25+
'claude-instant-1.2': 100000,
26+
};
1727

1828
/**
1929
* Anthropic-specific options
@@ -87,7 +97,7 @@ function addCacheControlToMessages(
8797
function tokenUsageFromMessage(
8898
message: Anthropic.Message,
8999
model: string,
90-
contextWindow?: number,
100+
contextWindow: number | undefined,
91101
) {
92102
const usage = new TokenUsage();
93103
usage.input = message.usage.input_tokens;
@@ -97,19 +107,10 @@ function tokenUsageFromMessage(
97107

98108
const totalTokens = usage.input + usage.output;
99109

100-
// Use provided context window or fallback to cached value
101-
const maxTokens = contextWindow || modelContextWindowCache[model];
102-
103-
if (!maxTokens) {
104-
throw new Error(
105-
`Context window size not available for model: ${model}. Make sure to initialize the model properly.`,
106-
);
107-
}
108-
109110
return {
110111
usage,
111112
totalTokens,
112-
maxTokens,
113+
contextWindow,
113114
};
114115
}
115116

@@ -120,13 +121,14 @@ export class AnthropicProvider implements LLMProvider {
120121
name: string = 'anthropic';
121122
provider: string = 'anthropic.messages';
122123
model: string;
124+
options: AnthropicOptions;
123125
private client: Anthropic;
124126
private apiKey: string;
125127
private baseUrl?: string;
126-
private modelContextWindow?: number;
127128

128129
constructor(model: string, options: AnthropicOptions = {}) {
129130
this.model = model;
131+
this.options = options;
130132
this.apiKey = options.apiKey ?? '';
131133
this.baseUrl = options.baseUrl;
132134

@@ -139,79 +141,18 @@ export class AnthropicProvider implements LLMProvider {
139141
apiKey: this.apiKey,
140142
...(this.baseUrl && { baseURL: this.baseUrl }),
141143
});
142-
143-
// Initialize model context window detection
144-
// This is async but we don't need to await it here
145-
// If it fails, an error will be thrown when the model is used
146-
this.initializeModelContextWindow().catch((error) => {
147-
console.error(
148-
`Failed to initialize model context window: ${error.message}. The model will not work until context window information is available.`,
149-
);
150-
});
151-
}
152-
153-
/**
154-
* Fetches the model context window size from the Anthropic API
155-
*
156-
* @returns The context window size
157-
* @throws Error if the context window size cannot be determined
158-
*/
159-
private async initializeModelContextWindow(): Promise<number> {
160-
try {
161-
const response = await this.client.models.list();
162-
163-
if (!response?.data || !Array.isArray(response.data)) {
164-
throw new Error(
165-
`Invalid response from models.list() for ${this.model}`,
166-
);
167-
}
168-
169-
// Try to find the exact model
170-
let model = response.data.find((m) => m.id === this.model);
171-
172-
// If not found, try to find a model that starts with the same name
173-
// This helps with model aliases like 'claude-3-sonnet-latest'
174-
if (!model) {
175-
// Split by '-latest' or '-20' to get the base model name
176-
const parts = this.model.split('-latest');
177-
const modelPrefix =
178-
parts.length > 1 ? parts[0] : this.model.split('-20')[0];
179-
180-
if (modelPrefix) {
181-
model = response.data.find((m) => m.id.startsWith(modelPrefix));
182-
183-
if (model) {
184-
console.info(
185-
`Model ${this.model} not found, using ${model.id} for context window size`,
186-
);
187-
}
188-
}
189-
}
190-
191-
// Using type assertion to access context_window property
192-
// The Anthropic API returns context_window but it may not be in the TypeScript definitions
193-
if (model && 'context_window' in model) {
194-
const contextWindow = (model as any).context_window;
195-
this.modelContextWindow = contextWindow;
196-
// Cache the result for future use
197-
modelContextWindowCache[this.model] = contextWindow;
198-
return contextWindow;
199-
} else {
200-
throw new Error(
201-
`No context window information found for model: ${this.model}`,
202-
);
203-
}
204-
} catch (error) {
205-
throw new Error(
206-
`Failed to determine context window size for model ${this.model}: ${(error as Error).message}`,
207-
);
208-
}
209144
}
210145

211146
/**
212147
* Generate text using Anthropic API
213148
*/
214149
async generateText(options: GenerateOptions): Promise<LLMResponse> {
150+
// Use configuration contextWindow if provided, otherwise use model-specific value
151+
let modelContextWindow = ANTHROPIC_CONTEXT_WINDOWS[this.model];
152+
if (!modelContextWindow && this.options.contextWindow) {
153+
modelContextWindow = this.options.contextWindow;
154+
}
155+
215156
const { messages, functions, temperature = 0.7, maxTokens, topP } = options;
216157

217158
// Extract system message
@@ -227,63 +168,56 @@ export class AnthropicProvider implements LLMProvider {
227168
})),
228169
);
229170

230-
try {
231-
const requestOptions: Anthropic.MessageCreateParams = {
232-
model: this.model,
233-
messages: addCacheControlToMessages(formattedMessages),
234-
temperature,
235-
max_tokens: maxTokens || 1024,
236-
system: systemMessage?.content
237-
? [
238-
{
239-
type: 'text',
240-
text: systemMessage?.content,
241-
cache_control: { type: 'ephemeral' },
242-
},
243-
]
244-
: undefined,
245-
top_p: topP,
246-
tools,
247-
stream: false,
248-
};
171+
const requestOptions: Anthropic.MessageCreateParams = {
172+
model: this.model,
173+
messages: addCacheControlToMessages(formattedMessages),
174+
temperature,
175+
max_tokens: maxTokens || 1024,
176+
system: systemMessage?.content
177+
? [
178+
{
179+
type: 'text',
180+
text: systemMessage?.content,
181+
cache_control: { type: 'ephemeral' },
182+
},
183+
]
184+
: undefined,
185+
top_p: topP,
186+
tools,
187+
stream: false,
188+
};
249189

250-
const response = await this.client.messages.create(requestOptions);
190+
const response = await this.client.messages.create(requestOptions);
251191

252-
// Extract content and tool calls
253-
const content =
254-
response.content.find((c) => c.type === 'text')?.text || '';
255-
const toolCalls = response.content
256-
.filter((c) => {
257-
const contentType = c.type;
258-
return contentType === 'tool_use';
259-
})
260-
.map((c) => {
261-
const toolUse = c as Anthropic.Messages.ToolUseBlock;
262-
return {
263-
id: toolUse.id,
264-
name: toolUse.name,
265-
content: JSON.stringify(toolUse.input),
266-
};
267-
});
192+
// Extract content and tool calls
193+
const content = response.content.find((c) => c.type === 'text')?.text || '';
194+
const toolCalls = response.content
195+
.filter((c) => {
196+
const contentType = c.type;
197+
return contentType === 'tool_use';
198+
})
199+
.map((c) => {
200+
const toolUse = c as Anthropic.Messages.ToolUseBlock;
201+
return {
202+
id: toolUse.id,
203+
name: toolUse.name,
204+
content: JSON.stringify(toolUse.input),
205+
};
206+
});
268207

269-
const tokenInfo = tokenUsageFromMessage(
270-
response,
271-
this.model,
272-
this.modelContextWindow,
273-
);
208+
const tokenInfo = tokenUsageFromMessage(
209+
response,
210+
this.model,
211+
modelContextWindow,
212+
);
274213

275-
return {
276-
text: content,
277-
toolCalls: toolCalls,
278-
tokenUsage: tokenInfo.usage,
279-
totalTokens: tokenInfo.totalTokens,
280-
maxTokens: tokenInfo.maxTokens,
281-
};
282-
} catch (error) {
283-
throw new Error(
284-
`Error calling Anthropic API: ${(error as Error).message}`,
285-
);
286-
}
214+
return {
215+
text: content,
216+
toolCalls: toolCalls,
217+
tokenUsage: tokenInfo.usage,
218+
totalTokens: tokenInfo.totalTokens,
219+
contextWindow: tokenInfo.contextWindow,
220+
};
287221
}
288222

289223
/**

‎packages/agent/src/core/llm/providers/ollama.ts

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@ import {
2424

2525
// Define model context window sizes for Ollama models
2626
// These are approximate and may vary based on specific model configurations
27-
const OLLAMA_MODEL_LIMITS: Record<string, number> = {
28-
default: 4096,
27+
const OLLAMA_CONTEXT_WINDOWS: Record<string, number> = {
2928
llama2: 4096,
3029
'llama2-uncensored': 4096,
3130
'llama2:13b': 4096,
@@ -53,10 +52,12 @@ export class OllamaProvider implements LLMProvider {
5352
name: string = 'ollama';
5453
provider: string = 'ollama.chat';
5554
model: string;
55+
options: OllamaOptions;
5656
private client: Ollama;
5757

5858
constructor(model: string, options: OllamaOptions = {}) {
5959
this.model = model;
60+
this.options = options;
6061
const baseUrl =
6162
options.baseUrl ||
6263
process.env.OLLAMA_BASE_URL ||
@@ -136,19 +137,26 @@ export class OllamaProvider implements LLMProvider {
136137
const totalTokens = tokenUsage.input + tokenUsage.output;
137138

138139
// Extract the base model name without specific parameters
139-
const baseModelName = this.model.split(':')[0];
140140
// Check if model exists in limits, otherwise use base model or default
141-
const modelMaxTokens =
142-
OLLAMA_MODEL_LIMITS[this.model] ||
143-
(baseModelName ? OLLAMA_MODEL_LIMITS[baseModelName] : undefined) ||
144-
4096; // Default fallback
141+
let contextWindow = OLLAMA_CONTEXT_WINDOWS[this.model];
142+
if (!contextWindow) {
143+
const baseModelName = this.model.split(':')[0];
144+
if (baseModelName) {
145+
contextWindow = OLLAMA_CONTEXT_WINDOWS[baseModelName];
146+
}
147+
148+
// If still no context window, use the one from configuration if available
149+
if (!contextWindow && this.options.contextWindow) {
150+
contextWindow = this.options.contextWindow;
151+
}
152+
}
145153

146154
return {
147155
text: content,
148156
toolCalls: toolCalls,
149157
tokenUsage: tokenUsage,
150158
totalTokens,
151-
maxTokens: modelMaxTokens,
159+
contextWindow,
152160
};
153161
}
154162

‎packages/agent/src/core/llm/providers/openai.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ import type {
2020
} from 'openai/resources/chat';
2121

2222
// Define model context window sizes for OpenAI models
23-
const OPENAI_MODEL_LIMITS: Record<string, number> = {
24-
default: 128000,
23+
const OPENA_CONTEXT_WINDOWS: Record<string, number> = {
2524
'o3-mini': 200000,
2625
'o1-pro': 200000,
2726
o1: 200000,
@@ -52,13 +51,15 @@ export class OpenAIProvider implements LLMProvider {
5251
name: string = 'openai';
5352
provider: string = 'openai.chat';
5453
model: string;
54+
options: OpenAIOptions;
5555
private client: OpenAI;
5656
private apiKey: string;
5757
private baseUrl?: string;
5858
private organization?: string;
5959

6060
constructor(model: string, options: OpenAIOptions = {}) {
6161
this.model = model;
62+
this.options = options;
6263
this.apiKey = options.apiKey ?? '';
6364
this.baseUrl = options.baseUrl;
6465

@@ -136,14 +137,19 @@ export class OpenAIProvider implements LLMProvider {
136137

137138
// Calculate total tokens and get max tokens for the model
138139
const totalTokens = tokenUsage.input + tokenUsage.output;
139-
const modelMaxTokens = OPENAI_MODEL_LIMITS[this.model] || 8192; // Default fallback
140+
141+
// Use configuration contextWindow if provided, otherwise use model-specific value
142+
let contextWindow = OPENA_CONTEXT_WINDOWS[this.model];
143+
if (!contextWindow && this.options.contextWindow) {
144+
contextWindow = this.options.contextWindow;
145+
}
140146

141147
return {
142148
text: content,
143149
toolCalls,
144150
tokenUsage,
145151
totalTokens,
146-
maxTokens: modelMaxTokens,
152+
contextWindow,
147153
};
148154
} catch (error) {
149155
throw new Error(`Error calling OpenAI API: ${(error as Error).message}`);

‎packages/agent/src/core/llm/types.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ export interface LLMResponse {
8282
tokenUsage: TokenUsage;
8383
// Add new fields for context window tracking
8484
totalTokens?: number; // Total tokens used in this request
85-
maxTokens?: number; // Maximum allowed tokens for this model
85+
contextWindow?: number; // Maximum allowed tokens for this model
8686
}
8787

8888
/**
@@ -107,5 +107,6 @@ export interface ProviderOptions {
107107
apiKey?: string;
108108
baseUrl?: string;
109109
organization?: string;
110+
contextWindow?: number; // Manual override for context window size
110111
[key: string]: any; // Allow for provider-specific options
111112
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy