mirror of
https://github.com/voideditor/void.git
synced 2025-03-14 13:59:21 +00:00
Merge branch 'model-selection' into HEAD
This commit is contained in:
Mathew Pareles
@ -795,26 +795,27 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ
|
||||
},
|
||||
useProviderFor: 'Autocomplete',
|
||||
logging: { loggingName: 'Autocomplete' },
|
||||
onText: async ({ fullText, newText }) => {
|
||||
onText: () => { }, // unused in FIMMessage
|
||||
// onText: async ({ fullText, newText }) => {
|
||||
|
||||
newAutocompletion.insertText = fullText
|
||||
// newAutocompletion.insertText = fullText
|
||||
|
||||
// count newlines in newText
|
||||
const numNewlines = newText.match(/\n|\r\n/g)?.length || 0
|
||||
newAutocompletion._newlineCount += numNewlines
|
||||
// // count newlines in newText
|
||||
// const numNewlines = newText.match(/\n|\r\n/g)?.length || 0
|
||||
// newAutocompletion._newlineCount += numNewlines
|
||||
|
||||
// if too many newlines, resolve up to last newline
|
||||
if (newAutocompletion._newlineCount > 10) {
|
||||
const lastNewlinePos = fullText.lastIndexOf('\n')
|
||||
newAutocompletion.insertText = fullText.substring(0, lastNewlinePos)
|
||||
resolve(newAutocompletion.insertText)
|
||||
return
|
||||
}
|
||||
// // if too many newlines, resolve up to last newline
|
||||
// if (newAutocompletion._newlineCount > 10) {
|
||||
// const lastNewlinePos = fullText.lastIndexOf('\n')
|
||||
// newAutocompletion.insertText = fullText.substring(0, lastNewlinePos)
|
||||
// resolve(newAutocompletion.insertText)
|
||||
// return
|
||||
// }
|
||||
|
||||
// if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) {
|
||||
// reject('LLM response did not match user\'s text.')
|
||||
// }
|
||||
},
|
||||
// // if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) {
|
||||
// // reject('LLM response did not match user\'s text.')
|
||||
// // }
|
||||
// },
|
||||
onFinalMessage: ({ fullText }) => {
|
||||
|
||||
// console.log('____res: ', JSON.stringify(newAutocompletion.insertText))
|
||||
|
||||
@ -59,7 +59,7 @@ class SurroundingsRemover {
|
||||
// return offset === suffix.length
|
||||
// }
|
||||
|
||||
removeFromStartUntil = (until: string, alsoRemoveUntilStr: boolean) => {
|
||||
removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => {
|
||||
const index = this.originalS.indexOf(until, this.i)
|
||||
|
||||
if (index === -1) {
|
||||
@ -86,7 +86,7 @@ class SurroundingsRemover {
|
||||
const foundCodeBlock = pm.removePrefix('```')
|
||||
if (!foundCodeBlock) return false
|
||||
|
||||
pm.removeFromStartUntil('\n', true) // language
|
||||
pm.removeFromStartUntilFullMatch('\n', true) // language
|
||||
|
||||
const j = pm.j
|
||||
let foundCodeBlockEnd = pm.removeSuffix('```')
|
||||
@ -159,27 +159,10 @@ export const extractCodeFromFIM = ({ text, recentlyAddedTextLen, midTag, }: { te
|
||||
const [delta, ignoredSuffix] = pm.deltaInfo(recentlyAddedTextLen)
|
||||
|
||||
return [s, delta, ignoredSuffix]
|
||||
|
||||
|
||||
// // const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?<MID>([\s\S]*?)(?:<\/MID>|`{1,3}|$)/;
|
||||
// const regex = new RegExp(
|
||||
// `[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:</${midTag}>|\`{1,3}|$)`,
|
||||
// ''
|
||||
// );
|
||||
// const match = text.match(regex);
|
||||
// if (match) {
|
||||
// const [_, languageName, codeBetweenMidTags] = match;
|
||||
// return [languageName, codeBetweenMidTags] as const
|
||||
|
||||
// } else {
|
||||
// return [undefined, extractCodeFromRegular(text)] as const
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
export type ExtractedSearchReplaceBlock = {
|
||||
state: 'writingOriginal' | 'writingFinal' | 'done',
|
||||
orig: string,
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from './llmMessageTypes.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, ServiceSendLLMMessageParams, MainSendLLMMessageParams, MainLLMMessageAbortParams, ServiceModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, MainModelListParams, OllamaModelResponse, VLLMModelResponse, } from './llmMessageTypes.js';
|
||||
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
|
||||
@ -24,27 +24,39 @@ export interface ILLMMessageService {
|
||||
sendLLMMessage: (params: ServiceSendLLMMessageParams) => string | null;
|
||||
abort: (requestId: string) => void;
|
||||
ollamaList: (params: ServiceModelListParams<OllamaModelResponse>) => void;
|
||||
openAICompatibleList: (params: ServiceModelListParams<OpenaiCompatibleModelResponse>) => void;
|
||||
vLLMList: (params: ServiceModelListParams<VLLMModelResponse>) => void;
|
||||
}
|
||||
|
||||
|
||||
// open this file side by side with llmMessageChannel
|
||||
export class LLMMessageService extends Disposable implements ILLMMessageService {
|
||||
|
||||
readonly _serviceBrand: undefined;
|
||||
private readonly channel: IChannel // LLMMessageChannel
|
||||
|
||||
// llmMessage
|
||||
private readonly onTextHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) } = {}
|
||||
private readonly onFinalMessageHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) } = {}
|
||||
private readonly onErrorHooks_llm: { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) } = {}
|
||||
// sendLLMMessage
|
||||
private readonly llmMessageHooks = {
|
||||
onText: {} as { [eventId: string]: ((params: EventLLMMessageOnTextParams) => void) },
|
||||
onFinalMessage: {} as { [eventId: string]: ((params: EventLLMMessageOnFinalMessageParams) => void) },
|
||||
onError: {} as { [eventId: string]: ((params: EventLLMMessageOnErrorParams) => void) },
|
||||
}
|
||||
|
||||
|
||||
// ollamaList
|
||||
private readonly onSuccess_ollama: { [eventId: string]: ((params: EventModelListOnSuccessParams<OllamaModelResponse>) => void) } = {}
|
||||
private readonly onError_ollama: { [eventId: string]: ((params: EventModelListOnErrorParams<OllamaModelResponse>) => void) } = {}
|
||||
|
||||
// openAICompatibleList
|
||||
private readonly onSuccess_openAICompatible: { [eventId: string]: ((params: EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>) => void) } = {}
|
||||
private readonly onError_openAICompatible: { [eventId: string]: ((params: EventModelListOnErrorParams<OpenaiCompatibleModelResponse>) => void) } = {}
|
||||
// list hooks
|
||||
private readonly listHooks = {
|
||||
ollama: {
|
||||
success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams<OllamaModelResponse>) => void) },
|
||||
error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams<OllamaModelResponse>) => void) },
|
||||
},
|
||||
vLLM: {
|
||||
success: {} as { [eventId: string]: ((params: EventModelListOnSuccessParams<VLLMModelResponse>) => void) },
|
||||
error: {} as { [eventId: string]: ((params: EventModelListOnErrorParams<VLLMModelResponse>) => void) },
|
||||
}
|
||||
} satisfies {
|
||||
[providerName: string]: {
|
||||
success: { [eventId: string]: ((params: EventModelListOnSuccessParams<any>) => void) },
|
||||
error: { [eventId: string]: ((params: EventModelListOnErrorParams<any>) => void) },
|
||||
}
|
||||
}
|
||||
|
||||
constructor(
|
||||
@IMainProcessService private readonly mainProcessService: IMainProcessService, // used as a renderer (only usable on client side)
|
||||
@ -59,32 +71,14 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
||||
|
||||
// .listen sets up an IPC channel and takes a few ms, so we set up listeners immediately and add hooks to them instead
|
||||
// llm
|
||||
this._register((this.channel.listen('onText_llm') satisfies Event<EventLLMMessageOnTextParams>)(e => {
|
||||
this.onTextHooks_llm[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onFinalMessage_llm') satisfies Event<EventLLMMessageOnFinalMessageParams>)(e => {
|
||||
this.onFinalMessageHooks_llm[e.requestId]?.(e)
|
||||
this._onRequestIdDone(e.requestId)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_llm') satisfies Event<EventLLMMessageOnErrorParams>)(e => {
|
||||
console.error('Error in LLMMessageService:', JSON.stringify(e))
|
||||
this.onErrorHooks_llm[e.requestId]?.(e)
|
||||
this._onRequestIdDone(e.requestId)
|
||||
}))
|
||||
this._register((this.channel.listen('onText_sendLLMMessage') satisfies Event<EventLLMMessageOnTextParams>)(e => { this.llmMessageHooks.onText[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onFinalMessage_sendLLMMessage') satisfies Event<EventLLMMessageOnFinalMessageParams>)(e => { this.llmMessageHooks.onFinalMessage[e.requestId]?.(e); this._onRequestIdDone(e.requestId) }))
|
||||
this._register((this.channel.listen('onError_sendLLMMessage') satisfies Event<EventLLMMessageOnErrorParams>)(e => { this.llmMessageHooks.onError[e.requestId]?.(e); this._onRequestIdDone(e.requestId); console.error('Error in LLMMessageService:', JSON.stringify(e)) }))
|
||||
// ollama .list()
|
||||
this._register((this.channel.listen('onSuccess_ollama') satisfies Event<EventModelListOnSuccessParams<OllamaModelResponse>>)(e => {
|
||||
this.onSuccess_ollama[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_ollama') satisfies Event<EventModelListOnErrorParams<OllamaModelResponse>>)(e => {
|
||||
this.onError_ollama[e.requestId]?.(e)
|
||||
}))
|
||||
// openaiCompatible .list()
|
||||
this._register((this.channel.listen('onSuccess_openAICompatible') satisfies Event<EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>>)(e => {
|
||||
this.onSuccess_openAICompatible[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onError_openAICompatible') satisfies Event<EventModelListOnErrorParams<OpenaiCompatibleModelResponse>>)(e => {
|
||||
this.onError_openAICompatible[e.requestId]?.(e)
|
||||
}))
|
||||
this._register((this.channel.listen('onSuccess_list_ollama') satisfies Event<EventModelListOnSuccessParams<OllamaModelResponse>>)(e => { this.listHooks.ollama.success[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onError_list_ollama') satisfies Event<EventModelListOnErrorParams<OllamaModelResponse>>)(e => { this.listHooks.ollama.error[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onSuccess_list_vLLM') satisfies Event<EventModelListOnSuccessParams<VLLMModelResponse>>)(e => { this.listHooks.vLLM.success[e.requestId]?.(e) }))
|
||||
this._register((this.channel.listen('onError_list_vLLM') satisfies Event<EventModelListOnErrorParams<VLLMModelResponse>>)(e => { this.listHooks.vLLM.error[e.requestId]?.(e) }))
|
||||
|
||||
}
|
||||
|
||||
@ -117,9 +111,9 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
||||
|
||||
// add state for request id
|
||||
const requestId = generateUuid();
|
||||
this.onTextHooks_llm[requestId] = onText
|
||||
this.onFinalMessageHooks_llm[requestId] = onFinalMessage
|
||||
this.onErrorHooks_llm[requestId] = onError
|
||||
this.llmMessageHooks.onText[requestId] = onText
|
||||
this.llmMessageHooks.onFinalMessage[requestId] = onFinalMessage
|
||||
this.llmMessageHooks.onError[requestId] = onError
|
||||
|
||||
const { aiInstructions } = this.voidSettingsService.state.globalSettings
|
||||
const { settingsOfProvider } = this.voidSettingsService.state
|
||||
@ -151,43 +145,46 @@ export class LLMMessageService extends Disposable implements ILLMMessageService
|
||||
|
||||
// add state for request id
|
||||
const requestId_ = generateUuid();
|
||||
this.onSuccess_ollama[requestId_] = onSuccess
|
||||
this.onError_ollama[requestId_] = onError
|
||||
this.listHooks.ollama.success[requestId_] = onSuccess
|
||||
this.listHooks.ollama.error[requestId_] = onError
|
||||
|
||||
this.channel.call('ollamaList', {
|
||||
...proxyParams,
|
||||
settingsOfProvider,
|
||||
providerName: 'ollama',
|
||||
requestId: requestId_,
|
||||
} satisfies MainModelListParams<OllamaModelResponse>)
|
||||
}
|
||||
|
||||
|
||||
openAICompatibleList = (params: ServiceModelListParams<OpenaiCompatibleModelResponse>) => {
|
||||
vLLMList = (params: ServiceModelListParams<VLLMModelResponse>) => {
|
||||
const { onSuccess, onError, ...proxyParams } = params
|
||||
|
||||
const { settingsOfProvider } = this.voidSettingsService.state
|
||||
|
||||
// add state for request id
|
||||
const requestId_ = generateUuid();
|
||||
this.onSuccess_openAICompatible[requestId_] = onSuccess
|
||||
this.onError_openAICompatible[requestId_] = onError
|
||||
this.listHooks.vLLM.success[requestId_] = onSuccess
|
||||
this.listHooks.vLLM.error[requestId_] = onError
|
||||
|
||||
this.channel.call('openAICompatibleList', {
|
||||
this.channel.call('vLLMList', {
|
||||
...proxyParams,
|
||||
settingsOfProvider,
|
||||
providerName: 'vLLM',
|
||||
requestId: requestId_,
|
||||
} satisfies MainModelListParams<OpenaiCompatibleModelResponse>)
|
||||
} satisfies MainModelListParams<VLLMModelResponse>)
|
||||
}
|
||||
|
||||
|
||||
|
||||
_onRequestIdDone(requestId: string) {
|
||||
delete this.onTextHooks_llm[requestId]
|
||||
delete this.onFinalMessageHooks_llm[requestId]
|
||||
delete this.onErrorHooks_llm[requestId]
|
||||
delete this.llmMessageHooks.onText[requestId]
|
||||
delete this.llmMessageHooks.onFinalMessage[requestId]
|
||||
delete this.llmMessageHooks.onError[requestId]
|
||||
|
||||
delete this.onSuccess_ollama[requestId]
|
||||
delete this.onError_ollama[requestId]
|
||||
delete this.listHooks.ollama.success[requestId]
|
||||
delete this.listHooks.ollama.error[requestId]
|
||||
|
||||
delete this.listHooks.vLLM.success[requestId]
|
||||
delete this.listHooks.vLLM.error[requestId]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -45,7 +45,7 @@ export type ToolCallType = {
|
||||
}
|
||||
|
||||
|
||||
export type OnText = (p: { newText: string, fullText: string }) => void
|
||||
export type OnText = (p: { newText: string, fullText: string; newReasoning: string; fullReasoning: string }) => void
|
||||
export type OnFinalMessage = (p: { fullText: string, toolCalls?: ToolCallType[] }) => void // id is tool_use_id
|
||||
export type OnError = (p: { message: string, fullError: Error | null }) => void
|
||||
export type AbortRef = { current: (() => void) | null }
|
||||
@ -65,7 +65,7 @@ export const toLLMChatMessage = (c: ChatMessage): LLMChatMessage => {
|
||||
}
|
||||
|
||||
|
||||
type _InternalSendFIMMessage = {
|
||||
export type LLMFIMMessage = {
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
stopTokens: string[];
|
||||
@ -77,7 +77,7 @@ type SendLLMType = {
|
||||
tools?: InternalToolInfo[];
|
||||
} | {
|
||||
messagesType: 'FIMMessage';
|
||||
messages: _InternalSendFIMMessage;
|
||||
messages: LLMFIMMessage;
|
||||
tools?: undefined;
|
||||
}
|
||||
|
||||
@ -118,38 +118,6 @@ export type EventLLMMessageOnFinalMessageParams = Parameters<OnFinalMessage>[0]
|
||||
export type EventLLMMessageOnErrorParams = Parameters<OnError>[0] & { requestId: string }
|
||||
|
||||
|
||||
export type _InternalSendLLMChatMessageFnType = (
|
||||
params: {
|
||||
aiInstructions: string;
|
||||
|
||||
onText: OnText;
|
||||
onFinalMessage: OnFinalMessage;
|
||||
onError: OnError;
|
||||
providerName: ProviderName;
|
||||
settingsOfProvider: SettingsOfProvider;
|
||||
modelName: string;
|
||||
_setAborter: (aborter: () => void) => void;
|
||||
|
||||
tools?: InternalToolInfo[],
|
||||
|
||||
messages: LLMChatMessage[];
|
||||
}
|
||||
) => void
|
||||
|
||||
export type _InternalSendLLMFIMMessageFnType = (
|
||||
params: {
|
||||
onText: OnText;
|
||||
onFinalMessage: OnFinalMessage;
|
||||
onError: OnError;
|
||||
providerName: ProviderName;
|
||||
settingsOfProvider: SettingsOfProvider;
|
||||
modelName: string;
|
||||
_setAborter: (aborter: () => void) => void;
|
||||
|
||||
messages: _InternalSendFIMMessage;
|
||||
}
|
||||
) => void
|
||||
|
||||
// service -> main -> internal -> event (back to main)
|
||||
// (browser)
|
||||
|
||||
@ -181,18 +149,22 @@ export type OllamaModelResponse = {
|
||||
size_vram: number;
|
||||
}
|
||||
|
||||
export type OpenaiCompatibleModelResponse = {
|
||||
type OpenaiCompatibleModelResponse = {
|
||||
id: string;
|
||||
created: number;
|
||||
object: 'model';
|
||||
owned_by: string;
|
||||
}
|
||||
|
||||
export type VLLMModelResponse = OpenaiCompatibleModelResponse
|
||||
|
||||
|
||||
|
||||
// params to the true list fn
|
||||
export type ModelListParams<modelResponse> = {
|
||||
export type ModelListParams<ModelResponse> = {
|
||||
providerName: ProviderName;
|
||||
settingsOfProvider: SettingsOfProvider;
|
||||
onSuccess: (param: { models: modelResponse[] }) => void;
|
||||
onSuccess: (param: { models: ModelResponse[] }) => void;
|
||||
onError: (param: { error: string }) => void;
|
||||
}
|
||||
|
||||
@ -211,4 +183,3 @@ export type EventModelListOnErrorParams<modelResponse> = Parameters<ModelListPar
|
||||
|
||||
|
||||
|
||||
export type _InternalModelListFnType<modelResponse> = (params: ModelListParams<modelResponse>) => void
|
||||
|
||||
@ -8,7 +8,7 @@ import { ILLMMessageService } from './llmMessageService.js';
|
||||
import { Emitter, Event } from '../../../../base/common/event.js';
|
||||
import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js';
|
||||
import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js';
|
||||
import { OllamaModelResponse, OpenaiCompatibleModelResponse } from './llmMessageTypes.js';
|
||||
import { OllamaModelResponse, VLLMModelResponse } from './llmMessageTypes.js';
|
||||
import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
|
||||
@ -160,9 +160,8 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
||||
}
|
||||
}
|
||||
const listFn = providerName === 'ollama' ? this.llmMessageService.ollamaList
|
||||
: providerName === 'vLLM' ? this.llmMessageService.openAICompatibleList
|
||||
: providerName === 'openAICompatible' ? this.llmMessageService.openAICompatibleList
|
||||
: () => { }
|
||||
: providerName === 'vLLM' ? this.llmMessageService.vLLMList
|
||||
: () => { }
|
||||
|
||||
listFn({
|
||||
onSuccess: ({ models }) => {
|
||||
@ -172,8 +171,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
||||
providerName,
|
||||
models.map(model => {
|
||||
if (providerName === 'ollama') return (model as OllamaModelResponse).name;
|
||||
else if (providerName === 'vLLM') return (model as OpenaiCompatibleModelResponse).id;
|
||||
else if (providerName === 'openAICompatible') return (model as OpenaiCompatibleModelResponse).id;
|
||||
else if (providerName === 'vLLM') return (model as VLLMModelResponse).id;
|
||||
else throw new Error('refreshMode fn: unknown provider', providerName);
|
||||
}),
|
||||
{ enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire }
|
||||
|
||||
@ -89,6 +89,13 @@ export const voidTools = {
|
||||
export type ToolName = keyof typeof voidTools
|
||||
export const toolNames = Object.keys(voidTools) as ToolName[]
|
||||
|
||||
const toolNamesSet = new Set<string>(toolNames)
|
||||
export const isAToolName = (toolName: string): toolName is ToolName => {
|
||||
const isAToolName = toolNamesSet.has(toolName)
|
||||
return isAToolName
|
||||
}
|
||||
|
||||
|
||||
export type ToolParamNames<T extends ToolName> = keyof typeof voidTools[T]['params']
|
||||
export type ToolParamsObj<T extends ToolName> = { [paramName in ToolParamNames<T>]: unknown }
|
||||
|
||||
|
||||
@ -4,367 +4,13 @@
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { defaultModelsOfProvider } from '../electron-main/llmMessage/MODELS.js';
|
||||
import { VoidSettingsState } from './voidSettingsService.js'
|
||||
|
||||
|
||||
|
||||
// developer info used in sendLLMMessage
|
||||
export type DeveloperInfoAtModel = {
|
||||
// USED:
|
||||
supportsSystemMessage: 'developer' | boolean, // if null, we will just do a string of system message. this is independent from separateSystemMessage, which takes priority and is passed directly in each provider's implementation.
|
||||
supportsTools: boolean, // we will just do a string of tool use if it doesn't support
|
||||
|
||||
// UNUSED (coming soon):
|
||||
// TODO!!! think tokens - deepseek
|
||||
_recognizedModelName: RecognizedModelName, // used to show user if model was auto-recognized
|
||||
_supportsStreaming: boolean, // we will just dump the final result if doesn't support it
|
||||
_supportsAutocompleteFIM: boolean, // we will just do a description of FIM if it doens't support <|fim_hole|>
|
||||
_maxTokens: number, // required
|
||||
}
|
||||
|
||||
export type DeveloperInfoAtProvider = {
|
||||
overrideSettingsForAllModels?: Partial<DeveloperInfoAtModel>; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
export type VoidModelInfo = { // <-- STATEFUL
|
||||
modelName: string,
|
||||
isDefault: boolean, // whether or not it's a default for its provider
|
||||
isHidden: boolean, // whether or not the user is hiding it (switched off)
|
||||
isAutodetected?: boolean, // whether the model was autodetected by polling
|
||||
} & DeveloperInfoAtModel
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
export const recognizedModels = [
|
||||
// chat
|
||||
'OpenAI 4o',
|
||||
'Anthropic Claude',
|
||||
'Llama 3.x',
|
||||
'Deepseek Chat', // deepseek coder v2 is now merged into chat (V3) https://api-docs.deepseek.com/updates#deepseek-coder--deepseek-chat-upgraded-to-deepseek-v25-model
|
||||
'xAI Grok',
|
||||
// 'xAI Grok',
|
||||
// 'Google Gemini, Gemma',
|
||||
// 'Microsoft Phi4',
|
||||
|
||||
|
||||
// coding (autocomplete)
|
||||
'Alibaba Qwen2.5 Coder Instruct', // we recommend this over Qwen2.5
|
||||
'Mistral Codestral',
|
||||
|
||||
// thinking
|
||||
'OpenAI o1',
|
||||
'Deepseek R1',
|
||||
|
||||
// general
|
||||
// 'Mixtral 8x7b'
|
||||
// 'Qwen2.5',
|
||||
|
||||
] as const
|
||||
|
||||
type RecognizedModelName = (typeof recognizedModels)[number] | '<GENERAL>'
|
||||
|
||||
|
||||
export function recognizedModelOfModelName(modelName: string): RecognizedModelName {
|
||||
const lower = modelName.toLowerCase();
|
||||
|
||||
if (lower.includes('gpt-4o'))
|
||||
return 'OpenAI 4o';
|
||||
if (lower.includes('claude'))
|
||||
return 'Anthropic Claude';
|
||||
if (lower.includes('llama'))
|
||||
return 'Llama 3.x';
|
||||
if (lower.includes('qwen2.5-coder'))
|
||||
return 'Alibaba Qwen2.5 Coder Instruct';
|
||||
if (lower.includes('mistral'))
|
||||
return 'Mistral Codestral';
|
||||
if (/\bo1\b/.test(lower) || /\bo3\b/.test(lower)) // o1, o3
|
||||
return 'OpenAI o1';
|
||||
if (lower.includes('deepseek-r1') || lower.includes('deepseek-reasoner'))
|
||||
return 'Deepseek R1';
|
||||
if (lower.includes('deepseek'))
|
||||
return 'Deepseek Chat'
|
||||
if (lower.includes('grok'))
|
||||
return 'xAI Grok'
|
||||
|
||||
return '<GENERAL>';
|
||||
}
|
||||
|
||||
|
||||
const developerInfoAtProvider: { [providerName in ProviderName]: DeveloperInfoAtProvider } = {
|
||||
'anthropic': {
|
||||
overrideSettingsForAllModels: {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: true,
|
||||
}
|
||||
},
|
||||
'deepseek': {
|
||||
overrideSettingsForAllModels: {
|
||||
}
|
||||
},
|
||||
'ollama': {
|
||||
},
|
||||
'openRouter': {
|
||||
},
|
||||
'openAICompatible': {
|
||||
},
|
||||
'openAI': {
|
||||
},
|
||||
'gemini': {
|
||||
},
|
||||
'mistral': {
|
||||
},
|
||||
'groq': {
|
||||
},
|
||||
'xAI': {
|
||||
},
|
||||
'vLLM': {
|
||||
},
|
||||
}
|
||||
export const developerInfoOfProviderName = (providerName: ProviderName): Partial<DeveloperInfoAtProvider> => {
|
||||
return developerInfoAtProvider[providerName] ?? {}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// providerName is optional, but gives some extra fallbacks if provided
|
||||
const developerInfoOfRecognizedModelName: { [recognizedModel in RecognizedModelName]: Omit<DeveloperInfoAtModel, '_recognizedModelName'> } = {
|
||||
'OpenAI 4o': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: true,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'Anthropic Claude': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: false,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'Llama 3.x': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'xAI Grok': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: true,
|
||||
_maxTokens: 4096,
|
||||
|
||||
},
|
||||
|
||||
'Deepseek Chat': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: false,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'Alibaba Qwen2.5 Coder Instruct': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'Mistral Codestral': {
|
||||
supportsSystemMessage: true,
|
||||
supportsTools: true,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'OpenAI o1': {
|
||||
supportsSystemMessage: 'developer',
|
||||
supportsTools: false,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: true,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
'Deepseek R1': {
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
|
||||
|
||||
'<GENERAL>': {
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
_supportsAutocompleteFIM: false,
|
||||
_supportsStreaming: false,
|
||||
_maxTokens: 4096,
|
||||
},
|
||||
}
|
||||
export const developerInfoOfModelName = (modelName: string, overrides?: Partial<DeveloperInfoAtModel>): DeveloperInfoAtModel => {
|
||||
const recognizedModelName = recognizedModelOfModelName(modelName)
|
||||
return {
|
||||
_recognizedModelName: recognizedModelName,
|
||||
...developerInfoOfRecognizedModelName[recognizedModelName],
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// creates `modelInfo` from `modelNames`
|
||||
export const modelInfoOfDefaultModelNames = (defaultModelNames: string[]): VoidModelInfo[] => {
|
||||
return defaultModelNames.map((modelName, i) => ({
|
||||
modelName,
|
||||
isDefault: true,
|
||||
isAutodetected: false,
|
||||
isHidden: defaultModelNames.length >= 10, // hide all models if there are a ton of them, and make user enable them individually
|
||||
...developerInfoOfModelName(modelName),
|
||||
}))
|
||||
}
|
||||
|
||||
export const modelInfoOfAutodetectedModelNames = (defaultModelNames: string[], options: { existingModels: VoidModelInfo[] }) => {
|
||||
const { existingModels } = options
|
||||
|
||||
const existingModelsMap: Record<string, VoidModelInfo> = {}
|
||||
for (const existingModel of existingModels) {
|
||||
existingModelsMap[existingModel.modelName] = existingModel
|
||||
}
|
||||
|
||||
return defaultModelNames.map((modelName, i) => ({
|
||||
modelName,
|
||||
isDefault: true,
|
||||
isAutodetected: true,
|
||||
isHidden: !!existingModelsMap[modelName]?.isHidden,
|
||||
...developerInfoOfModelName(modelName)
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// https://docs.anthropic.com/en/docs/about-claude/models
|
||||
export const defaultAnthropicModels = modelInfoOfDefaultModelNames([
|
||||
'claude-3-5-sonnet-20241022',
|
||||
'claude-3-5-haiku-20241022',
|
||||
'claude-3-opus-20240229',
|
||||
'claude-3-sonnet-20240229',
|
||||
// 'claude-3-haiku-20240307',
|
||||
])
|
||||
|
||||
|
||||
// https://platform.openai.com/docs/models/gp
|
||||
export const defaultOpenAIModels = modelInfoOfDefaultModelNames([
|
||||
'o1',
|
||||
'o1-mini',
|
||||
'o3-mini',
|
||||
'gpt-4o',
|
||||
'gpt-4o-mini',
|
||||
// 'gpt-4o-2024-05-13',
|
||||
// 'gpt-4o-2024-08-06',
|
||||
// 'gpt-4o-mini-2024-07-18',
|
||||
// 'gpt-4-turbo',
|
||||
// 'gpt-4-turbo-2024-04-09',
|
||||
// 'gpt-4-turbo-preview',
|
||||
// 'gpt-4-0125-preview',
|
||||
// 'gpt-4-1106-preview',
|
||||
// 'gpt-4',
|
||||
// 'gpt-4-0613',
|
||||
// 'gpt-3.5-turbo-0125',
|
||||
// 'gpt-3.5-turbo',
|
||||
// 'gpt-3.5-turbo-1106',
|
||||
])
|
||||
|
||||
// https://platform.openai.com/docs/models/gp
|
||||
export const defaultDeepseekModels = modelInfoOfDefaultModelNames([
|
||||
'deepseek-chat',
|
||||
'deepseek-reasoner',
|
||||
])
|
||||
|
||||
|
||||
// https://console.groq.com/docs/models
|
||||
export const defaultGroqModels = modelInfoOfDefaultModelNames([
|
||||
"llama3-70b-8192",
|
||||
"llama-3.3-70b-versatile",
|
||||
"llama-3.1-8b-instant",
|
||||
"gemma2-9b-it",
|
||||
"mixtral-8x7b-32768"
|
||||
])
|
||||
|
||||
|
||||
export const defaultGeminiModels = modelInfoOfDefaultModelNames([
|
||||
'gemini-1.5-flash',
|
||||
'gemini-1.5-pro',
|
||||
'gemini-1.5-flash-8b',
|
||||
'gemini-2.0-flash-exp',
|
||||
'gemini-2.0-flash-thinking-exp-1219',
|
||||
'learnlm-1.5-pro-experimental'
|
||||
])
|
||||
|
||||
export const defaultMistralModels = modelInfoOfDefaultModelNames([
|
||||
"codestral-latest",
|
||||
"open-codestral-mamba",
|
||||
"open-mistral-nemo",
|
||||
"mistral-large-latest",
|
||||
"pixtral-large-latest",
|
||||
"ministral-3b-latest",
|
||||
"ministral-8b-latest",
|
||||
"mistral-small-latest",
|
||||
])
|
||||
|
||||
export const defaultXAIModels = modelInfoOfDefaultModelNames([
|
||||
'grok-2-latest',
|
||||
'grok-3-latest',
|
||||
])
|
||||
// export const parseMaxTokensStr = (maxTokensStr: string) => {
|
||||
// // parse the string but only if the full string is a valid number, eg parseInt('100abc') should return NaN
|
||||
// const int = isNaN(Number(maxTokensStr)) ? undefined : parseInt(maxTokensStr)
|
||||
// if (Number.isNaN(int))
|
||||
// return undefined
|
||||
// return int
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
export const anthropicMaxPossibleTokens = (modelName: string) => {
|
||||
if (modelName === 'claude-3-5-sonnet-20241022'
|
||||
|| modelName === 'claude-3-5-haiku-20241022')
|
||||
return 8192
|
||||
if (modelName === 'claude-3-opus-20240229'
|
||||
|| modelName === 'claude-3-sonnet-20240229'
|
||||
|| modelName === 'claude-3-haiku-20240307')
|
||||
return 4096
|
||||
return 1024 // return a reasonably small number if they're using a different model
|
||||
}
|
||||
|
||||
|
||||
type UnionOfKeys<T> = T extends T ? keyof T : never;
|
||||
|
||||
|
||||
|
||||
export const defaultProviderSettings = {
|
||||
anthropic: {
|
||||
apiKey: '',
|
||||
@ -418,6 +64,14 @@ export const customSettingNamesOfProvider = (providerName: ProviderName) => {
|
||||
|
||||
|
||||
|
||||
export type VoidModelInfo = { // <-- STATEFUL
|
||||
modelName: string,
|
||||
isDefault: boolean, // whether or not it's a default for its provider
|
||||
isHidden: boolean, // whether or not the user is hiding it (switched off)
|
||||
isAutodetected?: boolean, // whether the model was autodetected by polling
|
||||
} // TODO!!! eventually we'd want to let the user change supportsFIM, etc on the model themselves
|
||||
|
||||
|
||||
|
||||
type CommonProviderSettings = {
|
||||
_didFillInProviderSettings: boolean | undefined, // undefined initially, computed when user types in all fields
|
||||
@ -434,10 +88,6 @@ export type SettingsOfProvider = {
|
||||
|
||||
export type SettingName = keyof SettingsAtProvider<ProviderName>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
type DisplayInfoForProviderName = {
|
||||
title: string,
|
||||
desc?: string,
|
||||
@ -584,110 +234,83 @@ const defaultCustomSettings: Record<CustomSettingName, undefined> = {
|
||||
}
|
||||
|
||||
|
||||
|
||||
export const voidInitModelOptions = {
|
||||
anthropic: {
|
||||
models: defaultAnthropicModels,
|
||||
},
|
||||
openAI: {
|
||||
models: defaultOpenAIModels,
|
||||
},
|
||||
deepseek: {
|
||||
models: defaultDeepseekModels,
|
||||
},
|
||||
ollama: {
|
||||
models: [],
|
||||
},
|
||||
vLLM: {
|
||||
models: [],
|
||||
},
|
||||
openRouter: {
|
||||
models: [], // any string
|
||||
},
|
||||
openAICompatible: {
|
||||
models: [],
|
||||
},
|
||||
gemini: {
|
||||
models: defaultGeminiModels,
|
||||
},
|
||||
groq: {
|
||||
models: defaultGroqModels,
|
||||
},
|
||||
mistral: {
|
||||
models: defaultMistralModels,
|
||||
},
|
||||
xAI: {
|
||||
models: defaultXAIModels,
|
||||
const modelInfoOfDefaultModelNames = (defaultModelNames: string[]): { models: VoidModelInfo[] } => {
|
||||
return {
|
||||
models: defaultModelNames.map((modelName, i) => ({
|
||||
modelName,
|
||||
isDefault: true,
|
||||
isAutodetected: false,
|
||||
isHidden: defaultModelNames.length >= 10, // hide all models if there are a ton of them, and make user enable them individually
|
||||
}))
|
||||
}
|
||||
} satisfies Record<ProviderName, any>
|
||||
|
||||
}
|
||||
|
||||
// used when waiting and for a type reference
|
||||
export const defaultSettingsOfProvider: SettingsOfProvider = {
|
||||
anthropic: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.anthropic,
|
||||
...voidInitModelOptions.anthropic,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.anthropic),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
openAI: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.openAI,
|
||||
...voidInitModelOptions.openAI,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openAI),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
deepseek: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.deepseek,
|
||||
...voidInitModelOptions.deepseek,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.deepseek),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
gemini: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.gemini,
|
||||
...voidInitModelOptions.gemini,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.gemini),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
mistral: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.mistral,
|
||||
...voidInitModelOptions.mistral,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.mistral),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
xAI: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.xAI,
|
||||
...voidInitModelOptions.xAI,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.xAI),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
groq: { // aggregator
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.groq,
|
||||
...voidInitModelOptions.groq,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.groq),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
openRouter: { // aggregator
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.openRouter,
|
||||
...voidInitModelOptions.openRouter,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openRouter),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
openAICompatible: { // aggregator
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.openAICompatible,
|
||||
...voidInitModelOptions.openAICompatible,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openAICompatible),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
ollama: { // aggregator
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.ollama,
|
||||
...voidInitModelOptions.ollama,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.ollama),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
vLLM: { // aggregator
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.vLLM,
|
||||
...voidInitModelOptions.vLLM,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.vLLM),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
}
|
||||
|
||||
776
src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts
Normal file
776
src/vs/workbench/contrib/void/electron-main/llmMessage/MODELS.ts
Normal file
@ -0,0 +1,776 @@
|
||||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import OpenAI, { ClientOptions } from 'openai';
|
||||
import { Model as OpenAIModel } from 'openai/resources/models.js';
|
||||
import { OllamaModelResponse, OnText, OnFinalMessage, OnError, LLMChatMessage, LLMFIMMessage, ModelListParams } from '../../common/llmMessageTypes.js';
|
||||
import { InternalToolInfo, isAToolName } from '../../common/toolsService.js';
|
||||
import { defaultProviderSettings, displayInfoOfProviderName, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js';
|
||||
import { prepareMessages } from './preprocessLLMMessages.js';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { Ollama } from 'ollama';
|
||||
|
||||
|
||||
|
||||
export const defaultModelsOfProvider = {
|
||||
anthropic: [ // https://docs.anthropic.com/en/docs/about-claude/models
|
||||
'claude-3-5-sonnet-latest',
|
||||
'claude-3-5-haiku-latest',
|
||||
'claude-3-opus-latest',
|
||||
],
|
||||
openAI: [ // https://platform.openai.com/docs/models/gp
|
||||
'o1',
|
||||
'o1-mini',
|
||||
'o3-mini',
|
||||
'gpt-4o',
|
||||
'gpt-4o-mini',
|
||||
],
|
||||
deepseek: [ // https://platform.openai.com/docs/models/gp
|
||||
'deepseek-chat',
|
||||
'deepseek-reasoner',
|
||||
],
|
||||
ollama: [],
|
||||
vLLM: [],
|
||||
openRouter: [],
|
||||
openAICompatible: [],
|
||||
gemini: [
|
||||
'gemini-1.5-flash',
|
||||
'gemini-1.5-pro',
|
||||
'gemini-1.5-flash-8b',
|
||||
'gemini-2.0-flash-exp',
|
||||
'gemini-2.0-flash-thinking-exp-1219',
|
||||
'learnlm-1.5-pro-experimental'
|
||||
],
|
||||
groq: [ // https://console.groq.com/docs/models
|
||||
"llama3-70b-8192",
|
||||
"llama-3.3-70b-versatile",
|
||||
"llama-3.1-8b-instant",
|
||||
"gemma2-9b-it",
|
||||
"mixtral-8x7b-32768"
|
||||
],
|
||||
mistral: [ // https://docs.mistral.ai/getting-started/models/models_overview/
|
||||
"codestral-latest",
|
||||
"open-codestral-mamba",
|
||||
"open-mistral-nemo",
|
||||
"mistral-large-latest",
|
||||
"pixtral-large-latest",
|
||||
"ministral-3b-latest",
|
||||
"ministral-8b-latest",
|
||||
"mistral-small-latest",
|
||||
],
|
||||
xAI: [ // https://docs.x.ai/docs/models?cluster=us-east-1
|
||||
'grok-3-latest',
|
||||
'grok-2-latest',
|
||||
],
|
||||
} satisfies Record<ProviderName, string[]>
|
||||
|
||||
|
||||
|
||||
type ModelOptions = {
|
||||
contextWindow: number;
|
||||
cost: {
|
||||
input: number;
|
||||
output: number;
|
||||
cache_read?: number;
|
||||
cache_write?: number;
|
||||
}
|
||||
supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated';
|
||||
supportsTools: false | 'anthropic-style' | 'openai-style';
|
||||
supportsFIM: false | 'TODO_FIM_FORMAT';
|
||||
|
||||
supportsReasoning: boolean; // not whether it reasons, but whether it outputs reasoning tokens
|
||||
manualMatchReasoningTokens?: [string, string]; // reasoning tokens if it's an OSS model
|
||||
}
|
||||
|
||||
type ProviderReasoningOptions = {
|
||||
// include this in payload to get reasoning
|
||||
input?: { includeInPayload?: { [key: string]: any }, };
|
||||
// nameOfFieldInDelta: reasoning output is in response.choices[0].delta[deltaReasoningField]
|
||||
// needsManualParse: whether we must manually parse out the <think> tags
|
||||
output?:
|
||||
| { nameOfFieldInDelta?: string, needsManualParse?: undefined, }
|
||||
| { nameOfFieldInDelta?: undefined, needsManualParse?: true, };
|
||||
}
|
||||
|
||||
type ProviderSettings = {
|
||||
providerReasoningOptions?: ProviderReasoningOptions;
|
||||
modelOptions: { [key: string]: ModelOptions };
|
||||
modelOptionsFallback: (modelName: string) => ModelOptions; // allowed to throw error if modeName is totally invalid
|
||||
}
|
||||
|
||||
|
||||
type ModelSettingsOfProvider = {
|
||||
[providerName in ProviderName]: ProviderSettings
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const modelNotRecognizedErrorMessage = (modelName: string, providerName: ProviderName) => `Void could not find a model matching ${modelName} for ${displayInfoOfProviderName(providerName).title}.`
|
||||
|
||||
|
||||
|
||||
// ---------------- OPENAI ----------------
|
||||
const openAIModelOptions = {
|
||||
"o1": {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 15.00, cache_read: 7.50, output: 60.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"o3-mini": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 1.10, cache_read: 0.55, output: 4.40, },
|
||||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"gpt-4o": {
|
||||
contextWindow: 128_000,
|
||||
cost: { input: 2.50, cache_read: 1.25, output: 10.00, },
|
||||
supportsFIM: false,
|
||||
supportsTools: 'openai-style',
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
} as const
|
||||
|
||||
const openAISettings: ProviderSettings = {
|
||||
modelOptions: openAIModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
if (modelName.includes('o1')) return openAIModelOptions['o1']
|
||||
if (modelName.includes('o3-mini')) return openAIModelOptions['o3-mini']
|
||||
if (modelName.includes('gpt-4o')) return openAIModelOptions['gpt-4o']
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'openAI'))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- ANTHROPIC ----------------
|
||||
const anthropicModelOptions = {
|
||||
"claude-3-5-sonnet-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 3.00, cache_read: 0.30, cache_write: 3.75, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
|
||||
},
|
||||
"claude-3-5-haiku-20241022": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 0.80, cache_read: 0.08, cache_write: 1.00, output: 4.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
contextWindow: 200_000,
|
||||
cost: { input: 15.00, cache_read: 1.50, cache_write: 18.75, output: 75.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
"claude-3-sonnet-20240229": {
|
||||
contextWindow: 200_000, cost: { input: 3.00, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
}
|
||||
} as const
|
||||
|
||||
const anthropicSettings: ProviderSettings = {
|
||||
modelOptions: anthropicModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'anthropic'))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ---------------- XAI ----------------
|
||||
const XAIModelOptions = {
|
||||
"grok-2-latest": {
|
||||
contextWindow: 131_072,
|
||||
cost: { input: 2.00, output: 10.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
},
|
||||
} as const
|
||||
|
||||
const XAISettings: ProviderSettings = {
|
||||
modelOptions: XAIModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
throw new Error(modelNotRecognizedErrorMessage(modelName, 'xAI'))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
const modelSettingsOfProvider: ModelSettingsOfProvider = {
|
||||
openAI: openAISettings,
|
||||
anthropic: anthropicSettings,
|
||||
xAI: XAISettings,
|
||||
gemini: {
|
||||
modelOptions: {
|
||||
|
||||
}
|
||||
},
|
||||
googleVertex: {
|
||||
|
||||
},
|
||||
microsoftAzure: {
|
||||
|
||||
},
|
||||
openRouter: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models
|
||||
input: { includeInPayload: { include_reasoning: true } },
|
||||
output: { nameOfFieldInDelta: 'reasoning' },
|
||||
}
|
||||
},
|
||||
vLLM: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions
|
||||
output: { nameOfFieldInDelta: 'reasoning_content' },
|
||||
}
|
||||
},
|
||||
deepseek: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://api-docs.deepseek.com/guides/reasoning_model
|
||||
output: { nameOfFieldInDelta: 'reasoning_content' },
|
||||
},
|
||||
},
|
||||
ollama: {
|
||||
providerReasoningOptions: {
|
||||
// reasoning: we need to filter out reasoning <think> tags manually
|
||||
output: { needsManualParse: true },
|
||||
},
|
||||
},
|
||||
|
||||
openAICompatible: {
|
||||
},
|
||||
mistral: {
|
||||
},
|
||||
groq: {
|
||||
},
|
||||
|
||||
|
||||
|
||||
} as const satisfies ModelSettingsOfProvider
|
||||
|
||||
|
||||
const modelOptionsOfProvider = (providerName: ProviderName, modelName: string) => {
|
||||
const { modelOptions, modelOptionsFallback } = modelSettingsOfProvider[providerName]
|
||||
if (modelName in modelOptions) return modelOptions[modelName]
|
||||
return modelOptionsFallback(modelName)
|
||||
}
|
||||
|
||||
|
||||
|
||||
type InternalCommonMessageParams = {
|
||||
aiInstructions: string;
|
||||
onText: OnText;
|
||||
onFinalMessage: OnFinalMessage;
|
||||
onError: OnError;
|
||||
providerName: ProviderName;
|
||||
settingsOfProvider: SettingsOfProvider;
|
||||
modelName: string;
|
||||
_setAborter: (aborter: () => void) => void;
|
||||
}
|
||||
|
||||
type SendChatParams_Internal = InternalCommonMessageParams & { messages: LLMChatMessage[]; tools?: InternalToolInfo[] }
|
||||
type SendFIMParams_Internal = InternalCommonMessageParams & { messages: LLMFIMMessage; }
|
||||
export type ListParams_Internal<ModelResponse> = ModelListParams<ModelResponse>
|
||||
|
||||
|
||||
// ------------ OPENAI-COMPATIBLE (HELPERS) ------------
|
||||
const toOpenAICompatibleTool = (toolInfo: InternalToolInfo) => {
|
||||
const { name, description, params, required } = toolInfo
|
||||
return {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: name,
|
||||
description: description,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: params,
|
||||
required: required,
|
||||
}
|
||||
}
|
||||
} satisfies OpenAI.Chat.Completions.ChatCompletionTool
|
||||
}
|
||||
|
||||
type ToolCallOfIndex = { [index: string]: { name: string, params: string, id: string } }
|
||||
|
||||
const toolCallsFrom_OpenAICompat = (toolCallOfIndex: ToolCallOfIndex) => {
|
||||
return Object.keys(toolCallOfIndex).map(index => {
|
||||
const tool = toolCallOfIndex[index]
|
||||
return isAToolName(tool.name) ? { name: tool.name, id: tool.id, params: tool.params } : null
|
||||
}).filter(t => !!t)
|
||||
}
|
||||
|
||||
|
||||
const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => {
|
||||
const commonPayloadOpts: ClientOptions = {
|
||||
dangerouslyAllowBrowser: true,
|
||||
...includeInPayload,
|
||||
}
|
||||
if (providerName === 'openAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'ollama') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'vLLM') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'openRouter') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
apiKey: thisConfig.apiKey,
|
||||
defaultHeaders: {
|
||||
'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings.
|
||||
'X-Title': 'Void', // Optional. Shows in rankings on openrouter.ai.
|
||||
},
|
||||
...commonPayloadOpts,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'gemini') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'deepseek') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'openAICompatible') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'groq') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'xAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
|
||||
else throw new Error(`Void providerName was invalid: ${providerName}.`)
|
||||
}
|
||||
|
||||
|
||||
|
||||
const manualParseOnText = (
|
||||
providerName: ProviderName,
|
||||
modelName: string,
|
||||
onText_: OnText
|
||||
): OnText => {
|
||||
return onText_
|
||||
}
|
||||
|
||||
|
||||
const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const {
|
||||
supportsReasoning: modelSupportsReasoning,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
} = modelOptionsOfProvider(providerName, modelName)
|
||||
|
||||
const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, })
|
||||
const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAICompatibleTool(tool)) : undefined
|
||||
|
||||
const includeInPayload = modelSupportsReasoning ? {} : modelSettingsOfProvider[providerName].providerReasoningOptions?.input?.includeInPayload || {}
|
||||
|
||||
const toolsObj = tools ? { tools: tools, tool_choice: 'auto', parallel_tool_calls: false, } as const : {}
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages, stream: true, ...toolsObj }
|
||||
|
||||
const { nameOfFieldInDelta: nameOfReasoningFieldInDelta, needsManualParse: needsManualReasoningParse } = modelSettingsOfProvider[providerName].providerReasoningOptions?.output ?? {}
|
||||
if (needsManualReasoningParse) onText = manualParseOnText(providerName, modelName, onText)
|
||||
|
||||
let fullReasoning = ''
|
||||
let fullText = ''
|
||||
const toolCallOfIndex: ToolCallOfIndex = {}
|
||||
openai.chat.completions
|
||||
.create(options)
|
||||
.then(async response => {
|
||||
_setAborter(() => response.controller.abort())
|
||||
// when receive text
|
||||
for await (const chunk of response) {
|
||||
// tool call
|
||||
for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) {
|
||||
const index = tool.index
|
||||
if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', params: '', id: '' }
|
||||
toolCallOfIndex[index].name += tool.function?.name ?? ''
|
||||
toolCallOfIndex[index].params += tool.function?.arguments ?? '';
|
||||
toolCallOfIndex[index].id = tool.id ?? ''
|
||||
}
|
||||
// message
|
||||
const newText = chunk.choices[0]?.delta?.content ?? ''
|
||||
fullText += newText
|
||||
|
||||
// reasoning
|
||||
let newReasoning = ''
|
||||
if (nameOfReasoningFieldInDelta) {
|
||||
// @ts-ignore
|
||||
newReasoning = (chunk.choices[0]?.delta?.[nameOfFieldInDelta] || '') + ''
|
||||
fullReasoning += newReasoning
|
||||
}
|
||||
|
||||
onText({ newText, fullText, newReasoning, fullReasoning })
|
||||
}
|
||||
onFinalMessage({ fullText, toolCalls: toolCallsFrom_OpenAICompat(toolCallOfIndex) });
|
||||
})
|
||||
// when error/fail - this catches errors of both .create() and .then(for await)
|
||||
.catch(error => {
|
||||
if (error instanceof OpenAI.APIError && error.status === 401) { onError({ message: 'Invalid API key.', fullError: error }); }
|
||||
else { onError({ message: error + '', fullError: error }); }
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal<OpenAIModel>) => {
|
||||
const onSuccess = ({ models }: { models: OpenAIModel[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
const onError = ({ error }: { error: string }) => {
|
||||
onError_({ error })
|
||||
}
|
||||
try {
|
||||
const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
openai.models.list()
|
||||
.then(async (response) => {
|
||||
const models: OpenAIModel[] = []
|
||||
models.push(...response.data)
|
||||
while (response.hasNextPage()) {
|
||||
models.push(...(await response.getNextPage()).data)
|
||||
}
|
||||
onSuccess({ models })
|
||||
})
|
||||
.catch((error) => {
|
||||
onError({ error: error + '' })
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
onError({ error: error + '' })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ------------ OPENAI ------------
|
||||
const sendOpenAIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ ANTHROPIC ------------
|
||||
const toAnthropicTool = (toolInfo: InternalToolInfo) => {
|
||||
const { name, description, params, required } = toolInfo
|
||||
return {
|
||||
name: name,
|
||||
description: description,
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: params,
|
||||
required: required,
|
||||
}
|
||||
} satisfies Anthropic.Messages.Tool
|
||||
}
|
||||
|
||||
const toolCallsFromAnthropicContent = (content: Anthropic.Messages.ContentBlock[]) => {
|
||||
return content.map(c => {
|
||||
if (c.type !== 'tool_use') return null
|
||||
if (!isAToolName(c.name)) return null
|
||||
return c.type === 'tool_use' ? { name: c.name, params: JSON.stringify(c.input), id: c.id } : null
|
||||
}).filter(t => !!t)
|
||||
}
|
||||
|
||||
const sendAnthropicChat = ({ messages: messages_, onText, providerName, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const {
|
||||
// supportsReasoning: modelSupportsReasoning,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
contextWindow,
|
||||
} = modelOptionsOfProvider(providerName, modelName)
|
||||
|
||||
const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, })
|
||||
|
||||
const thisConfig = settingsOfProvider.anthropic
|
||||
const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
const tools = ((tools_?.length ?? 0) !== 0) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined
|
||||
|
||||
const stream = anthropic.messages.stream({
|
||||
system: separateSystemMessageStr,
|
||||
messages: messages,
|
||||
model: modelName,
|
||||
max_tokens: contextWindow,
|
||||
tools: tools,
|
||||
tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time
|
||||
})
|
||||
// when receive text
|
||||
stream.on('text', (newText, fullText) => {
|
||||
onText({ newText, fullText, newReasoning: '', fullReasoning: '' })
|
||||
})
|
||||
// when we get the final message on this stream (or when error/fail)
|
||||
stream.on('finalMessage', (response) => {
|
||||
const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n')
|
||||
const toolCalls = toolCallsFromAnthropicContent(response.content)
|
||||
onFinalMessage({ fullText: content, toolCalls })
|
||||
})
|
||||
// on error
|
||||
stream.on('error', (error) => {
|
||||
if (error instanceof Anthropic.APIError && error.status === 401) { onError({ message: 'Invalid API key.', fullError: error }) }
|
||||
else { onError({ message: error + '', fullError: error }) }
|
||||
})
|
||||
_setAborter(() => stream.controller.abort())
|
||||
}
|
||||
|
||||
// // in future, can do tool_use streaming in anthropic, but it's pretty fast even without streaming...
|
||||
// const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {}
|
||||
// stream.on('streamEvent', e => {
|
||||
// if (e.type === 'content_block_start') {
|
||||
// if (e.content_block.type !== 'tool_use') return
|
||||
// const index = e.index
|
||||
// if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' }
|
||||
// toolCallOfIndex[index].name += e.content_block.name ?? ''
|
||||
// toolCallOfIndex[index].args += e.content_block.input ?? ''
|
||||
// }
|
||||
// else if (e.type === 'content_block_delta') {
|
||||
// if (e.delta.type !== 'input_json_delta') return
|
||||
// toolCallOfIndex[e.index].args += e.delta.partial_json
|
||||
// }
|
||||
// })
|
||||
|
||||
|
||||
// ------------ XAI ------------
|
||||
const sendXAIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ GEMINI ------------
|
||||
const sendGeminiAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ OLLAMA ------------
|
||||
const newOllamaSDK = ({ endpoint }: { endpoint: string }) => {
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
if (!endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} in Void if you want the default url).`)
|
||||
const ollama = new Ollama({ host: endpoint })
|
||||
return ollama
|
||||
}
|
||||
|
||||
const ollamaList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }: ListParams_Internal<OllamaModelResponse>) => {
|
||||
const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
const onError = ({ error }: { error: string }) => {
|
||||
onError_({ error })
|
||||
}
|
||||
try {
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint })
|
||||
ollama.list()
|
||||
.then((response) => {
|
||||
const { models } = response
|
||||
onSuccess({ models })
|
||||
})
|
||||
.catch((error) => {
|
||||
onError({ error: error + '' })
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
onError({ error: error + '' })
|
||||
}
|
||||
}
|
||||
|
||||
const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }: SendFIMParams_Internal) => {
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
const ollama = newOllamaSDK({ endpoint: thisConfig.endpoint })
|
||||
|
||||
let fullText = ''
|
||||
ollama.generate({
|
||||
model: modelName,
|
||||
prompt: messages.prefix,
|
||||
suffix: messages.suffix,
|
||||
options: {
|
||||
stop: messages.stopTokens,
|
||||
num_predict: 300, // max tokens
|
||||
// repeat_penalty: 1,
|
||||
},
|
||||
raw: true,
|
||||
stream: true, // stream is not necessary but lets us expose the
|
||||
})
|
||||
.then(async stream => {
|
||||
_setAborter(() => stream.abort())
|
||||
for await (const chunk of stream) {
|
||||
const newText = chunk.response
|
||||
fullText += newText
|
||||
}
|
||||
onFinalMessage({ fullText })
|
||||
})
|
||||
// when error/fail
|
||||
.catch((error) => {
|
||||
onError({ message: error + '', fullError: error })
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// ollama's implementation of openai-compatible SDK dumps all reasoning tokens out with message, and supports tools, so we can use it for chat!
|
||||
const sendOllamaChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ OPENAI-COMPATIBLE ------------
|
||||
// TODO!!! FIM
|
||||
|
||||
// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration
|
||||
const sendOpenAICompatibleChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ OPENROUTER ------------
|
||||
const sendOpenRouterChat = (params: SendChatParams_Internal) => {
|
||||
_sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ VLLM ------------
|
||||
const vLLMList = async (params: ListParams_Internal<OpenAIModel>) => {
|
||||
return _openaiCompatibleList(params)
|
||||
}
|
||||
const sendVLLMFIM = (params: SendFIMParams_Internal) => {
|
||||
// TODO!!!
|
||||
}
|
||||
|
||||
// using openai's SDK is not ideal (your implementation might not do tools, reasoning, FIM etc correctly), talk to us for a custom integration
|
||||
const sendVLLMChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ DEEPSEEK API ------------
|
||||
const sendDeepSeekAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ MISTRAL ------------
|
||||
const sendMistralAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
// ------------ GROQ ------------
|
||||
const sendGroqAPIChat = (params: SendChatParams_Internal) => {
|
||||
return _sendOpenAICompatibleChat(params)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
FIM:
|
||||
|
||||
qwen2.5-coder https://ollama.com/library/qwen2.5-coder/blobs/e94a8ecb9327
|
||||
<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|>
|
||||
|
||||
codestral https://ollama.com/library/codestral/blobs/51707752a87c
|
||||
[SUFFIX]{{ .Suffix }}[PREFIX] {{ .Prompt }}
|
||||
|
||||
deepseek-coder-v2 https://ollama.com/library/deepseek-coder-v2/blobs/22091531faf0
|
||||
<｜fim▁begin｜>{{ .Prompt }}<｜fim▁hole｜>{{ .Suffix }}<｜fim▁end｜>
|
||||
|
||||
starcoder2 https://ollama.com/library/starcoder2/blobs/3b190e68fefe
|
||||
<file_sep>
|
||||
<fim_prefix>
|
||||
{{ .Prompt }}<fim_suffix>{{ .Suffix }}<fim_middle>
|
||||
<|end_of_text|>
|
||||
|
||||
codegemma https://ollama.com/library/codegemma:2b/blobs/48d9a8140749
|
||||
<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|>
|
||||
|
||||
*/
|
||||
|
||||
|
||||
|
||||
type CallFnOfProvider = {
|
||||
[providerName in ProviderName]: {
|
||||
sendChat: (params: SendChatParams_Internal) => void;
|
||||
sendFIM: ((params: SendFIMParams_Internal) => void) | null;
|
||||
list: ((params: ListParams_Internal<any>) => void) | null;
|
||||
}
|
||||
}
|
||||
export const sendLLMMessageToProviderImplementation = {
|
||||
openAI: {
|
||||
sendChat: sendOpenAIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
anthropic: {
|
||||
sendChat: sendAnthropicChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
xAI: {
|
||||
sendChat: sendXAIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
gemini: {
|
||||
sendChat: sendGeminiAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
ollama: {
|
||||
sendChat: sendOllamaChat,
|
||||
sendFIM: sendOllamaFIM,
|
||||
list: ollamaList,
|
||||
},
|
||||
openAICompatible: {
|
||||
sendChat: sendOpenAICompatibleChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
openRouter: {
|
||||
sendChat: sendOpenRouterChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
vLLM: {
|
||||
sendChat: sendVLLMChat,
|
||||
sendFIM: sendVLLMFIM,
|
||||
list: vLLMList,
|
||||
},
|
||||
deepseek: {
|
||||
sendChat: sendDeepSeekAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
groq: {
|
||||
sendChat: sendGroqAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
mistral: {
|
||||
sendChat: sendMistralAPIChat,
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
|
||||
} satisfies CallFnOfProvider
|
||||
@ -1,96 +0,0 @@
|
||||
// /*--------------------------------------------------------------------------------------
|
||||
// * Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
// *--------------------------------------------------------------------------------------*/
|
||||
|
||||
// import Groq from 'groq-sdk';
|
||||
// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
|
||||
// // Groq
|
||||
// export const sendGroqChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
// let fullText = '';
|
||||
|
||||
// const thisConfig = settingsOfProvider.groq
|
||||
|
||||
// const groq = new Groq({
|
||||
// apiKey: thisConfig.apiKey,
|
||||
// dangerouslyAllowBrowser: true
|
||||
// });
|
||||
|
||||
// await groq.chat.completions
|
||||
// .create({
|
||||
// messages: messages,
|
||||
// model: modelName,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async response => {
|
||||
// _setAborter(() => response.controller.abort())
|
||||
// // when receive text
|
||||
// for await (const chunk of response) {
|
||||
// const newText = chunk.choices[0]?.delta?.content || '';
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// .catch(error => {
|
||||
// onError({ message: error + '', fullError: error });
|
||||
// })
|
||||
|
||||
|
||||
// };
|
||||
|
||||
|
||||
|
||||
// /*--------------------------------------------------------------------------------------
|
||||
// * Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
// *--------------------------------------------------------------------------------------*/
|
||||
|
||||
// import { Mistral } from '@mistralai/mistralai';
|
||||
// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
|
||||
// // Mistral
|
||||
// export const sendMistralChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
// let fullText = '';
|
||||
|
||||
// const thisConfig = settingsOfProvider.mistral;
|
||||
|
||||
// const mistral = new Mistral({
|
||||
// apiKey: thisConfig.apiKey,
|
||||
// })
|
||||
|
||||
// await mistral.chat
|
||||
// .stream({
|
||||
// messages: messages,
|
||||
// model: modelName,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async response => {
|
||||
// // Mistral has a really nonstandard API - no interrupt and weird stream types
|
||||
// _setAborter(() => { console.log('Mistral does not support interrupts! Further messages will just be ignored.') });
|
||||
// // when receive text
|
||||
// for await (const chunk of response) {
|
||||
// const c = chunk.data.choices[0].delta.content || ''
|
||||
// const newText = (
|
||||
// typeof c === 'string' ? c
|
||||
// : c?.map(c => c.type === 'text' ? c.text : c.type).join('\n')
|
||||
// )
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// .catch(error => {
|
||||
// onError({ message: error + '', fullError: error });
|
||||
// })
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
import { anthropicMaxPossibleTokens, developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
||||
import { InternalToolInfo } from '../../common/toolsService.js';
|
||||
import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js';
|
||||
import { isAToolName } from './postprocessToolCalls.js';
|
||||
|
||||
|
||||
|
||||
|
||||
export const toAnthropicTool = (toolInfo: InternalToolInfo) => {
|
||||
const { name, description, params, required } = toolInfo
|
||||
return {
|
||||
name: name,
|
||||
description: description,
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: params,
|
||||
required: required,
|
||||
}
|
||||
} satisfies Anthropic.Messages.Tool
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, aiInstructions, tools: tools_ }) => {
|
||||
|
||||
const thisConfig = settingsOfProvider.anthropic
|
||||
|
||||
const maxTokens = anthropicMaxPossibleTokens(modelName)
|
||||
if (maxTokens === undefined) {
|
||||
onError({ message: `Please set a value for Max Tokens.`, fullError: null })
|
||||
return
|
||||
}
|
||||
|
||||
const { messages, separateSystemMessageStr } = addSystemMessageAndToolSupport(modelName, providerName, messages_, aiInstructions, { separateSystemMessage: true })
|
||||
|
||||
const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName)
|
||||
const { supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels)
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
|
||||
const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined
|
||||
|
||||
const stream = anthropic.messages.stream({
|
||||
system: separateSystemMessageStr,
|
||||
messages: messages,
|
||||
model: modelName,
|
||||
max_tokens: maxTokens,
|
||||
tools: tools,
|
||||
tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time
|
||||
})
|
||||
|
||||
|
||||
// when receive text
|
||||
stream.on('text', (newText, fullText) => {
|
||||
onText({ newText, fullText })
|
||||
})
|
||||
|
||||
|
||||
// // can do tool use streaming
|
||||
// const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {}
|
||||
// stream.on('streamEvent', e => {
|
||||
// if (e.type === 'content_block_start') {
|
||||
// if (e.content_block.type !== 'tool_use') return
|
||||
// const index = e.index
|
||||
// if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' }
|
||||
// toolCallOfIndex[index].name += e.content_block.name ?? ''
|
||||
// toolCallOfIndex[index].args += e.content_block.input ?? ''
|
||||
// }
|
||||
// else if (e.type === 'content_block_delta') {
|
||||
// if (e.delta.type !== 'input_json_delta') return
|
||||
// toolCallOfIndex[e.index].args += e.delta.partial_json
|
||||
// }
|
||||
// // TODO!!!!!
|
||||
// // onText({})
|
||||
// })
|
||||
|
||||
// when we get the final message on this stream (or when error/fail)
|
||||
stream.on('finalMessage', (response) => {
|
||||
// stringify the response's content
|
||||
const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n')
|
||||
const toolCalls = response.content
|
||||
.map(c => {
|
||||
if (c.type !== 'tool_use') return null
|
||||
if (!isAToolName(c.name)) return null
|
||||
return c.type === 'tool_use' ? { name: c.name, params: JSON.stringify(c.input), id: c.id } : null
|
||||
})
|
||||
.filter(t => !!t)
|
||||
|
||||
onFinalMessage({ fullText: content, toolCalls })
|
||||
})
|
||||
|
||||
stream.on('error', (error) => {
|
||||
// the most common error will be invalid API key (401), so we handle this with a nice message
|
||||
if (error instanceof Anthropic.APIError && error.status === 401) {
|
||||
onError({ message: 'Invalid API key.', fullError: error })
|
||||
}
|
||||
else {
|
||||
onError({ message: error + '', fullError: error }) // anthropic errors can be stringified nicely like this
|
||||
}
|
||||
})
|
||||
|
||||
// TODO need to test this to make sure it works, it might throw an error
|
||||
_setAborter(() => stream.controller.abort())
|
||||
|
||||
};
|
||||
@ -1,124 +0,0 @@
|
||||
|
||||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Ollama } from 'ollama';
|
||||
import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType, OllamaModelResponse } from '../../common/llmMessageTypes.js';
|
||||
import { defaultProviderSettings } from '../../common/voidSettingsTypes.js';
|
||||
|
||||
export const ollamaList: _InternalModelListFnType<OllamaModelResponse> = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }) => {
|
||||
|
||||
const onSuccess = ({ models }: { models: OllamaModelResponse[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
|
||||
const onError = ({ error }: { error: string }) => {
|
||||
onError_({ error })
|
||||
}
|
||||
|
||||
try {
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} in Void if you want the default url).`)
|
||||
|
||||
const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
ollama.list()
|
||||
.then((response) => {
|
||||
const { models } = response
|
||||
onSuccess({ models })
|
||||
})
|
||||
.catch((error) => {
|
||||
onError({ error: error + '' })
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
onError({ error: error + '' })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
// const thisConfig = settingsOfProvider.ollama
|
||||
// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
|
||||
// let fullText = ''
|
||||
|
||||
// const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
|
||||
// ollama.generate({
|
||||
// model: modelName,
|
||||
// prompt: messages.prefix,
|
||||
// suffix: messages.suffix,
|
||||
// options: {
|
||||
// stop: messages.stopTokens,
|
||||
// num_predict: 300, // max tokens
|
||||
// // repeat_penalty: 1,
|
||||
// },
|
||||
// raw: true,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async stream => {
|
||||
// _setAborter(() => stream.abort())
|
||||
// // iterate through the stream
|
||||
// for await (const chunk of stream) {
|
||||
// const newText = chunk.response;
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// // when error/fail
|
||||
// .catch((error) => {
|
||||
// onError({ message: error + '', fullError: error })
|
||||
// })
|
||||
// };
|
||||
|
||||
|
||||
// // Ollama
|
||||
// export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
// const thisConfig = settingsOfProvider.ollama
|
||||
// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
|
||||
// let fullText = ''
|
||||
|
||||
// const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
|
||||
// ollama.chat({
|
||||
// model: modelName,
|
||||
// messages: messages,
|
||||
// stream: true,
|
||||
// // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens
|
||||
// })
|
||||
// .then(async stream => {
|
||||
// _setAborter(() => stream.abort())
|
||||
// // iterate through the stream
|
||||
// for await (const chunk of stream) {
|
||||
// const newText = chunk.message.content;
|
||||
|
||||
// // chunk.message.tool_calls[0].function.arguments
|
||||
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
|
||||
// })
|
||||
// // when error/fail
|
||||
// .catch((error) => {
|
||||
// onError({ message: error + '', fullError: error })
|
||||
// })
|
||||
|
||||
// };
|
||||
|
||||
|
||||
|
||||
// // ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',]
|
||||
@ -1,231 +0,0 @@
|
||||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
import { Model } from 'openai/resources/models.js';
|
||||
import { InternalToolInfo } from '../../common/toolsService.js';
|
||||
import { addSystemMessageAndToolSupport } from './preprocessLLMMessages.js';
|
||||
import { developerInfoOfModelName, developerInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
||||
import { isAToolName } from './postprocessToolCalls.js';
|
||||
|
||||
|
||||
// developer command - https://cdn.openai.com/spec/model-spec-2024-05-08.html#follow-the-chain-of-command
|
||||
// prompting - https://platform.openai.com/docs/guides/reasoning#advice-on-prompting
|
||||
|
||||
// npm i @openrouter/ai-sdk-provider ai ollama-ai-provider
|
||||
|
||||
export const toOpenAITool = (toolInfo: InternalToolInfo) => {
|
||||
const { name, description, params, required } = toolInfo
|
||||
return {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: name,
|
||||
description: description,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: params,
|
||||
required: required,
|
||||
}
|
||||
}
|
||||
} satisfies OpenAI.Chat.Completions.ChatCompletionTool
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
type NewParams = Pick<Parameters<_InternalSendLLMChatMessageFnType>[0] & Parameters<_InternalSendLLMFIMMessageFnType>[0], 'settingsOfProvider' | 'providerName'>
|
||||
const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => {
|
||||
|
||||
if (providerName === 'openAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true
|
||||
})
|
||||
}
|
||||
else if (providerName === 'ollama') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'vLLM') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'openRouter') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
defaultHeaders: {
|
||||
'HTTP-Referer': 'https://voideditor.com', // Optional, for including your app on openrouter.ai rankings.
|
||||
'X-Title': 'Void Editor', // Optional. Shows in rankings on openrouter.ai.
|
||||
},
|
||||
})
|
||||
}
|
||||
else if (providerName === 'gemini') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'deepseek') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'openAICompatible') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'groq') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'xAI') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else {
|
||||
console.error(`sendOpenAICompatibleMsg: invalid providerName: ${providerName}`)
|
||||
throw new Error(`Void providerName was invalid: ${providerName}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// might not currently be used in the code
|
||||
export const openaiCompatibleList: _InternalModelListFnType<Model> = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider }) => {
|
||||
const onSuccess = ({ models }: { models: Model[] }) => {
|
||||
onSuccess_({ models })
|
||||
}
|
||||
|
||||
const onError = ({ error }: { error: string }) => {
|
||||
onError_({ error })
|
||||
}
|
||||
|
||||
try {
|
||||
const openai = newOpenAI({ providerName: 'openAICompatible', settingsOfProvider })
|
||||
|
||||
openai.models.list()
|
||||
.then(async (response) => {
|
||||
const models: Model[] = []
|
||||
models.push(...response.data)
|
||||
while (response.hasNextPage()) {
|
||||
models.push(...(await response.getNextPage()).data)
|
||||
}
|
||||
onSuccess({ models })
|
||||
})
|
||||
.catch((error) => {
|
||||
onError({ error: error + '' })
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
onError({ error: error + '' })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
export const sendOpenAIFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName }) => {
|
||||
|
||||
|
||||
// openai.completions has a FIM parameter called `suffix`, but it's deprecated and only works for ~GPT 3 era models
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
// OpenAI, OpenRouter, OpenAICompatible
|
||||
export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools: tools_ }) => {
|
||||
|
||||
let fullText = ''
|
||||
const toolCallOfIndex: { [index: string]: { name: string, params: string, id: string } } = {}
|
||||
|
||||
const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName)
|
||||
const { supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels)
|
||||
|
||||
const { messages } = addSystemMessageAndToolSupport(modelName, providerName, messages_, aiInstructions, { separateSystemMessage: false })
|
||||
|
||||
const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAITool(tool)) : undefined
|
||||
|
||||
const openai: OpenAI = newOpenAI({ providerName, settingsOfProvider })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
|
||||
model: modelName,
|
||||
messages: messages,
|
||||
stream: true,
|
||||
tools: tools,
|
||||
tool_choice: tools ? 'auto' : undefined,
|
||||
parallel_tool_calls: tools ? false : undefined,
|
||||
}
|
||||
|
||||
openai.chat.completions
|
||||
.create(options)
|
||||
.then(async response => {
|
||||
_setAborter(() => response.controller.abort())
|
||||
|
||||
// when receive text
|
||||
for await (const chunk of response) {
|
||||
|
||||
// tool call
|
||||
for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) {
|
||||
const index = tool.index
|
||||
if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', params: '', id: '' }
|
||||
toolCallOfIndex[index].name += tool.function?.name ?? ''
|
||||
toolCallOfIndex[index].params += tool.function?.arguments ?? '';
|
||||
toolCallOfIndex[index].id = tool.id ?? ''
|
||||
|
||||
}
|
||||
|
||||
// message
|
||||
let newText = ''
|
||||
newText += chunk.choices[0]?.delta?.content ?? ''
|
||||
console.log('!!!!', JSON.stringify(chunk, null, 2))
|
||||
fullText += newText;
|
||||
|
||||
onText({ newText, fullText });
|
||||
}
|
||||
onFinalMessage({
|
||||
fullText,
|
||||
toolCalls: Object.keys(toolCallOfIndex)
|
||||
.map(index => {
|
||||
const tool = toolCallOfIndex[index]
|
||||
if (isAToolName(tool.name))
|
||||
return { name: tool.name, id: tool.id, params: tool.params }
|
||||
return null
|
||||
})
|
||||
.filter(t => !!t)
|
||||
});
|
||||
})
|
||||
// when error/fail - this catches errors of both .create() and .then(for await)
|
||||
.catch(error => {
|
||||
if (error instanceof OpenAI.APIError && error.status === 401) {
|
||||
onError({ message: 'Invalid API key.', fullError: error });
|
||||
}
|
||||
else {
|
||||
onError({ message: error + '', fullError: error });
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
import { ToolName, toolNames } from '../../common/toolsService.js';
|
||||
|
||||
const toolNamesSet = new Set<string>(toolNames)
|
||||
|
||||
export const isAToolName = (toolName: string): toolName is ToolName => {
|
||||
const isAToolName = toolNamesSet.has(toolName)
|
||||
return isAToolName
|
||||
}
|
||||
@ -1,7 +1,6 @@
|
||||
|
||||
|
||||
import { LLMChatMessage } from '../../common/llmMessageTypes.js';
|
||||
import { developerInfoOfModelName, developerInfoOfProviderName, ProviderName } from '../../common/voidSettingsTypes.js';
|
||||
import { deepClone } from '../../../../../base/common/objects.js';
|
||||
|
||||
|
||||
@ -14,16 +13,24 @@ export const parseObject = (args: unknown) => {
|
||||
return {}
|
||||
}
|
||||
|
||||
// no matter whether the model supports a system message or not (or what format it supports), add it in some way
|
||||
// also take into account tools if the model doesn't support tool use
|
||||
export const addSystemMessageAndToolSupport = (modelName: string, providerName: ProviderName, messages_: LLMChatMessage[], aiInstructions: string, { separateSystemMessage }: { separateSystemMessage: boolean }): { separateSystemMessageStr?: string, messages: any[] } => {
|
||||
|
||||
const prepareMessages_cloneAndTrim = ({ messages: messages_ }: { messages: LLMChatMessage[] }) => {
|
||||
const messages = deepClone(messages_).map(m => ({ ...m, content: m.content.trim(), }))
|
||||
return { messages }
|
||||
}
|
||||
|
||||
const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName)
|
||||
const { supportsSystemMessage, supportsTools } = developerInfoOfModelName(modelName, overrideSettingsForAllModels)
|
||||
// no matter whether the model supports a system message or not (or what format it supports), add it in some way
|
||||
const prepareMessages_systemMessage = ({
|
||||
messages,
|
||||
aiInstructions,
|
||||
supportsSystemMessage,
|
||||
}: {
|
||||
messages: LLMChatMessage[],
|
||||
aiInstructions: string,
|
||||
supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated',
|
||||
})
|
||||
: { separateSystemMessageStr?: string, messages: any[] } => {
|
||||
|
||||
// 1. SYSTEM MESSAGE
|
||||
// find system messages and concatenate them
|
||||
let systemMessageStr = messages
|
||||
.filter(msg => msg.role === 'system')
|
||||
@ -33,7 +40,6 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName:
|
||||
if (aiInstructions)
|
||||
systemMessageStr = `${(systemMessageStr ? `${systemMessageStr}\n\n` : '')}GUIDELINES\n${aiInstructions}`
|
||||
|
||||
|
||||
let separateSystemMessageStr: string | undefined = undefined
|
||||
|
||||
// remove all system messages
|
||||
@ -49,11 +55,12 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName:
|
||||
if (systemMessageStr) {
|
||||
// if supports system message
|
||||
if (supportsSystemMessage) {
|
||||
if (separateSystemMessage)
|
||||
if (supportsSystemMessage === 'separated')
|
||||
separateSystemMessageStr = systemMessageStr
|
||||
else {
|
||||
newMessages.unshift({ role: supportsSystemMessage === 'developer' ? 'developer' : 'system', content: systemMessageStr }) // add new first message
|
||||
}
|
||||
else if (supportsSystemMessage === 'system-role')
|
||||
newMessages.unshift({ role: 'system', content: systemMessageStr }) // add new first message
|
||||
else if (supportsSystemMessage === 'developer-role')
|
||||
newMessages.unshift({ role: 'developer', content: systemMessageStr }) // add new first message
|
||||
}
|
||||
// if does not support system message
|
||||
else {
|
||||
@ -79,225 +86,239 @@ export const addSystemMessageAndToolSupport = (modelName: string, providerName:
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// 2. MAKE TOOLS FORMAT CORRECT in messages
|
||||
let finalMessages: any[]
|
||||
if (!supportsTools) {
|
||||
// do nothing
|
||||
finalMessages = newMessages
|
||||
}
|
||||
|
||||
// anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples
|
||||
// "content": [
|
||||
// {
|
||||
// "type": "text",
|
||||
// "text": "<thinking>I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.</thinking>"
|
||||
// },
|
||||
// {
|
||||
// "type": "tool_use",
|
||||
// "id": "toolu_01A09q90qw90lq917835lq9",
|
||||
// "name": "get_weather",
|
||||
// "input": { "location": "San Francisco, CA", "unit": "celsius" }
|
||||
// }
|
||||
// ]
|
||||
|
||||
// anthropic user message response will be:
|
||||
// "content": [
|
||||
// {
|
||||
// "type": "tool_result",
|
||||
// "tool_use_id": "toolu_01A09q90qw90lq917835lq9",
|
||||
// "content": "15 degrees"
|
||||
// }
|
||||
// ]
|
||||
|
||||
|
||||
else if (providerName === 'anthropic') { // convert role:'tool' to anthropic's type
|
||||
const newMessagesTools: (
|
||||
Exclude<typeof newMessages[0], { role: 'assistant' | 'user' }> | {
|
||||
role: 'assistant',
|
||||
content: string | ({
|
||||
type: 'text';
|
||||
text: string;
|
||||
} | {
|
||||
type: 'tool_use';
|
||||
name: string;
|
||||
input: Record<string, any>;
|
||||
id: string;
|
||||
})[]
|
||||
} | {
|
||||
role: 'user',
|
||||
content: string | ({
|
||||
type: 'text';
|
||||
text: string;
|
||||
} | {
|
||||
type: 'tool_result';
|
||||
tool_use_id: string;
|
||||
content: string;
|
||||
})[]
|
||||
}
|
||||
)[] = newMessages;
|
||||
|
||||
|
||||
for (let i = 0; i < newMessagesTools.length; i += 1) {
|
||||
const currMsg = newMessagesTools[i]
|
||||
|
||||
if (currMsg.role !== 'tool') continue
|
||||
|
||||
const prevMsg = 0 <= i - 1 && i - 1 <= newMessagesTools.length ? newMessagesTools[i - 1] : undefined
|
||||
|
||||
if (prevMsg?.role === 'assistant') {
|
||||
if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: prevMsg.content }]
|
||||
prevMsg.content.push({ type: 'tool_use', id: currMsg.id, name: currMsg.name, input: parseObject(currMsg.params) })
|
||||
}
|
||||
|
||||
// turn each tool into a user message with tool results at the end
|
||||
newMessagesTools[i] = {
|
||||
role: 'user',
|
||||
content: [
|
||||
...[{ type: 'tool_result', tool_use_id: currMsg.id, content: currMsg.content }] as const,
|
||||
...currMsg.content ? [{ type: 'text', text: currMsg.content }] as const : [],
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
finalMessages = newMessagesTools
|
||||
}
|
||||
|
||||
// openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps
|
||||
// "tool_calls":[
|
||||
// {
|
||||
// "type": "function",
|
||||
// "id": "call_12345xyz",
|
||||
// "function": {
|
||||
// "name": "get_weather",
|
||||
// "arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}"
|
||||
// }
|
||||
// }]
|
||||
|
||||
// openai user response will be:
|
||||
// {
|
||||
// "role": "tool",
|
||||
// "tool_call_id": tool_call.id,
|
||||
// "content": str(result)
|
||||
// }
|
||||
|
||||
// treat all other providers like openai tool message for now
|
||||
else {
|
||||
|
||||
const newMessagesTools: (
|
||||
Exclude<typeof newMessages[0], { role: 'assistant' | 'tool' }> | {
|
||||
role: 'assistant',
|
||||
content: string;
|
||||
tool_calls?: {
|
||||
type: 'function';
|
||||
id: string;
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
}
|
||||
}[]
|
||||
} | {
|
||||
role: 'tool',
|
||||
id: string; // old val
|
||||
tool_call_id: string; // new val
|
||||
content: string;
|
||||
}
|
||||
)[] = [];
|
||||
|
||||
for (let i = 0; i < newMessages.length; i += 1) {
|
||||
const currMsg = newMessages[i]
|
||||
|
||||
if (currMsg.role !== 'tool') {
|
||||
newMessagesTools.push(currMsg)
|
||||
continue
|
||||
}
|
||||
|
||||
// edit previous assistant message to have called the tool
|
||||
const prevMsg = 0 <= i - 1 && i - 1 <= newMessagesTools.length ? newMessagesTools[i - 1] : undefined
|
||||
if (prevMsg?.role === 'assistant') {
|
||||
prevMsg.tool_calls = [{
|
||||
type: 'function',
|
||||
id: currMsg.id,
|
||||
function: {
|
||||
name: currMsg.name,
|
||||
arguments: JSON.stringify(currMsg.params)
|
||||
}
|
||||
}]
|
||||
}
|
||||
|
||||
// add the tool
|
||||
newMessagesTools.push({
|
||||
role: 'tool',
|
||||
id: currMsg.id,
|
||||
content: currMsg.content,
|
||||
tool_call_id: currMsg.id,
|
||||
})
|
||||
}
|
||||
finalMessages = newMessagesTools
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// 3. CROP MESSAGES SO EVERYTHING FITS IN CONTEXT
|
||||
// TODO!!!
|
||||
|
||||
|
||||
console.log('SYSMG', separateSystemMessage)
|
||||
console.log('FINAL MESSAGES', JSON.stringify(finalMessages, null, 2))
|
||||
|
||||
|
||||
return {
|
||||
separateSystemMessageStr,
|
||||
messages: finalMessages,
|
||||
}
|
||||
return { messages: newMessages, separateSystemMessageStr }
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// convert messages as if about to send to openai
|
||||
/*
|
||||
reference - https://platform.openai.com/docs/guides/function-calling#function-calling-steps
|
||||
openai MESSAGE (role=assistant):
|
||||
"tool_calls":[{
|
||||
"type": "function",
|
||||
"id": "call_12345xyz",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}"
|
||||
}]
|
||||
|
||||
openai RESPONSE (role=user):
|
||||
{ "role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": str(result) }
|
||||
|
||||
also see
|
||||
openai on prompting - https://platform.openai.com/docs/guides/reasoning#advice-on-prompting
|
||||
openai on developer system message - https://cdn.openai.com/spec/model-spec-2024-05-08.html#follow-the-chain-of-command
|
||||
*/
|
||||
|
||||
const prepareMessages_tools_openai = ({ messages }: { messages: LLMChatMessage[], }) => {
|
||||
|
||||
const newMessages: (
|
||||
Exclude<LLMChatMessage, { role: 'assistant' | 'tool' }> | {
|
||||
role: 'assistant',
|
||||
content: string;
|
||||
tool_calls?: {
|
||||
type: 'function';
|
||||
id: string;
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
}
|
||||
}[]
|
||||
} | {
|
||||
role: 'tool',
|
||||
id: string; // old val
|
||||
tool_call_id: string; // new val
|
||||
content: string;
|
||||
}
|
||||
)[] = [];
|
||||
|
||||
for (let i = 0; i < messages.length; i += 1) {
|
||||
const currMsg = messages[i]
|
||||
|
||||
if (currMsg.role !== 'tool') {
|
||||
newMessages.push(currMsg)
|
||||
continue
|
||||
}
|
||||
|
||||
// edit previous assistant message to have called the tool
|
||||
const prevMsg = 0 <= i - 1 && i - 1 <= newMessages.length ? newMessages[i - 1] : undefined
|
||||
if (prevMsg?.role === 'assistant') {
|
||||
prevMsg.tool_calls = [{
|
||||
type: 'function',
|
||||
id: currMsg.id,
|
||||
function: {
|
||||
name: currMsg.name,
|
||||
arguments: JSON.stringify(currMsg.params)
|
||||
}
|
||||
}]
|
||||
}
|
||||
|
||||
// add the tool
|
||||
newMessages.push({
|
||||
role: 'tool',
|
||||
id: currMsg.id,
|
||||
content: currMsg.content,
|
||||
tool_call_id: currMsg.id,
|
||||
})
|
||||
}
|
||||
return { messages: newMessages }
|
||||
|
||||
}
|
||||
|
||||
|
||||
// convert messages as if about to send to anthropic
|
||||
/*
|
||||
https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples
|
||||
anthropic MESSAGE (role=assistant):
|
||||
"content": [{
|
||||
"type": "text",
|
||||
"text": "<thinking>I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.</thinking>"
|
||||
}, {
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"name": "get_weather",
|
||||
"input": { "location": "San Francisco, CA", "unit": "celsius" }
|
||||
}]
|
||||
anthropic RESPONSE (role=user):
|
||||
"content": [{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"content": "15 degrees"
|
||||
}]
|
||||
*/
|
||||
|
||||
const prepareMessages_tools_anthropic = ({ messages }: { messages: LLMChatMessage[], }) => {
|
||||
const newMessages: (
|
||||
Exclude<LLMChatMessage, { role: 'assistant' | 'user' }> | {
|
||||
role: 'assistant',
|
||||
content: string | ({
|
||||
type: 'text';
|
||||
text: string;
|
||||
} | {
|
||||
type: 'tool_use';
|
||||
name: string;
|
||||
input: Record<string, any>;
|
||||
id: string;
|
||||
})[]
|
||||
} | {
|
||||
role: 'user',
|
||||
content: string | ({
|
||||
type: 'text';
|
||||
text: string;
|
||||
} | {
|
||||
type: 'tool_result';
|
||||
tool_use_id: string;
|
||||
content: string;
|
||||
})[]
|
||||
}
|
||||
)[] = messages;
|
||||
|
||||
|
||||
for (let i = 0; i < newMessages.length; i += 1) {
|
||||
const currMsg = newMessages[i]
|
||||
|
||||
if (currMsg.role !== 'tool') continue
|
||||
|
||||
const prevMsg = 0 <= i - 1 && i - 1 <= newMessages.length ? newMessages[i - 1] : undefined
|
||||
|
||||
if (prevMsg?.role === 'assistant') {
|
||||
if (typeof prevMsg.content === 'string') prevMsg.content = [{ type: 'text', text: prevMsg.content }]
|
||||
prevMsg.content.push({ type: 'tool_use', id: currMsg.id, name: currMsg.name, input: parseObject(currMsg.params) })
|
||||
}
|
||||
|
||||
// turn each tool into a user message with tool results at the end
|
||||
newMessages[i] = {
|
||||
role: 'user',
|
||||
content: [
|
||||
...[{ type: 'tool_result', tool_use_id: currMsg.id, content: currMsg.content }] as const,
|
||||
...currMsg.content ? [{ type: 'text', text: currMsg.content }] as const : [],
|
||||
]
|
||||
}
|
||||
}
|
||||
return { messages: newMessages }
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const prepareMessages_tools = ({ messages, supportsTools }: { messages: LLMChatMessage[], supportsTools: false | 'anthropic-style' | 'openai-style' }) => {
|
||||
if (!supportsTools) {
|
||||
return { messages: messages }
|
||||
}
|
||||
else if (supportsTools === 'anthropic-style') {
|
||||
return prepareMessages_tools_anthropic({ messages })
|
||||
}
|
||||
else if (supportsTools === 'openai-style') {
|
||||
return prepareMessages_tools_openai({ messages })
|
||||
}
|
||||
else {
|
||||
throw 1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
Gemini has this, but they're openai-compat so we don't need to implement this
|
||||
gemini request:
|
||||
{ "role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message)
|
||||
gemini request: {
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
gemini response:
|
||||
{
|
||||
"role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
{ "role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+ anthropic
|
||||
|
||||
+ openai-compat (4)
|
||||
+ gemini
|
||||
|
||||
ollama
|
||||
|
||||
|
||||
mistral: same as openai
|
||||
|
||||
*/
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
export const prepareMessages = ({
|
||||
messages,
|
||||
aiInstructions,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
}: {
|
||||
messages: LLMChatMessage[],
|
||||
aiInstructions: string,
|
||||
supportsSystemMessage: false | 'system-role' | 'developer-role' | 'separated',
|
||||
supportsTools: false | 'anthropic-style' | 'openai-style',
|
||||
}) => {
|
||||
const { messages: messages1 } = prepareMessages_cloneAndTrim({ messages })
|
||||
const { messages: messages2, separateSystemMessageStr } = prepareMessages_systemMessage({ messages: messages1, aiInstructions, supportsSystemMessage })
|
||||
const { messages: messages3 } = prepareMessages_tools({ messages: messages2, supportsTools })
|
||||
return {
|
||||
messages: messages3 as any,
|
||||
separateSystemMessageStr
|
||||
} as const
|
||||
}
|
||||
|
||||
|
||||
@ -6,9 +6,7 @@
|
||||
import { SendLLMMessageParams, OnText, OnFinalMessage, OnError } from '../../common/llmMessageTypes.js';
|
||||
import { IMetricsService } from '../../common/metricsService.js';
|
||||
import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
||||
|
||||
import { sendAnthropicChat } from './anthropic.js';
|
||||
import { sendOpenAIChat } from './openai.js';
|
||||
import { sendLLMMessageToProviderImplementation } from './MODELS.js';
|
||||
|
||||
|
||||
export const sendLLMMessage = ({
|
||||
@ -58,9 +56,10 @@ export const sendLLMMessage = ({
|
||||
let _setAborter = (fn: () => void) => { _aborter = fn }
|
||||
let _didAbort = false
|
||||
|
||||
const onText: OnText = ({ newText, fullText }) => {
|
||||
const onText: OnText = (params) => {
|
||||
const { fullText } = params
|
||||
if (_didAbort) return
|
||||
onText_({ newText, fullText })
|
||||
onText_(params)
|
||||
_fullTextSoFar = fullText
|
||||
}
|
||||
|
||||
@ -95,29 +94,27 @@ export const sendLLMMessage = ({
|
||||
else if (messagesType === 'FIMMessage')
|
||||
captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics
|
||||
|
||||
|
||||
try {
|
||||
switch (providerName) {
|
||||
case 'openAI':
|
||||
case 'openRouter':
|
||||
case 'deepseek':
|
||||
case 'openAICompatible':
|
||||
case 'mistral':
|
||||
case 'ollama':
|
||||
case 'vLLM':
|
||||
case 'groq':
|
||||
case 'gemini':
|
||||
case 'xAI':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI API FIM', toolCalls: [] })
|
||||
else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools });
|
||||
break;
|
||||
case 'anthropic':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', toolCalls: [] })
|
||||
else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools });
|
||||
break;
|
||||
default:
|
||||
onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null })
|
||||
break;
|
||||
const implementation = sendLLMMessageToProviderImplementation[providerName]
|
||||
if (!implementation) {
|
||||
onError({ message: `Error: Provider "${providerName}" not recognized.`, fullError: null })
|
||||
return
|
||||
}
|
||||
const { sendFIM, sendChat } = implementation
|
||||
if (messagesType === 'chatMessages') {
|
||||
sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions, tools })
|
||||
return
|
||||
}
|
||||
if (messagesType === 'FIMMessage') {
|
||||
if (sendFIM) {
|
||||
sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, aiInstructions })
|
||||
return
|
||||
}
|
||||
onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null })
|
||||
return
|
||||
}
|
||||
onError({ message: `Error: Message type "${messagesType}" not recognized.`, fullError: null })
|
||||
}
|
||||
|
||||
catch (error) {
|
||||
|
||||
@ -8,30 +8,42 @@
|
||||
|
||||
import { IServerChannel } from '../../../../base/parts/ipc/common/ipc.js';
|
||||
import { Emitter, Event } from '../../../../base/common/event.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, MainModelListParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, OpenaiCompatibleModelResponse, } from '../common/llmMessageTypes.js';
|
||||
import { EventLLMMessageOnTextParams, EventLLMMessageOnErrorParams, EventLLMMessageOnFinalMessageParams, MainSendLLMMessageParams, AbortRef, SendLLMMessageParams, MainLLMMessageAbortParams, ModelListParams, EventModelListOnSuccessParams, EventModelListOnErrorParams, OllamaModelResponse, VLLMModelResponse, MainModelListParams, } from '../common/llmMessageTypes.js';
|
||||
import { sendLLMMessage } from './llmMessage/sendLLMMessage.js'
|
||||
import { IMetricsService } from '../common/metricsService.js';
|
||||
import { ollamaList } from './llmMessage/ollama.js';
|
||||
import { openaiCompatibleList } from './llmMessage/openai.js';
|
||||
import { sendLLMMessageToProviderImplementation } from './llmMessage/MODELS.js';
|
||||
|
||||
// NODE IMPLEMENTATION - calls actual sendLLMMessage() and returns listeners to it
|
||||
|
||||
export class LLMMessageChannel implements IServerChannel {
|
||||
|
||||
// sendLLMMessage
|
||||
private readonly _onText_llm = new Emitter<EventLLMMessageOnTextParams>();
|
||||
private readonly _onFinalMessage_llm = new Emitter<EventLLMMessageOnFinalMessageParams>();
|
||||
private readonly _onError_llm = new Emitter<EventLLMMessageOnErrorParams>();
|
||||
private readonly llmMessageEmitters = {
|
||||
onText: new Emitter<EventLLMMessageOnTextParams>(),
|
||||
onFinalMessage: new Emitter<EventLLMMessageOnFinalMessageParams>(),
|
||||
onError: new Emitter<EventLLMMessageOnErrorParams>(),
|
||||
}
|
||||
|
||||
// abort
|
||||
private readonly _abortRefOfRequestId_llm: Record<string, AbortRef> = {}
|
||||
// aborters for above
|
||||
private readonly abortRefOfRequestId: Record<string, AbortRef> = {}
|
||||
|
||||
// ollamaList
|
||||
private readonly _onSuccess_ollama = new Emitter<EventModelListOnSuccessParams<OllamaModelResponse>>();
|
||||
private readonly _onError_ollama = new Emitter<EventModelListOnErrorParams<OllamaModelResponse>>();
|
||||
|
||||
// openaiCompatibleList
|
||||
private readonly _onSuccess_openAICompatible = new Emitter<EventModelListOnSuccessParams<OpenaiCompatibleModelResponse>>();
|
||||
private readonly _onError_openAICompatible = new Emitter<EventModelListOnErrorParams<OpenaiCompatibleModelResponse>>();
|
||||
// list
|
||||
private readonly listEmitters = {
|
||||
ollama: {
|
||||
success: new Emitter<EventModelListOnSuccessParams<OllamaModelResponse>>(),
|
||||
error: new Emitter<EventModelListOnErrorParams<OllamaModelResponse>>(),
|
||||
},
|
||||
vLLM: {
|
||||
success: new Emitter<EventModelListOnSuccessParams<VLLMModelResponse>>(),
|
||||
error: new Emitter<EventModelListOnErrorParams<VLLMModelResponse>>(),
|
||||
}
|
||||
} satisfies {
|
||||
[providerName: string]: {
|
||||
success: Emitter<EventModelListOnSuccessParams<any>>,
|
||||
error: Emitter<EventModelListOnErrorParams<any>>,
|
||||
}
|
||||
}
|
||||
|
||||
// stupidly, channels can't take in @IService
|
||||
constructor(
|
||||
@ -40,30 +52,17 @@ export class LLMMessageChannel implements IServerChannel {
|
||||
|
||||
// browser uses this to listen for changes
|
||||
listen(_: unknown, event: string): Event<any> {
|
||||
if (event === 'onText_llm') {
|
||||
return this._onText_llm.event;
|
||||
}
|
||||
else if (event === 'onFinalMessage_llm') {
|
||||
return this._onFinalMessage_llm.event;
|
||||
}
|
||||
else if (event === 'onError_llm') {
|
||||
return this._onError_llm.event;
|
||||
}
|
||||
else if (event === 'onSuccess_ollama') {
|
||||
return this._onSuccess_ollama.event;
|
||||
}
|
||||
else if (event === 'onError_ollama') {
|
||||
return this._onError_ollama.event;
|
||||
}
|
||||
else if (event === 'onSuccess_openAICompatible') {
|
||||
return this._onSuccess_openAICompatible.event;
|
||||
}
|
||||
else if (event === 'onError_openAICompatible') {
|
||||
return this._onError_openAICompatible.event;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Event not found: ${event}`);
|
||||
}
|
||||
// text
|
||||
if (event === 'onText_sendLLMMessage') return this.llmMessageEmitters.onText.event;
|
||||
else if (event === 'onFinalMessage_sendLLMMessage') return this.llmMessageEmitters.onFinalMessage.event;
|
||||
else if (event === 'onError_sendLLMMessage') return this.llmMessageEmitters.onError.event;
|
||||
// list
|
||||
else if (event === 'onSuccess_list_ollama') return this.listEmitters.ollama.success.event;
|
||||
else if (event === 'onError_list_ollama') return this.listEmitters.ollama.error.event;
|
||||
else if (event === 'onSuccess_list_vLLM') return this.listEmitters.vLLM.success.event;
|
||||
else if (event === 'onError_list_vLLM') return this.listEmitters.vLLM.error.event;
|
||||
|
||||
else throw new Error(`Event not found: ${event}`);
|
||||
}
|
||||
|
||||
// browser uses this to call (see this.channel.call() in llmMessageService.ts for all usages)
|
||||
@ -78,8 +77,8 @@ export class LLMMessageChannel implements IServerChannel {
|
||||
else if (command === 'ollamaList') {
|
||||
this._callOllamaList(params)
|
||||
}
|
||||
else if (command === 'openAICompatibleList') {
|
||||
this._callOpenAICompatibleList(params)
|
||||
else if (command === 'vLLMList') {
|
||||
this._callVLLMList(params)
|
||||
}
|
||||
else {
|
||||
throw new Error(`Void sendLLM: command "${command}" not recognized.`)
|
||||
@ -94,47 +93,50 @@ export class LLMMessageChannel implements IServerChannel {
|
||||
private async _callSendLLMMessage(params: MainSendLLMMessageParams) {
|
||||
const { requestId } = params;
|
||||
|
||||
if (!(requestId in this._abortRefOfRequestId_llm))
|
||||
this._abortRefOfRequestId_llm[requestId] = { current: null }
|
||||
if (!(requestId in this.abortRefOfRequestId))
|
||||
this.abortRefOfRequestId[requestId] = { current: null }
|
||||
|
||||
const mainThreadParams: SendLLMMessageParams = {
|
||||
...params,
|
||||
onText: ({ newText, fullText }) => { this._onText_llm.fire({ requestId, newText, fullText }); },
|
||||
onFinalMessage: ({ fullText, toolCalls }) => { this._onFinalMessage_llm.fire({ requestId, fullText, toolCalls }); },
|
||||
onError: ({ message: error, fullError }) => { console.log('sendLLM: firing err'); this._onError_llm.fire({ requestId, message: error, fullError }); },
|
||||
abortRef: this._abortRefOfRequestId_llm[requestId],
|
||||
onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); },
|
||||
onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); },
|
||||
onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); },
|
||||
abortRef: this.abortRefOfRequestId[requestId],
|
||||
}
|
||||
sendLLMMessage(mainThreadParams, this.metricsService);
|
||||
}
|
||||
|
||||
private _callAbort(params: MainLLMMessageAbortParams) {
|
||||
const { requestId } = params;
|
||||
if (!(requestId in this._abortRefOfRequestId_llm)) return
|
||||
this._abortRefOfRequestId_llm[requestId].current?.()
|
||||
delete this._abortRefOfRequestId_llm[requestId]
|
||||
}
|
||||
|
||||
private _callOllamaList(params: MainModelListParams<OllamaModelResponse>) {
|
||||
const { requestId } = params;
|
||||
|
||||
_callOllamaList = (params: MainModelListParams<OllamaModelResponse>) => {
|
||||
const { requestId } = params
|
||||
const emitters = this.listEmitters.ollama
|
||||
const mainThreadParams: ModelListParams<OllamaModelResponse> = {
|
||||
...params,
|
||||
onSuccess: ({ models }) => { this._onSuccess_ollama.fire({ requestId, models }); },
|
||||
onError: ({ error }) => { this._onError_ollama.fire({ requestId, error }); },
|
||||
onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); },
|
||||
onError: (p) => { emitters.error.fire({ requestId, ...p }); },
|
||||
}
|
||||
ollamaList(mainThreadParams)
|
||||
sendLLMMessageToProviderImplementation.ollama.list(mainThreadParams)
|
||||
}
|
||||
|
||||
private _callOpenAICompatibleList(params: MainModelListParams<OpenaiCompatibleModelResponse>) {
|
||||
const { requestId } = params;
|
||||
|
||||
const mainThreadParams: ModelListParams<OpenaiCompatibleModelResponse> = {
|
||||
_callVLLMList = (params: MainModelListParams<VLLMModelResponse>) => {
|
||||
const { requestId } = params
|
||||
const emitters = this.listEmitters.vLLM
|
||||
const mainThreadParams: ModelListParams<VLLMModelResponse> = {
|
||||
...params,
|
||||
onSuccess: ({ models }) => { this._onSuccess_openAICompatible.fire({ requestId, models }); },
|
||||
onError: ({ error }) => { this._onError_openAICompatible.fire({ requestId, error }); },
|
||||
onSuccess: (p) => { emitters.success.fire({ requestId, ...p }); },
|
||||
onError: (p) => { emitters.error.fire({ requestId, ...p }); },
|
||||
}
|
||||
openaiCompatibleList(mainThreadParams)
|
||||
sendLLMMessageToProviderImplementation.vLLM.list(mainThreadParams)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private _callAbort(params: MainLLMMessageAbortParams) {
|
||||
const { requestId } = params;
|
||||
if (!(requestId in this.abortRefOfRequestId)) return
|
||||
this.abortRefOfRequestId[requestId].current?.()
|
||||
delete this.abortRefOfRequestId[requestId]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user