From 054dd382f46eee48a3ecd453fdb2fee9e400f8a5 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Wed, 19 Jun 2024 18:44:06 -0700 Subject: [PATCH] Use all available tools through function calling --- chat-sample/package.json | 26 +- chat-sample/src/extension.ts | 275 ++---------------- ...ode.proposed.chatParticipantAdditions.d.ts | 252 ++++++++++++++++ chat-sample/vscode.proposed.chatTools.d.ts | 35 +++ 4 files changed, 319 insertions(+), 269 deletions(-) create mode 100644 chat-sample/vscode.proposed.chatParticipantAdditions.d.ts create mode 100644 chat-sample/vscode.proposed.chatTools.d.ts diff --git a/chat-sample/package.json b/chat-sample/package.json index 36d6fa5f..8fc8bc56 100644 --- a/chat-sample/package.json +++ b/chat-sample/package.json @@ -23,27 +23,11 @@ "contributes": { "chatParticipants": [ { - "id": "chat-sample.cat", - "fullName": "Cat", - "name": "cat", - "description": "Meow! What can I teach you?", - "isSticky": true, - "commands": [ - { - "name": "teach", - "description": "Pick at random a computer science concept then explain it in purfect way of a cat" - }, - { - "name": "play", - "description": "Do whatever you want, you are a cat after all" - } - ] - } - ], - "commands": [ - { - "command": "cat.namesInEditor", - "title": "Use Cat Names in Editor" + "id": "chat-sample.tools", + "fullName": "Tool User", + "name": "tools", + "description": "I use tools", + "isSticky": true } ] }, diff --git a/chat-sample/src/extension.ts b/chat-sample/src/extension.ts index 6c8fd32e..600dcf03 100644 --- a/chat-sample/src/extension.ts +++ b/chat-sample/src/extension.ts @@ -1,171 +1,39 @@ import * as vscode from 'vscode'; -const CAT_NAMES_COMMAND_ID = 'cat.namesInEditor'; -const CAT_PARTICIPANT_ID = 'chat-sample.cat'; - -interface ICatChatResult extends vscode.ChatResult { - metadata: { - command: string; - } -} - -const MODEL_SELECTOR: vscode.LanguageModelChatSelector = { vendor: 'copilot', family: 'gpt-3.5-turbo' }; +const PARTICIPANT_ID = 'chat-sample.tools'; export function activate(context: vscode.ExtensionContext) { - // Define a Cat chat handler. const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, chatContext: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken) => { const models = await vscode.lm.selectChatModels({ vendor: 'copilot', + family: 'gpt-4-turbo' }); - if (!models || !models.length) { + const chat = models[0]; + + if (!chat) { console.log('NO MODELS') return {}; } - // for (const model of models) { - // stream.markdown(`- ${model.name} (${model.family} - ${model.vendor})\n`) - // } - - const chat = models[Math.floor(Math.random() * models.length)]; - - - stream.progress(`Using ${chat.name} (${context.languageModelAccessInformation.canSendRequest(chat)})...`); - - - abstract class FunctionTool { - - static All = new Map Promise - }>(); - - static register(metadata: vscode.LanguageModelChatFunction, run: (...args: any[]) => Promise) { - FunctionTool.All.set(metadata.name, { metadata, run: run }); - } - } - - // get the size of an editor - FunctionTool.register({ - name: "get_length_of_editor", - description: "Get the length of an editor", - parametersSchema: { - "type": "object", - "properties": { - "nth": { - "type": "number", - "description": "The index of the editor, starting at 0", - }, - }, - "required": ["nth"], - }, - }, async (arg: { nth: number }) => { - if (!(arg && typeof arg === 'object' && typeof arg.nth === 'number')) { - return 'Error: Invalid arguments, expected { nth: number}'; - } - const editor = vscode.window.visibleTextEditors[arg.nth]; - if (!editor) { - return `Warning: No editor found at index ${arg.nth}, please try a different index between 0 and ${vscode.window.visibleTextEditors.length - 1}`; - } - return editor.document.getText().length.toString(); - }) - - FunctionTool.register({ - name: "show_user_message", - description: "Show a message to the user", - parametersSchema: { - "type": "object", - "properties": { - "message": { - "type": "string", - "description": "The message to show", - }, - }, - "required": ["message"], - }, - }, async (arg: { message: string }) => { - if (!(arg && typeof arg === 'object' && typeof arg.message === 'string')) { - return 'Error: Invalid arguments, expected { message: string}'; - } - vscode.window.showInformationMessage(arg.message); - return 'done'; - }); - - FunctionTool.register({ - name: "current_temperature", - description: "Get the current temperature for a location", - parametersSchema: { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to check the tepmerature for", - }, - }, - "required": ["location"], - }, - }, async (arg: { location: string }) => { - if (!(arg && typeof arg === 'object' && typeof arg.location === 'string')) { - return 'Error: Invalid arguments, expected { location: string}'; - } - return 'The temperature in ' + arg.location + ' is 25°C'; - }); - - FunctionTool.register({ - name: "start_debugging", - description: "Start debugging the given file", - parametersSchema: { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "The file to debug", - }, - }, - "required": ["filename"], - }, - }, async (arg: { filename: string }) => { - if (!(arg && typeof arg === 'object' && typeof arg.filename === 'string')) { - return 'Error: Invalid arguments, expected { filename: string}'; - } - vscode.debug.startDebugging(undefined, { - name: 'debug', - type: 'node', - request: 'launch', - program: vscode.Uri.joinPath(vscode.workspace.workspaceFolders![0].uri, arg.filename).fsPath, - }); - return 'done'; - }); - - FunctionTool.register({ - name: "create_terminal", - description: "Create a terminal with the given name", - parametersSchema: { - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The terminal name", - }, - }, - "required": ["name"], - }, - }, async (arg: { name: string }) => { - if (!(arg && typeof arg === 'object' && typeof arg.name === 'string')) { - return 'Error: Invalid arguments, expected { name: string}'; - } - vscode.window.createTerminal(arg.name).show(); - return 'done'; - }); - + stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.id).join(', ')}\n\n`); const options: vscode.LanguageModelChatRequestOptions = { - tools: Array.from(FunctionTool.All.values()).map(tool => tool.metadata), + tools: vscode.lm.tools.map((tool): vscode.LanguageModelChatFunction => { + return { + name: tool.id.replace(/\./g, '_'), + description: tool.description, + parametersSchema: tool.parametersSchema ?? {} + } + }), justification: 'Just because!', } - const messages = [vscode.LanguageModelChatMessage.User(request.prompt)]; + const messages = [ + vscode.LanguageModelChatMessage.User(`There is a selection of tools that may give helpful context to answer the user's query. If you aren't sure which tool is relevant, you can call multiple tools.`), + vscode.LanguageModelChatMessage.User(request.prompt) + ]; const runWithFunctions = async () => { let didReceiveFunctionUse = false; @@ -173,30 +41,31 @@ export function activate(context: vscode.ExtensionContext) { const response = await chat.sendRequest(messages, options, token); for await (const part of response.stream) { - if (part instanceof vscode.LanguageModelChatResponseTextPart) { stream.markdown(part.value) - } else if (part instanceof vscode.LanguageModelChatResponseFunctionUsePart) { - const tool = FunctionTool.All.get(part.name); + const tool = vscode.lm.tools.find(tool => tool.id.replace(/\./g, '_') === part.name); if (!tool) { // BAD tool choice? continue; } - stream.progress(`FUNCTION_CALL: ${tool.metadata.name} with \`${part.parameters}\``) + const resultPromise = vscode.lm.invokeTool(tool.id, JSON.parse(part.parameters), token); + stream.progress(`FUNCTION_CALL: ${tool.id} with \`${part.parameters}\``, async (progress) => { + await resultPromise; + }); - const result = await tool.run(JSON.parse(part.parameters)); + const result = await resultPromise; // NOTE that the result of calling a function is a special content type of a USER-message let message = vscode.LanguageModelChatMessage.User(''); - message.content2 = new vscode.LanguageModelChatMessageFunctionResultPart(tool.metadata.name, result) + message.content2 = new vscode.LanguageModelChatMessageFunctionResultPart(tool.id, result) messages.push(message) // IMPORTANT // IMPORTANT working around CAPI always wanting to end with a `User`-message // IMPORTANT - messages.push(vscode.LanguageModelChatMessage.User('Above is the result of calling the function ${tool.metadata.name}')) + messages.push(vscode.LanguageModelChatMessage.User(`Above is the result of calling the function ${tool.id}. The user cannot see this result, so you should explain it to the user if referencing it in your answer.`)) didReceiveFunctionUse = true; } } @@ -210,99 +79,9 @@ export function activate(context: vscode.ExtensionContext) { await runWithFunctions() }; - // Chat participants appear as top-level options in the chat input - // when you type `@`, and can contribute sub-commands in the chat input - // that appear when you type `/`. - const cat = vscode.chat.createChatParticipant(CAT_PARTICIPANT_ID, handler); - cat.iconPath = vscode.Uri.joinPath(context.extensionUri, 'cat.jpeg'); - cat.followupProvider = { - provideFollowups(result: ICatChatResult, context: vscode.ChatContext, token: vscode.CancellationToken) { - return [{ - prompt: 'let us play', - label: vscode.l10n.t('Play with the cat'), - command: 'play' - } satisfies vscode.ChatFollowup]; - } - }; - - context.subscriptions.push( - cat, - // Register the command handler for the /meow followup - vscode.commands.registerTextEditorCommand(CAT_NAMES_COMMAND_ID, async (textEditor: vscode.TextEditor) => { - // Replace all variables in active editor with cat names and words - const text = textEditor.document.getText(); - const messages = [ - vscode.LanguageModelChatMessage.User(`You are a cat! Think carefully and step by step like a cat would. - Your job is to replace all variable names in the following code with funny cat variable names. Be creative. IMPORTANT respond just with code. Do not use markdown!`), - vscode.LanguageModelChatMessage.User(text) - ]; - - let chatResponse: vscode.LanguageModelChatResponse | undefined; - try { - const [model] = await vscode.lm.selectChatModels({ vendor: 'copilot', family: 'gpt-3.5-turbo' }); - if (!model) { - console.log('Model not found. Please make sure the GitHub Copilot Chat extension is installed and enabled.') - return; - } - - chatResponse = await model.sendRequest(messages, {}, new vscode.CancellationTokenSource().token); - - } catch (err) { - // making the chat request might fail because - // - model does not exist - // - user consent not given - // - quote limits exceeded - if (err instanceof vscode.LanguageModelError) { - console.log(err.message, err.code, err.cause) - } - return; - } - - // Clear the editor content before inserting new content - await textEditor.edit(edit => { - const start = new vscode.Position(0, 0); - const end = new vscode.Position(textEditor.document.lineCount - 1, textEditor.document.lineAt(textEditor.document.lineCount - 1).text.length); - edit.delete(new vscode.Range(start, end)); - }); - - // Stream the code into the editor as it is coming in from the Language Model - try { - for await (const fragment of chatResponse.text) { - await textEditor.edit(edit => { - const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); - const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); - edit.insert(position, fragment); - }); - } - } catch (err) { - // async response stream may fail, e.g network interruption or server side error - await textEditor.edit(edit => { - const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); - const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); - edit.insert(position, (err).message); - }); - } - }), - ); + const toolUser = vscode.chat.createChatParticipant(PARTICIPANT_ID, handler); + toolUser.iconPath = new vscode.ThemeIcon('tools'); } -// Get a random topic that the cat has not taught in the chat history yet -function getTopic(history: ReadonlyArray): string { - const topics = ['linked list', 'recursion', 'stack', 'queue', 'pointers']; - // Filter the chat history to get only the responses from the cat - const previousCatResponses = history.filter(h => { - return h instanceof vscode.ChatResponseTurn && h.participant == CAT_PARTICIPANT_ID - }) as vscode.ChatResponseTurn[]; - // Filter the topics to get only the topics that have not been taught by the cat yet - const topicsNoRepetition = topics.filter(topic => { - return !previousCatResponses.some(catResponse => { - return catResponse.response.some(r => { - return r instanceof vscode.ChatResponseMarkdownPart && r.value.value.includes(topic) - }); - }); - }); - - return topicsNoRepetition[Math.floor(Math.random() * topicsNoRepetition.length)] || 'I have taught you everything I know. Meow!'; -} export function deactivate() { } diff --git a/chat-sample/vscode.proposed.chatParticipantAdditions.d.ts b/chat-sample/vscode.proposed.chatParticipantAdditions.d.ts new file mode 100644 index 00000000..cd2ec7ba --- /dev/null +++ b/chat-sample/vscode.proposed.chatParticipantAdditions.d.ts @@ -0,0 +1,252 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module 'vscode' { + + export interface ChatParticipant { + onDidPerformAction: Event; + } + + /** + * Now only used for the "intent detection" API below + */ + export interface ChatCommand { + readonly name: string; + readonly description: string; + } + + export class ChatResponseDetectedParticipantPart { + participant: string; + // TODO@API validate this against statically-declared slash commands? + command?: ChatCommand; + constructor(participant: string, command?: ChatCommand); + } + + export interface ChatVulnerability { + title: string; + description: string; + // id: string; // Later we will need to be able to link these across multiple content chunks. + } + + export class ChatResponseMarkdownWithVulnerabilitiesPart { + value: MarkdownString; + vulnerabilities: ChatVulnerability[]; + constructor(value: string | MarkdownString, vulnerabilities: ChatVulnerability[]); + } + + /** + * Displays a {@link Command command} as a button in the chat response. + */ + export interface ChatCommandButton { + command: Command; + } + + export interface ChatDocumentContext { + uri: Uri; + version: number; + ranges: Range[]; + } + + export class ChatResponseTextEditPart { + uri: Uri; + edits: TextEdit[]; + constructor(uri: Uri, edits: TextEdit | TextEdit[]); + } + + export class ChatResponseConfirmationPart { + title: string; + message: string; + data: any; + constructor(title: string, message: string, data: any); + } + + export type ExtendedChatResponsePart = ChatResponsePart | ChatResponseTextEditPart | ChatResponseDetectedParticipantPart | ChatResponseConfirmationPart; + + export class ChatResponseWarningPart { + value: MarkdownString; + constructor(value: string | MarkdownString); + } + + export class ChatResponseProgressPart2 extends ChatResponseProgressPart { + value: string; + task?: (progress: Progress) => Thenable; + constructor(value: string, task?: (progress: Progress) => Thenable); + } + + export interface ChatResponseStream { + + /** + * Push a progress part to this stream. Short-hand for + * `push(new ChatResponseProgressPart(value))`. + * + * @param value A progress message + * @param task If provided, a task to run while the progress is displayed. When the Thenable resolves, the progress will be marked complete in the UI, and the progress message will be updated to the resolved string if one is specified. + * @returns This stream. + */ + progress(value: string, task?: (progress: Progress) => Thenable): void; + + textEdit(target: Uri, edits: TextEdit | TextEdit[]): void; + markdownWithVulnerabilities(value: string | MarkdownString, vulnerabilities: ChatVulnerability[]): void; + detectedParticipant(participant: string, command?: ChatCommand): void; + push(part: ChatResponsePart | ChatResponseTextEditPart | ChatResponseDetectedParticipantPart | ChatResponseWarningPart | ChatResponseProgressPart2): void; + + /** + * Show an inline message in the chat view asking the user to confirm an action. + * Multiple confirmations may be shown per response. The UI might show "Accept All" / "Reject All" actions. + * @param title The title of the confirmation entry + * @param message An extra message to display to the user + * @param data An arbitrary JSON-stringifiable object that will be included in the ChatRequest when + * the confirmation is accepted or rejected + * TODO@API should this be MarkdownString? + * TODO@API should actually be a more generic function that takes an array of buttons + */ + confirmation(title: string, message: string, data: any): void; + + /** + * Push a warning to this stream. Short-hand for + * `push(new ChatResponseWarningPart(message))`. + * + * @param message A warning message + * @returns This stream. + */ + warning(message: string | MarkdownString): void; + + reference(value: Uri | Location | { variableName: string; value?: Uri | Location }, iconPath?: Uri | ThemeIcon | { light: Uri; dark: Uri }): void; + + push(part: ExtendedChatResponsePart): void; + } + + /** + * Does this piggy-back on the existing ChatRequest, or is it a different type of request entirely? + * Does it show up in history? + */ + export interface ChatRequest { + /** + * The `data` for any confirmations that were accepted + */ + acceptedConfirmationData?: any[]; + + /** + * The `data` for any confirmations that were rejected + */ + rejectedConfirmationData?: any[]; + } + + // TODO@API fit this into the stream + export interface ChatUsedContext { + documents: ChatDocumentContext[]; + } + + export interface ChatParticipant { + /** + * Provide a set of variables that can only be used with this participant. + */ + participantVariableProvider?: { provider: ChatParticipantCompletionItemProvider; triggerCharacters: string[] }; + } + + export interface ChatParticipantCompletionItemProvider { + provideCompletionItems(query: string, token: CancellationToken): ProviderResult; + } + + export class ChatCompletionItem { + id: string; + label: string | CompletionItemLabel; + values: ChatVariableValue[]; + fullName?: string; + icon?: ThemeIcon; + insertText?: string; + detail?: string; + documentation?: string | MarkdownString; + command?: Command; + + constructor(id: string, label: string | CompletionItemLabel, values: ChatVariableValue[]); + } + + export type ChatExtendedRequestHandler = (request: ChatRequest, context: ChatContext, response: ChatResponseStream, token: CancellationToken) => ProviderResult; + + export namespace chat { + /** + * Create a chat participant with the extended progress type + */ + export function createChatParticipant(id: string, handler: ChatExtendedRequestHandler): ChatParticipant; + + /** + * Current version of the proposal. Changes whenever backwards-incompatible changes are made. + * If a new feature is added that doesn't break existing code, the version is not incremented. When the extension uses this new feature, it should set its engines.vscode version appropriately. + * But if a change is made to an existing feature that would break existing code, the version should be incremented. + * The chat extension should not activate if it doesn't support the current version. + */ + export const _version: 1 | number; + } + + /* + * User action events + */ + + export enum ChatCopyKind { + // Keyboard shortcut or context menu + Action = 1, + Toolbar = 2 + } + + export interface ChatCopyAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'copy'; + codeBlockIndex: number; + copyKind: ChatCopyKind; + copiedCharacters: number; + totalCharacters: number; + copiedText: string; + } + + export interface ChatInsertAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'insert'; + codeBlockIndex: number; + totalCharacters: number; + newFile?: boolean; + } + + export interface ChatTerminalAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'runInTerminal'; + codeBlockIndex: number; + languageId?: string; + } + + export interface ChatCommandAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'command'; + commandButton: ChatCommandButton; + } + + export interface ChatFollowupAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'followUp'; + followup: ChatFollowup; + } + + export interface ChatBugReportAction { + // eslint-disable-next-line local/vscode-dts-string-type-literals + kind: 'bug'; + } + + export interface ChatEditorAction { + kind: 'editor'; + accepted: boolean; + } + + export interface ChatUserActionEvent { + readonly result: ChatResult; + readonly action: ChatCopyAction | ChatInsertAction | ChatTerminalAction | ChatCommandAction | ChatFollowupAction | ChatBugReportAction | ChatEditorAction; + } + + export interface ChatPromptReference { + /** + * TODO Needed for now to drive the variableName-type reference, but probably both of these should go away in the future. + */ + readonly name: string; + } +} diff --git a/chat-sample/vscode.proposed.chatTools.d.ts b/chat-sample/vscode.proposed.chatTools.d.ts new file mode 100644 index 00000000..97740f66 --- /dev/null +++ b/chat-sample/vscode.proposed.chatTools.d.ts @@ -0,0 +1,35 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module 'vscode' { + + export namespace lm { + /** + * Register a LanguageModelTool. The tool must also be registered in the package.json `languageModelTools` contribution point. + */ + export function registerTool(toolId: string, tool: LanguageModelTool): Disposable; + + /** + * A list of all available tools. + */ + export const tools: ReadonlyArray; + + /** + * Invoke a tool with the given parameters. + */ + export function invokeTool(toolId: string, parameters: Object, token: CancellationToken): Thenable; + } + + export interface LanguageModelToolDescription { + id: string; + description: string; + parametersSchema?: JSONSchema; + displayName?: string; + } + + export interface LanguageModelTool { + invoke(parameters: any, token: CancellationToken): Thenable; + } +}