From d3e0eea50b39fd60e220d42bfdc2b821e5bbb480 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Sat, 12 Oct 2024 15:01:33 -0700 Subject: [PATCH] Adopt lmTools updates --- chat-tools-sample/package.json | 2 +- chat-tools-sample/src/extension.ts | 82 ++----------------- chat-tools-sample/src/tools.ts | 4 +- chat-tools-sample/src/toolsPrompt.tsx | 8 +- chat-tools-sample/src/tsxParticipant.ts | 6 +- .../vscode.proposed.lmTools.d.ts | 54 ++++-------- 6 files changed, 34 insertions(+), 122 deletions(-) diff --git a/chat-tools-sample/package.json b/chat-tools-sample/package.json index 21d3f03d..0a20e7fc 100644 --- a/chat-tools-sample/package.json +++ b/chat-tools-sample/package.json @@ -51,7 +51,7 @@ "editors", "chat-tools-sample" ], - "name2": "tabCount", + "toolReferenceName": "tabCount", "displayName": "Tab Count", "modelDescription": "The number of active tabs in a tab group", "icon": "$(files)", diff --git a/chat-tools-sample/src/extension.ts b/chat-tools-sample/src/extension.ts index dba72dee..7f48c0c6 100644 --- a/chat-tools-sample/src/extension.ts +++ b/chat-tools-sample/src/extension.ts @@ -37,15 +37,9 @@ function registerChatParticipant(context: vscode.ExtensionContext) { }); const model = models[0]; - stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.id).join(', ')}\n\n`); + stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.name).join(', ')}\n\n`); - const allTools = vscode.lm.tools.map((tool): vscode.LanguageModelChatTool => { - return { - name: tool.id, - description: tool.description, - parametersSchema: tool.parametersSchema ?? {} - }; - }); + const allTools = vscode.lm.tools; const options: vscode.LanguageModelChatRequestOptions = { justification: 'Just because!', @@ -69,76 +63,16 @@ function registerChatParticipant(context: vscode.ExtensionContext) { options.tools = JSON.parse(`[{"type":"function","function":{"name":"copilot_codebase","description":"Search for relevant file chunks, symbols, and other info about the current workspace or codebase","parameters":{"type":"object","properties":{"query":{"type":"string","description":"The query to search the codebase for. Should contain all relevant context. Can be a full natural language sentence, or keywords."}},"required":["query"]}}},{"type":"function","function":{"name":"copilot_vscodeAPI","description":"Use VS Code API references to answer questions about VS Code extension development.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"The query to search vscode documentation for. Should contain all relevant context."}},"required":["query"]}}},{"type":"function","function":{"name":"ada-data_findFiles","description":"Search for files in the current workspace","parameters":{"type":"object","properties":{"pattern":{"type":"string","description":"Search for files that match this glob pattern"}},"required":["pattern"]}}},{"type":"function","function":{"name":"ada-data_runPython","description":"Execute Python code locally using Pyodide, providing access to Python's extensive functionality. This tool extends the LLM's capabilities by allowing it to run Python code for a wide range of computational tasks and data manipulations that it cannot perform directly. When you know the workspace folder path and the file path, use the relative path to the file when generating code.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The Python code to run"}},"required":["code"]}}}`) } else { options.toolChoice = undefined; - options.tools = allTools; + options.tools = [...allTools]; } const toolCalls: IToolCall[] = []; - - messages.splice(0, messages.length, - vscode.LanguageModelChatMessage.User(`Instructions: - The user will ask a question, or ask you to perform a task, and it may require lots of research to answer correctly. There is a selection of tools that let you perform actions or retrieve helpful context to answer the user's question. - If you aren't sure which tool is relevant, you can call multiple tools. You can call tools repeatedly to take actions or gather as much context as needed until you have completed the task fully. Don't give up unless you are sure the request cannot be fulfilled with the tools you have. - Don't make assumptions about the situation- gather context first, then perform the task or answer the question. - Don't ask the user for confirmation to use tools, just use them. - Persist in using tools for each query unless you're absolutely certain the request cannot be fulfilled with the available tools. - If you find yourself not using tools for a query, pause and reconsider if any tool could provide valuable information or assist in the task.`), - vscode.LanguageModelChatMessage.User(`Think of yourself as a data scientist who is analyzing a csv file using Python programming language. Ask user to clean up their missing data value and ask them for updated csv file. Provide them with code to remove the missing data value to remain interactive. Make sure to generate a pandas dataframe using the given csv file before performing data analysis. Make sure to perform statistical analysis on the data with actual numbers. Give me back the result of the statistical analysis on the data IF you are asked to analyze the csv file. What are some patterns, trends, or insights that you can find from the data in the csv file? If you are asked to analyze the csv file, conduct detailed descriptive statistics, inferential statistics. Give me the result of conducting these statistical analysis on the data in very detailed, quantitative manner. Be detailed and descriptive in your analysis. Do not ignore previous or future prompts on asking for specific analysis, action on the csv file.`), - vscode.LanguageModelChatMessage.User(`analyze housing.csv`), - vscode.LanguageModelChatMessage.Assistant(`Here is the full content of the \`housing.csv\` dataset: - -\`\`\`plaintext - longitude latitude housing_median_age total_rooms total_bedrooms \ - 0 - 122.23 37.88 41.0 880.0 129.0 -1 - 122.22 37.86 21.0 7099.0 1106.0 -2 - 122.24 37.85 52.0 1467.0 190.0 -3 - 122.25 37.85 52.0 1274.0 235.0 -4 - 122.25 37.85 52.0 1627.0 280.0 -... ... ... ... ... ... - 20635 - 121.09 39.48 25.0 1665.0 374.0 -20636 - 121.21 39.49 18.0 697.0 150.0 -20637 - 121.22 39.43 17.0 2254.0 485.0 -20638 - 121.32 39.43 18.0 1860.0 409.0 -20639 - 121.24 39.37 16.0 2785.0 616.0 - - population households median_income median_house_value \ - 0 322.0 126.0 8.3252 452600.0 -1 2401.0 1138.0 8.3014 358500.0 -2 496.0 177.0 7.2574 352100.0 -3 558.0 219.0 5.6431 341300.0 -4 565.0 259.0 3.8462 342200.0 -... ... ... ... ... - 20635 845.0 330.0 1.5603 78100.0 -20636 356.0 114.0 2.5568 77100.0 -20637 1007.0 433.0 1.7000 92300.0 -20638 741.0 349.0 1.8672 84700.0 -20639 1387.0 530.0 2.3886 89400.0 - - ocean_proximity -0 NEAR BAY -1 NEAR BAY -2 NEAR BAY -3 NEAR BAY -4 NEAR BAY -... ... - 20635 INLAND -20636 INLAND -20637 INLAND -20638 INLAND -20639 INLAND - - [20640 rows x 10 columns] - \`\`\``), - vscode.LanguageModelChatMessage.User(`Code executed from the tool: -\`\`\`import pandas as pd - -# Load the dataset -file_path = 'housing.csv' -df = pd.read_csv(file_path) -df\`\`\``), - vscode.LanguageModelChatMessage.User(`Make sure to use tools (copilot_codebase, copilot_vscodeAPI, ada-data_findFiles, ada-data_runPython) unless you're absolutely certain the request cannot be fulfilled with the available tools.`), - vscode.LanguageModelChatMessage.User(`help me visualize it`), - ); const response = await model.sendRequest(messages, options, token); - for await (const part of response.stream) { if (part instanceof vscode.LanguageModelTextPart) { stream.markdown(part.value); } else if (part instanceof vscode.LanguageModelToolCallPart) { - const tool = vscode.lm.tools.find(tool => tool.id === part.name); + const tool = vscode.lm.tools.find(tool => tool.name === part.name); if (!tool) { // BAD tool choice? throw new Error('Got invalid tool choice: ' + part.name); @@ -146,7 +80,7 @@ df\`\`\``), let parameters: any; try { - parameters = JSON.parse(part.parameters); + parameters = part.parameters; } catch (err) { throw new Error(`Got invalid tool use parameters: "${part.parameters}". (${(err as Error).message})`); } @@ -155,7 +89,7 @@ df\`\`\``), const requestedContentType = 'text/plain'; toolCalls.push({ call: part, - result: vscode.lm.invokeTool(tool.id, { parameters: JSON.parse(part.parameters), toolInvocationToken: request.toolInvocationToken, requestedContentTypes: [requestedContentType] }, token), + result: vscode.lm.invokeTool(tool.name, { parameters: part.parameters, toolInvocationToken: request.toolInvocationToken, requestedContentTypes: [requestedContentType] }, token), tool }); } @@ -163,7 +97,7 @@ df\`\`\``), if (toolCalls.length) { const assistantMsg = vscode.LanguageModelChatMessage.Assistant(''); - assistantMsg.content2 = toolCalls.map(toolCall => new vscode.LanguageModelToolCallPart(toolCall.tool.id, toolCall.call.toolCallId, toolCall.call.parameters)); + assistantMsg.content2 = toolCalls.map(toolCall => new vscode.LanguageModelToolCallPart(toolCall.tool.name, toolCall.call.toolCallId, toolCall.call.parameters)); messages.push(assistantMsg); for (const toolCall of toolCalls) { // NOTE that the result of calling a function is a special content type of a USER-message @@ -174,7 +108,7 @@ df\`\`\``), } // IMPORTANT The prompt must end with a USER message (with no tool call) - messages.push(vscode.LanguageModelChatMessage.User(`Above is the result of calling the functions ${toolCalls.map(call => call.tool.id).join(', ')}. The user cannot see this result, so you should explain it to the user if referencing it in your answer.`)); + messages.push(vscode.LanguageModelChatMessage.User(`Above is the result of calling the functions ${toolCalls.map(call => call.tool.name).join(', ')}. The user cannot see this result, so you should explain it to the user if referencing it in your answer.`)); // RE-enter return runWithFunctions(); diff --git a/chat-tools-sample/src/tools.ts b/chat-tools-sample/src/tools.ts index 99fc6a3c..65799e6d 100644 --- a/chat-tools-sample/src/tools.ts +++ b/chat-tools-sample/src/tools.ts @@ -36,7 +36,7 @@ export class TabCountTool implements vscode.LanguageModelTool { // Note- the final prompt must end with a UserMessage return <> - {this.props.toolCallRounds.map(round => this.renderOneToolCallRound(round, sizing))} + {this.props.toolCallRounds.map(round => this.renderOneToolCallRound(round))} Above is the result of calling one or more tools. The user cannot see the results, so you should explain them to the user if referencing them in your answer. } - private renderOneToolCallRound(round: ToolCallRound, sizing: PromptSizing) { + private renderOneToolCallRound(round: ToolCallRound) { const assistantToolCalls: ToolCall[] = round.toolCalls.map(tc => ({ type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.parameters) }, id: tc.toolCallId })); // TODO- just need to adopt prompt-tsx update in vscode-copilot return ( @@ -108,7 +108,7 @@ interface ToolCallElementProps extends BasePromptElementProps { class ToolCallElement extends PromptElement { async render(state: void, sizing: PromptSizing): Promise { - const tool = vscode.lm.tools.find(t => t.id === this.props.toolCall.name); + const tool = vscode.lm.tools.find(t => t.name === this.props.toolCall.name); if (!tool) { console.error(`Tool not found: ${this.props.toolCall.name}`); return Tool not found; @@ -116,7 +116,7 @@ class ToolCallElement extends PromptElement { const contentType = agentSupportedContentTypes.find(type => tool.supportedContentTypes.includes(type)); if (!contentType) { - console.error(`Tool does not support any of the agent's content types: ${tool.id}`); + console.error(`Tool does not support any of the agent's content types: ${tool.name}`); return Tool unsupported; } diff --git a/chat-tools-sample/src/tsxParticipant.ts b/chat-tools-sample/src/tsxParticipant.ts index ac488b85..b4a84b2f 100644 --- a/chat-tools-sample/src/tsxParticipant.ts +++ b/chat-tools-sample/src/tsxParticipant.ts @@ -22,7 +22,7 @@ export function isTsxToolUserMetadata(obj: unknown): obj is TsxToolUserMetadata export function registerTsxChatParticipant(context: vscode.ExtensionContext) { const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, chatContext: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken) => { if (request.command === 'list') { - stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.id).join(', ')}\n\n`); + stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.name).join(', ')}\n\n`); return; } @@ -34,7 +34,7 @@ export function registerTsxChatParticipant(context: vscode.ExtensionContext) { const model = models[0]; const allTools = vscode.lm.tools.map((tool): vscode.LanguageModelChatTool => { return { - name: tool.id, + name: tool.name, description: tool.description, parametersSchema: tool.parametersSchema ?? {} }; @@ -82,8 +82,6 @@ export function registerTsxChatParticipant(context: vscode.ExtensionContext) { stream.markdown(part.value); responseStr += part.value; } else if (part instanceof vscode.LanguageModelToolCallPart) { - // TODO vscode should be doing this - part.parameters = JSON.parse(part.parameters); toolCalls.push(part); } } diff --git a/chat-tools-sample/vscode.proposed.lmTools.d.ts b/chat-tools-sample/vscode.proposed.lmTools.d.ts index c5ecb486..3d3aec27 100644 --- a/chat-tools-sample/vscode.proposed.lmTools.d.ts +++ b/chat-tools-sample/vscode.proposed.lmTools.d.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 8 +// version: 9 // https://github.com/microsoft/vscode/issues/213274 declare module 'vscode' { @@ -12,10 +12,9 @@ declare module 'vscode' { // API -> LM: an tool/function that is available to the language model export interface LanguageModelChatTool { - // TODO@API should use "id" here to match vscode tools, or keep name to match OpenAI? Align everything. name: string; description: string; - parametersSchema?: JSONSchema; + parametersSchema?: object; } // API -> LM: add tools as request option @@ -33,9 +32,9 @@ declare module 'vscode' { export class LanguageModelToolCallPart { name: string; toolCallId: string; - parameters: any; + parameters: object; - constructor(name: string, toolCallId: string, parameters: any); + constructor(name: string, toolCallId: string, parameters: object); } // LM -> USER: text chunk @@ -93,7 +92,7 @@ declare module 'vscode' { * point. A registered tool is available in the {@link lm.tools} list for any extension to see. But in order for it to * be seen by a language model, it must be passed in the list of available tools in {@link LanguageModelChatRequestOptions.tools}. */ - export function registerTool(id: string, tool: LanguageModelTool): Disposable; + export function registerTool(name: string, tool: LanguageModelTool): Disposable; /** * A list of all available tools. @@ -103,7 +102,7 @@ declare module 'vscode' { /** * Invoke a tool with the given parameters. */ - export function invokeTool(id: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; + export function invokeTool(id: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; } /** @@ -120,6 +119,8 @@ declare module 'vscode' { * {@link ChatRequest.toolInvocationToken}. In that case, a progress bar will be automatically shown for the tool * invocation in the chat response view, and if the tool requires user confirmation, it will show up inline in the chat * view. If the tool is being invoked outside of a chat request, `undefined` should be passed instead. + * + * If a tool invokes another tool during its invocation, it can pass along the `toolInvocationToken` that it received. */ toolInvocationToken: ChatParticipantToolToken | undefined; @@ -154,26 +155,14 @@ declare module 'vscode' { }; } - /** - * Represents a JSON Schema. - * TODO@API - is this worth it? - */ - export type JSONSchema = Object; - /** * A description of an available tool. */ export interface LanguageModelToolDescription { /** - * A unique identifier for the tool. + * A unique name for the tool. */ - readonly id: string; - - /** - * A human-readable name for this tool that may be used to describe it in the UI. - * TODO@API keep? - */ - readonly displayName: string | undefined; + readonly name: string; /** * A description of this tool that may be passed to a language model. @@ -183,7 +172,7 @@ declare module 'vscode' { /** * A JSON schema for the parameters this tool accepts. */ - readonly parametersSchema?: JSONSchema; + readonly parametersSchema?: object; /** * The list of content types that the tool has declared support for. See {@link LanguageModelToolResult}. @@ -198,8 +187,8 @@ declare module 'vscode' { } /** - * Messages shown in the chat view when a tool needs confirmation from the user to run. These messages will be shown with - * buttons that say Continue and Cancel. + * When this is returned in {@link PreparedToolInvocation}, the user will be asked to confirm before running the tool. These + * messages will be shown with buttons that say "Continue" and "Cancel". */ export interface LanguageModelToolConfirmationMessages { /** @@ -208,10 +197,7 @@ declare module 'vscode' { title: string; /** - * The body of the confirmation message. This should be phrased as an action of the participant that is invoking the tool - * from {@link LanguageModelToolInvocationPrepareOptions.participantName}. An example of a good message would be - * `${participantName} will run the command ${echo 'hello world'} in the terminal.` - * TODO@API keep this? + * The body of the confirmation message. */ message: string | MarkdownString; } @@ -220,12 +206,6 @@ declare module 'vscode' { * Options for {@link LanguageModelTool.prepareToolInvocation}. */ export interface LanguageModelToolInvocationPrepareOptions { - /** - * The name of the participant invoking the tool. - * TODO@API keep this? - */ - participantName: string; - /** * The parameters that the tool is being invoked with. */ @@ -242,8 +222,8 @@ declare module 'vscode' { invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; /** - * Called once before a tool is invoked. May be implemented to customize the progress message that appears while the tool - * is running, and the messages that appear when the tool needs confirmation. + * Called once before a tool is invoked. May be implemented to signal that a tool needs user confirmation before running, + * and to customize the progress message that appears while the tool is running. */ prepareToolInvocation?(options: LanguageModelToolInvocationPrepareOptions, token: CancellationToken): ProviderResult; } @@ -258,7 +238,7 @@ declare module 'vscode' { invocationMessage?: string; /** - * Customized messages to show when asking for user confirmation to run the tool. + * The presence of this property indicates that the user should be asked to confirm before running the tool. */ confirmationMessages?: LanguageModelToolConfirmationMessages; }