From 118e189f59fd4c80143678442a8fd8f7b203e6a0 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Wed, 9 Oct 2024 21:12:17 -0700 Subject: [PATCH] Basic tsx tool user --- chat-tools-sample/package.json | 8 + chat-tools-sample/src/extension.ts | 147 +++++++++++++++- chat-tools-sample/src/toolsPrompt.tsx | 232 ++++++++++++++++++++++++++ 3 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 chat-tools-sample/src/toolsPrompt.tsx diff --git a/chat-tools-sample/package.json b/chat-tools-sample/package.json index 5238fd87..0afa5276 100644 --- a/chat-tools-sample/package.json +++ b/chat-tools-sample/package.json @@ -28,6 +28,14 @@ "description": "I use tools", "isSticky": true, "supportsToolReferences": true + }, + { + "id": "chat-tools-sample.tools2", + "fullName": "Tool User TSX", + "name": "toolsTSX", + "description": "I use tools with prompt-tsx", + "isSticky": true, + "supportsToolReferences": true } ], "languageModelTools": [ diff --git a/chat-tools-sample/src/extension.ts b/chat-tools-sample/src/extension.ts index 2c668ce3..8b9903da 100644 --- a/chat-tools-sample/src/extension.ts +++ b/chat-tools-sample/src/extension.ts @@ -1,9 +1,12 @@ import * as vscode from 'vscode'; import { FindFilesTool, RunInTerminalTool, TabCountTool } from './tools'; +import { renderPrompt } from '@vscode/prompt-tsx'; +import { ToolUserPrompt } from './toolsPrompt'; export function activate(context: vscode.ExtensionContext) { registerChatTool(context); registerChatParticipant(context); + registerChatParticipant2(context); } function registerChatTool(context: vscode.ExtensionContext) { @@ -61,7 +64,8 @@ function registerChatParticipant(context: vscode.ExtensionContext) { const requestedTool = toolReferences.shift(); if (requestedTool) { options.toolChoice = requestedTool.id; - options.tools = allTools.filter(tool => tool.name === requestedTool.id); + // options.tools = allTools.filter(tool => tool.name === requestedTool.id); + options.tools = JSON.parse(`[{"type":"function","function":{"name":"copilot_codebase","description":"Search for relevant file chunks, symbols, and other info about the current workspace or codebase","parameters":{"type":"object","properties":{"query":{"type":"string","description":"The query to search the codebase for. Should contain all relevant context. Can be a full natural language sentence, or keywords."}},"required":["query"]}}},{"type":"function","function":{"name":"copilot_vscodeAPI","description":"Use VS Code API references to answer questions about VS Code extension development.","parameters":{"type":"object","properties":{"query":{"type":"string","description":"The query to search vscode documentation for. Should contain all relevant context."}},"required":["query"]}}},{"type":"function","function":{"name":"ada-data_findFiles","description":"Search for files in the current workspace","parameters":{"type":"object","properties":{"pattern":{"type":"string","description":"Search for files that match this glob pattern"}},"required":["pattern"]}}},{"type":"function","function":{"name":"ada-data_runPython","description":"Execute Python code locally using Pyodide, providing access to Python's extensive functionality. This tool extends the LLM's capabilities by allowing it to run Python code for a wide range of computational tasks and data manipulations that it cannot perform directly. When you know the workspace folder path and the file path, use the relative path to the file when generating code.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The Python code to run"}},"required":["code"]}}}`) } else { options.toolChoice = undefined; options.tools = allTools; @@ -69,6 +73,64 @@ function registerChatParticipant(context: vscode.ExtensionContext) { const toolCalls: IToolCall[] = []; + messages.splice(0, messages.length, + vscode.LanguageModelChatMessage.User(`Instructions: - The user will ask a question, or ask you to perform a task, and it may require lots of research to answer correctly. There is a selection of tools that let you perform actions or retrieve helpful context to answer the user's question. - If you aren't sure which tool is relevant, you can call multiple tools. You can call tools repeatedly to take actions or gather as much context as needed until you have completed the task fully. Don't give up unless you are sure the request cannot be fulfilled with the tools you have. - Don't make assumptions about the situation- gather context first, then perform the task or answer the question. - Don't ask the user for confirmation to use tools, just use them. - Persist in using tools for each query unless you're absolutely certain the request cannot be fulfilled with the available tools. - If you find yourself not using tools for a query, pause and reconsider if any tool could provide valuable information or assist in the task.`), + vscode.LanguageModelChatMessage.User(`Think of yourself as a data scientist who is analyzing a csv file using Python programming language. Ask user to clean up their missing data value and ask them for updated csv file. Provide them with code to remove the missing data value to remain interactive. Make sure to generate a pandas dataframe using the given csv file before performing data analysis. Make sure to perform statistical analysis on the data with actual numbers. Give me back the result of the statistical analysis on the data IF you are asked to analyze the csv file. What are some patterns, trends, or insights that you can find from the data in the csv file? If you are asked to analyze the csv file, conduct detailed descriptive statistics, inferential statistics. Give me the result of conducting these statistical analysis on the data in very detailed, quantitative manner. Be detailed and descriptive in your analysis. Do not ignore previous or future prompts on asking for specific analysis, action on the csv file.`), + vscode.LanguageModelChatMessage.User(`analyze housing.csv`), + vscode.LanguageModelChatMessage.Assistant(`Here is the full content of the \`housing.csv\` dataset: + +\`\`\`plaintext + longitude latitude housing_median_age total_rooms total_bedrooms \ + 0 - 122.23 37.88 41.0 880.0 129.0 +1 - 122.22 37.86 21.0 7099.0 1106.0 +2 - 122.24 37.85 52.0 1467.0 190.0 +3 - 122.25 37.85 52.0 1274.0 235.0 +4 - 122.25 37.85 52.0 1627.0 280.0 +... ... ... ... ... ... + 20635 - 121.09 39.48 25.0 1665.0 374.0 +20636 - 121.21 39.49 18.0 697.0 150.0 +20637 - 121.22 39.43 17.0 2254.0 485.0 +20638 - 121.32 39.43 18.0 1860.0 409.0 +20639 - 121.24 39.37 16.0 2785.0 616.0 + + population households median_income median_house_value \ + 0 322.0 126.0 8.3252 452600.0 +1 2401.0 1138.0 8.3014 358500.0 +2 496.0 177.0 7.2574 352100.0 +3 558.0 219.0 5.6431 341300.0 +4 565.0 259.0 3.8462 342200.0 +... ... ... ... ... + 20635 845.0 330.0 1.5603 78100.0 +20636 356.0 114.0 2.5568 77100.0 +20637 1007.0 433.0 1.7000 92300.0 +20638 741.0 349.0 1.8672 84700.0 +20639 1387.0 530.0 2.3886 89400.0 + + ocean_proximity +0 NEAR BAY +1 NEAR BAY +2 NEAR BAY +3 NEAR BAY +4 NEAR BAY +... ... + 20635 INLAND +20636 INLAND +20637 INLAND +20638 INLAND +20639 INLAND + + [20640 rows x 10 columns] + \`\`\``), + vscode.LanguageModelChatMessage.User(`Code executed from the tool: +\`\`\`import pandas as pd + +# Load the dataset +file_path = 'housing.csv' +df = pd.read_csv(file_path) +df\`\`\``), + vscode.LanguageModelChatMessage.User(`Make sure to use tools (copilot_codebase, copilot_vscodeAPI, ada-data_findFiles, ada-data_runPython) unless you're absolutely certain the request cannot be fulfilled with the available tools.`), + vscode.LanguageModelChatMessage.User(`help me visualize it`), + ); const response = await model.sendRequest(messages, options, token); for await (const part of response.stream) { @@ -126,6 +188,89 @@ function registerChatParticipant(context: vscode.ExtensionContext) { context.subscriptions.push(toolUser); } +function registerChatParticipant2(context: vscode.ExtensionContext) { + const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, chatContext: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken) => { + const models = await vscode.lm.selectChatModels({ + vendor: 'copilot', + family: 'gpt-4o' + }); + + const model = models[0]; + stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.id).join(', ')}\n\n`); + + const allTools = vscode.lm.tools.map((tool): vscode.LanguageModelChatTool => { + return { + name: tool.id, + description: tool.description, + parametersSchema: tool.parametersSchema ?? {} + }; + }); + + const options: vscode.LanguageModelChatRequestOptions = { + justification: 'Just because!', + }; + + let { messages } = await renderPrompt( + ToolUserPrompt, + { + context: chatContext, + request, + toolCalls: [], + }, + { modelMaxPromptTokens: model.maxInputTokens }, + model) + + const toolReferences = [...request.toolReferences]; + const runWithFunctions = async (): Promise => { + const requestedTool = toolReferences.shift(); + if (requestedTool) { + options.toolChoice = requestedTool.id; + options.tools = allTools.filter(tool => tool.name === requestedTool.id); + } else { + options.toolChoice = undefined; + options.tools = allTools; + } + + const toolCalls: vscode.LanguageModelToolCallPart[] = []; + + const response = await model.sendRequest(messages, options, token); + let responseStr = ''; + for await (const part of response.stream) { + if (part instanceof vscode.LanguageModelTextPart) { + stream.markdown(part.value); + responseStr += part.value; + } else if (part instanceof vscode.LanguageModelToolCallPart) { + // TODO vscode should be doing this + part.parameters = JSON.parse(part.parameters); + + toolCalls.push(part); + } + } + + if (toolCalls.length) { + messages = (await renderPrompt( + ToolUserPrompt, + { + context: chatContext, + request, + toolCalls, + }, + { modelMaxPromptTokens: model.maxInputTokens }, + model)).messages; + + // RE-enter + return runWithFunctions(); + } + }; + + await runWithFunctions(); + }; + + const toolUser = vscode.chat.createChatParticipant('chat-tools-sample.tools2', handler); + toolUser.iconPath = new vscode.ThemeIcon('tools'); + context.subscriptions.push(toolUser); +} + async function getContextMessage(references: ReadonlyArray): Promise { const contextParts = (await Promise.all(references.map(async ref => { if (ref.value instanceof vscode.Uri) { diff --git a/chat-tools-sample/src/toolsPrompt.tsx b/chat-tools-sample/src/toolsPrompt.tsx new file mode 100644 index 00000000..6ee4ce1c --- /dev/null +++ b/chat-tools-sample/src/toolsPrompt.tsx @@ -0,0 +1,232 @@ +import { + AssistantMessage, + BasePromptElementProps, + contentType as promptTsxContentType, + PrioritizedList, + PromptElement, + PromptElementProps, + PromptPiece, + PromptSizing, + UserMessage, +} from '@vscode/prompt-tsx'; +import { ToolMessage, ToolResult } from '@vscode/prompt-tsx/dist/base/promptElements'; +import * as vscode from 'vscode'; + +export interface ToolUserProps extends BasePromptElementProps { + request: vscode.ChatRequest; + context: vscode.ChatContext; + toolCalls: vscode.LanguageModelToolCallPart[]; +} + +export class ToolUserPrompt extends PromptElement { + render(state: void, sizing: PromptSizing) { + return ( + <> + + Instructions:
+ - The user will ask a question, or ask you to perform a task, and it may + require lots of research to answer correctly. There is a selection of + tools that let you perform actions or retrieve helpful context to answer + the user's question.
+ - If you aren't sure which tool is relevant, you can call multiple + tools. You can call tools repeatedly to take actions or gather as much + context as needed until you have completed the task fully. Don't give up + unless you are sure the request cannot be fulfilled with the tools you + have.
+ - Don't make assumptions about the situation- gather context first, then + perform the task or answer the question.
+ - Don't ask the user for confirmation to use tools, just use them. +
- After editing a file, DO NOT show the user a codeblock with the + edit or new file contents. Assume that the user can see the result. + + + + {this.props.request.prompt} + + + ); + } +} + +interface ToolCallsProps extends BasePromptElementProps { + toolCalls: vscode.LanguageModelToolCallPart[]; + toolInvocationToken: vscode.ChatParticipantToolToken; +} + +class ToolCalls extends PromptElement { + render(state: void, sizing: PromptSizing) { + // TODO- prompt-tsx export this type? + // TODO- at what level do the parameters get stringified? + const assistantToolCalls: any[] = this.props.toolCalls.map(tc => ({ type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.parameters) }, id: tc.toolCallId })); + return <> + test + {this.props.toolCalls.map(toolCall => { + const tool = vscode.lm.tools.find(t => t.id === toolCall.name); + if (!tool) { + console.error(`Tool not found: ${toolCall.name}`); + return undefined; + } + + return ; + })} + Above is the result of calling one or more tools. The user cannot see the results, so you should explain them to the user if referencing them in your answer. + ; + } +} + +interface ToolCallProps extends BasePromptElementProps { + tool: vscode.LanguageModelToolDescription; + toolCall: vscode.LanguageModelToolCallPart; + toolInvocationToken: vscode.ChatParticipantToolToken; +} + +const agentSupportedContentTypes = [promptTsxContentType, 'text/plain']; + +const dummyCancellationToken: vscode.CancellationToken = new vscode.CancellationTokenSource().token; +class ToolCall extends PromptElement { + async render(state: void, sizing: PromptSizing) { + const contentType = agentSupportedContentTypes.find(type => this.props.tool.supportedContentTypes.includes(type)); + if (!contentType) { + console.error(`Tool does not support any of the agent's content types: ${this.props.tool.id}`); + return Tool unsupported; + } + + const tokenOptions: vscode.LanguageModelToolInvocationOptions['tokenOptions'] = { + tokenBudget: sizing.tokenBudget, + countTokens: async (content: string) => sizing.countTokens(content), + }; + + const result = await vscode.lm.invokeTool(this.props.toolCall.name, { parameters: this.props.toolCall.parameters, requestedContentTypes: [contentType], toolInvocationToken: this.props.toolInvocationToken, tokenOptions }, dummyCancellationToken); + return + {contentType === 'text/plain' ? + result[contentType] : + } + ; + } +} + +interface HistoryProps extends BasePromptElementProps { + priority: number; + context: vscode.ChatContext; +} + +class History extends PromptElement { + render(state: void, sizing: PromptSizing) { + return ( + + {this.props.context.history.map((message) => { + if (message instanceof vscode.ChatRequestTurn) { + return ( + <> + {} + {message.prompt} + + ); + } else if (message instanceof vscode.ChatResponseTurn) { + return ( + + {chatResponseToString(message)} + + ); + } + })} + + ); + } +} + +function chatResponseToString(response: vscode.ChatResponseTurn): string { + return response.response + .map((r) => { + if (r instanceof vscode.ChatResponseMarkdownPart) { + return r.value.value; + } else if (r instanceof vscode.ChatResponseAnchorPart) { + if (r.value instanceof vscode.Uri) { + return r.value.fsPath; + } else { + return r.value.uri.fsPath; + } + } + + return ''; + }) + .join(''); +} + +interface PromptReferencesProps extends BasePromptElementProps { + references: ReadonlyArray; +} + +class PromptReferences extends PromptElement { + render(state: void, sizing: PromptSizing): PromptPiece { + return ( + + {this.props.references.map((ref, index) => ( + + ))} + + ); + } +} + +interface PromptReferenceProps extends BasePromptElementProps { + ref: vscode.ChatPromptReference; +} + +class PromptReference extends PromptElement { + async render(state: void, sizing: PromptSizing): Promise { + const value = this.props.ref.value; + if (value instanceof vscode.Uri) { + const fileContents = (await vscode.workspace.fs.readFile(value)).toString(); + return ( + + {value.fsPath}:
+ ```
+ {fileContents}
+ ```
+ + ); + } else if (value instanceof vscode.Location) { + const rangeText = (await vscode.workspace.openTextDocument(value.uri)).getText(value.range); + return ( + + {value.uri.fsPath}:{value.range.start.line + 1}-$
+ {value.range.end.line + 1}: ```
+ {rangeText}
+ ``` + + ); + } else if (typeof value === 'string') { + return {value}; + } + } +} + +export type TagProps = PromptElementProps<{ + name: string; +}>; + +export class Tag extends PromptElement { + private static readonly _regex = /^[a-zA-Z_][\w\.\-]*$/; + + render() { + const { name } = this.props; + + if (!Tag._regex.test(name)) { + throw new Error(`Invalid tag name: ${this.props.name}`); + } + + return ( + <> + {'<' + name + '>'}
+ <> + {this.props.children}
+ + {''}
+ + ); + } +}