Use all available tools through function calling

This commit is contained in:
Rob Lourens
2024-06-19 18:44:06 -07:00
parent 6e7bc20380
commit 054dd382f4
4 changed files with 319 additions and 269 deletions

View File

@ -23,27 +23,11 @@
"contributes": {
"chatParticipants": [
{
"id": "chat-sample.cat",
"fullName": "Cat",
"name": "cat",
"description": "Meow! What can I teach you?",
"isSticky": true,
"commands": [
{
"name": "teach",
"description": "Pick at random a computer science concept then explain it in purfect way of a cat"
},
{
"name": "play",
"description": "Do whatever you want, you are a cat after all"
}
]
}
],
"commands": [
{
"command": "cat.namesInEditor",
"title": "Use Cat Names in Editor"
"id": "chat-sample.tools",
"fullName": "Tool User",
"name": "tools",
"description": "I use tools",
"isSticky": true
}
]
},

View File

@ -1,171 +1,39 @@
import * as vscode from 'vscode';
const CAT_NAMES_COMMAND_ID = 'cat.namesInEditor';
const CAT_PARTICIPANT_ID = 'chat-sample.cat';
interface ICatChatResult extends vscode.ChatResult {
metadata: {
command: string;
}
}
const MODEL_SELECTOR: vscode.LanguageModelChatSelector = { vendor: 'copilot', family: 'gpt-3.5-turbo' };
const PARTICIPANT_ID = 'chat-sample.tools';
export function activate(context: vscode.ExtensionContext) {
// Define a Cat chat handler.
const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, chatContext: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken) => {
const models = await vscode.lm.selectChatModels({
vendor: 'copilot',
family: 'gpt-4-turbo'
});
if (!models || !models.length) {
const chat = models[0];
if (!chat) {
console.log('NO MODELS')
return {};
}
// for (const model of models) {
// stream.markdown(`- ${model.name} (${model.family} - ${model.vendor})\n`)
// }
const chat = models[Math.floor(Math.random() * models.length)];
stream.progress(`Using ${chat.name} (${context.languageModelAccessInformation.canSendRequest(chat)})...`);
abstract class FunctionTool {
static All = new Map<string, {
metadata: vscode.LanguageModelChatFunction,
run: (...args: any[]) => Promise<string>
}>();
static register(metadata: vscode.LanguageModelChatFunction, run: (...args: any[]) => Promise<string>) {
FunctionTool.All.set(metadata.name, { metadata, run: run });
}
}
// get the size of an editor
FunctionTool.register({
name: "get_length_of_editor",
description: "Get the length of an editor",
parametersSchema: {
"type": "object",
"properties": {
"nth": {
"type": "number",
"description": "The index of the editor, starting at 0",
},
},
"required": ["nth"],
},
}, async (arg: { nth: number }) => {
if (!(arg && typeof arg === 'object' && typeof arg.nth === 'number')) {
return 'Error: Invalid arguments, expected { nth: number}';
}
const editor = vscode.window.visibleTextEditors[arg.nth];
if (!editor) {
return `Warning: No editor found at index ${arg.nth}, please try a different index between 0 and ${vscode.window.visibleTextEditors.length - 1}`;
}
return editor.document.getText().length.toString();
})
FunctionTool.register({
name: "show_user_message",
description: "Show a message to the user",
parametersSchema: {
"type": "object",
"properties": {
"message": {
"type": "string",
"description": "The message to show",
},
},
"required": ["message"],
},
}, async (arg: { message: string }) => {
if (!(arg && typeof arg === 'object' && typeof arg.message === 'string')) {
return 'Error: Invalid arguments, expected { message: string}';
}
vscode.window.showInformationMessage(arg.message);
return 'done';
});
FunctionTool.register({
name: "current_temperature",
description: "Get the current temperature for a location",
parametersSchema: {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The location to check the tepmerature for",
},
},
"required": ["location"],
},
}, async (arg: { location: string }) => {
if (!(arg && typeof arg === 'object' && typeof arg.location === 'string')) {
return 'Error: Invalid arguments, expected { location: string}';
}
return 'The temperature in ' + arg.location + ' is 25°C';
});
FunctionTool.register({
name: "start_debugging",
description: "Start debugging the given file",
parametersSchema: {
"type": "object",
"properties": {
"filename": {
"type": "string",
"description": "The file to debug",
},
},
"required": ["filename"],
},
}, async (arg: { filename: string }) => {
if (!(arg && typeof arg === 'object' && typeof arg.filename === 'string')) {
return 'Error: Invalid arguments, expected { filename: string}';
}
vscode.debug.startDebugging(undefined, {
name: 'debug',
type: 'node',
request: 'launch',
program: vscode.Uri.joinPath(vscode.workspace.workspaceFolders![0].uri, arg.filename).fsPath,
});
return 'done';
});
FunctionTool.register({
name: "create_terminal",
description: "Create a terminal with the given name",
parametersSchema: {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "The terminal name",
},
},
"required": ["name"],
},
}, async (arg: { name: string }) => {
if (!(arg && typeof arg === 'object' && typeof arg.name === 'string')) {
return 'Error: Invalid arguments, expected { name: string}';
}
vscode.window.createTerminal(arg.name).show();
return 'done';
});
stream.markdown(`Available tools: ${vscode.lm.tools.map(tool => tool.id).join(', ')}\n\n`);
const options: vscode.LanguageModelChatRequestOptions = {
tools: Array.from(FunctionTool.All.values()).map(tool => tool.metadata),
tools: vscode.lm.tools.map((tool): vscode.LanguageModelChatFunction => {
return {
name: tool.id.replace(/\./g, '_'),
description: tool.description,
parametersSchema: tool.parametersSchema ?? {}
}
}),
justification: 'Just because!',
}
const messages = [vscode.LanguageModelChatMessage.User(request.prompt)];
const messages = [
vscode.LanguageModelChatMessage.User(`There is a selection of tools that may give helpful context to answer the user's query. If you aren't sure which tool is relevant, you can call multiple tools.`),
vscode.LanguageModelChatMessage.User(request.prompt)
];
const runWithFunctions = async () => {
let didReceiveFunctionUse = false;
@ -173,30 +41,31 @@ export function activate(context: vscode.ExtensionContext) {
const response = await chat.sendRequest(messages, options, token);
for await (const part of response.stream) {
if (part instanceof vscode.LanguageModelChatResponseTextPart) {
stream.markdown(part.value)
} else if (part instanceof vscode.LanguageModelChatResponseFunctionUsePart) {
const tool = FunctionTool.All.get(part.name);
const tool = vscode.lm.tools.find(tool => tool.id.replace(/\./g, '_') === part.name);
if (!tool) {
// BAD tool choice?
continue;
}
stream.progress(`FUNCTION_CALL: ${tool.metadata.name} with \`${part.parameters}\``)
const resultPromise = vscode.lm.invokeTool(tool.id, JSON.parse(part.parameters), token);
stream.progress(`FUNCTION_CALL: ${tool.id} with \`${part.parameters}\``, async (progress) => {
await resultPromise;
});
const result = await tool.run(JSON.parse(part.parameters));
const result = await resultPromise;
// NOTE that the result of calling a function is a special content type of a USER-message
let message = vscode.LanguageModelChatMessage.User('');
message.content2 = new vscode.LanguageModelChatMessageFunctionResultPart(tool.metadata.name, result)
message.content2 = new vscode.LanguageModelChatMessageFunctionResultPart(tool.id, result)
messages.push(message)
// IMPORTANT
// IMPORTANT working around CAPI always wanting to end with a `User`-message
// IMPORTANT
messages.push(vscode.LanguageModelChatMessage.User('Above is the result of calling the function ${tool.metadata.name}'))
messages.push(vscode.LanguageModelChatMessage.User(`Above is the result of calling the function ${tool.id}. The user cannot see this result, so you should explain it to the user if referencing it in your answer.`))
didReceiveFunctionUse = true;
}
}
@ -210,99 +79,9 @@ export function activate(context: vscode.ExtensionContext) {
await runWithFunctions()
};
// Chat participants appear as top-level options in the chat input
// when you type `@`, and can contribute sub-commands in the chat input
// that appear when you type `/`.
const cat = vscode.chat.createChatParticipant(CAT_PARTICIPANT_ID, handler);
cat.iconPath = vscode.Uri.joinPath(context.extensionUri, 'cat.jpeg');
cat.followupProvider = {
provideFollowups(result: ICatChatResult, context: vscode.ChatContext, token: vscode.CancellationToken) {
return [{
prompt: 'let us play',
label: vscode.l10n.t('Play with the cat'),
command: 'play'
} satisfies vscode.ChatFollowup];
}
};
context.subscriptions.push(
cat,
// Register the command handler for the /meow followup
vscode.commands.registerTextEditorCommand(CAT_NAMES_COMMAND_ID, async (textEditor: vscode.TextEditor) => {
// Replace all variables in active editor with cat names and words
const text = textEditor.document.getText();
const messages = [
vscode.LanguageModelChatMessage.User(`You are a cat! Think carefully and step by step like a cat would.
Your job is to replace all variable names in the following code with funny cat variable names. Be creative. IMPORTANT respond just with code. Do not use markdown!`),
vscode.LanguageModelChatMessage.User(text)
];
let chatResponse: vscode.LanguageModelChatResponse | undefined;
try {
const [model] = await vscode.lm.selectChatModels({ vendor: 'copilot', family: 'gpt-3.5-turbo' });
if (!model) {
console.log('Model not found. Please make sure the GitHub Copilot Chat extension is installed and enabled.')
return;
}
chatResponse = await model.sendRequest(messages, {}, new vscode.CancellationTokenSource().token);
} catch (err) {
// making the chat request might fail because
// - model does not exist
// - user consent not given
// - quote limits exceeded
if (err instanceof vscode.LanguageModelError) {
console.log(err.message, err.code, err.cause)
}
return;
}
// Clear the editor content before inserting new content
await textEditor.edit(edit => {
const start = new vscode.Position(0, 0);
const end = new vscode.Position(textEditor.document.lineCount - 1, textEditor.document.lineAt(textEditor.document.lineCount - 1).text.length);
edit.delete(new vscode.Range(start, end));
});
// Stream the code into the editor as it is coming in from the Language Model
try {
for await (const fragment of chatResponse.text) {
await textEditor.edit(edit => {
const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1);
const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length);
edit.insert(position, fragment);
});
}
} catch (err) {
// async response stream may fail, e.g network interruption or server side error
await textEditor.edit(edit => {
const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1);
const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length);
edit.insert(position, (<Error>err).message);
});
}
}),
);
const toolUser = vscode.chat.createChatParticipant(PARTICIPANT_ID, handler);
toolUser.iconPath = new vscode.ThemeIcon('tools');
}
// Get a random topic that the cat has not taught in the chat history yet
function getTopic(history: ReadonlyArray<vscode.ChatRequestTurn | vscode.ChatResponseTurn>): string {
const topics = ['linked list', 'recursion', 'stack', 'queue', 'pointers'];
// Filter the chat history to get only the responses from the cat
const previousCatResponses = history.filter(h => {
return h instanceof vscode.ChatResponseTurn && h.participant == CAT_PARTICIPANT_ID
}) as vscode.ChatResponseTurn[];
// Filter the topics to get only the topics that have not been taught by the cat yet
const topicsNoRepetition = topics.filter(topic => {
return !previousCatResponses.some(catResponse => {
return catResponse.response.some(r => {
return r instanceof vscode.ChatResponseMarkdownPart && r.value.value.includes(topic)
});
});
});
return topicsNoRepetition[Math.floor(Math.random() * topicsNoRepetition.length)] || 'I have taught you everything I know. Meow!';
}
export function deactivate() { }

View File

@ -0,0 +1,252 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
declare module 'vscode' {
export interface ChatParticipant {
onDidPerformAction: Event<ChatUserActionEvent>;
}
/**
* Now only used for the "intent detection" API below
*/
export interface ChatCommand {
readonly name: string;
readonly description: string;
}
export class ChatResponseDetectedParticipantPart {
participant: string;
// TODO@API validate this against statically-declared slash commands?
command?: ChatCommand;
constructor(participant: string, command?: ChatCommand);
}
export interface ChatVulnerability {
title: string;
description: string;
// id: string; // Later we will need to be able to link these across multiple content chunks.
}
export class ChatResponseMarkdownWithVulnerabilitiesPart {
value: MarkdownString;
vulnerabilities: ChatVulnerability[];
constructor(value: string | MarkdownString, vulnerabilities: ChatVulnerability[]);
}
/**
* Displays a {@link Command command} as a button in the chat response.
*/
export interface ChatCommandButton {
command: Command;
}
export interface ChatDocumentContext {
uri: Uri;
version: number;
ranges: Range[];
}
export class ChatResponseTextEditPart {
uri: Uri;
edits: TextEdit[];
constructor(uri: Uri, edits: TextEdit | TextEdit[]);
}
export class ChatResponseConfirmationPart {
title: string;
message: string;
data: any;
constructor(title: string, message: string, data: any);
}
export type ExtendedChatResponsePart = ChatResponsePart | ChatResponseTextEditPart | ChatResponseDetectedParticipantPart | ChatResponseConfirmationPart;
export class ChatResponseWarningPart {
value: MarkdownString;
constructor(value: string | MarkdownString);
}
export class ChatResponseProgressPart2 extends ChatResponseProgressPart {
value: string;
task?: (progress: Progress<ChatResponseWarningPart | ChatResponseReferencePart>) => Thenable<string | void>;
constructor(value: string, task?: (progress: Progress<ChatResponseWarningPart | ChatResponseReferencePart>) => Thenable<string | void>);
}
export interface ChatResponseStream {
/**
* Push a progress part to this stream. Short-hand for
* `push(new ChatResponseProgressPart(value))`.
*
* @param value A progress message
* @param task If provided, a task to run while the progress is displayed. When the Thenable resolves, the progress will be marked complete in the UI, and the progress message will be updated to the resolved string if one is specified.
* @returns This stream.
*/
progress(value: string, task?: (progress: Progress<ChatResponseWarningPart | ChatResponseReferencePart>) => Thenable<string | void>): void;
textEdit(target: Uri, edits: TextEdit | TextEdit[]): void;
markdownWithVulnerabilities(value: string | MarkdownString, vulnerabilities: ChatVulnerability[]): void;
detectedParticipant(participant: string, command?: ChatCommand): void;
push(part: ChatResponsePart | ChatResponseTextEditPart | ChatResponseDetectedParticipantPart | ChatResponseWarningPart | ChatResponseProgressPart2): void;
/**
* Show an inline message in the chat view asking the user to confirm an action.
* Multiple confirmations may be shown per response. The UI might show "Accept All" / "Reject All" actions.
* @param title The title of the confirmation entry
* @param message An extra message to display to the user
* @param data An arbitrary JSON-stringifiable object that will be included in the ChatRequest when
* the confirmation is accepted or rejected
* TODO@API should this be MarkdownString?
* TODO@API should actually be a more generic function that takes an array of buttons
*/
confirmation(title: string, message: string, data: any): void;
/**
* Push a warning to this stream. Short-hand for
* `push(new ChatResponseWarningPart(message))`.
*
* @param message A warning message
* @returns This stream.
*/
warning(message: string | MarkdownString): void;
reference(value: Uri | Location | { variableName: string; value?: Uri | Location }, iconPath?: Uri | ThemeIcon | { light: Uri; dark: Uri }): void;
push(part: ExtendedChatResponsePart): void;
}
/**
* Does this piggy-back on the existing ChatRequest, or is it a different type of request entirely?
* Does it show up in history?
*/
export interface ChatRequest {
/**
* The `data` for any confirmations that were accepted
*/
acceptedConfirmationData?: any[];
/**
* The `data` for any confirmations that were rejected
*/
rejectedConfirmationData?: any[];
}
// TODO@API fit this into the stream
export interface ChatUsedContext {
documents: ChatDocumentContext[];
}
export interface ChatParticipant {
/**
* Provide a set of variables that can only be used with this participant.
*/
participantVariableProvider?: { provider: ChatParticipantCompletionItemProvider; triggerCharacters: string[] };
}
export interface ChatParticipantCompletionItemProvider {
provideCompletionItems(query: string, token: CancellationToken): ProviderResult<ChatCompletionItem[]>;
}
export class ChatCompletionItem {
id: string;
label: string | CompletionItemLabel;
values: ChatVariableValue[];
fullName?: string;
icon?: ThemeIcon;
insertText?: string;
detail?: string;
documentation?: string | MarkdownString;
command?: Command;
constructor(id: string, label: string | CompletionItemLabel, values: ChatVariableValue[]);
}
export type ChatExtendedRequestHandler = (request: ChatRequest, context: ChatContext, response: ChatResponseStream, token: CancellationToken) => ProviderResult<ChatResult | void>;
export namespace chat {
/**
* Create a chat participant with the extended progress type
*/
export function createChatParticipant(id: string, handler: ChatExtendedRequestHandler): ChatParticipant;
/**
* Current version of the proposal. Changes whenever backwards-incompatible changes are made.
* If a new feature is added that doesn't break existing code, the version is not incremented. When the extension uses this new feature, it should set its engines.vscode version appropriately.
* But if a change is made to an existing feature that would break existing code, the version should be incremented.
* The chat extension should not activate if it doesn't support the current version.
*/
export const _version: 1 | number;
}
/*
* User action events
*/
export enum ChatCopyKind {
// Keyboard shortcut or context menu
Action = 1,
Toolbar = 2
}
export interface ChatCopyAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'copy';
codeBlockIndex: number;
copyKind: ChatCopyKind;
copiedCharacters: number;
totalCharacters: number;
copiedText: string;
}
export interface ChatInsertAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'insert';
codeBlockIndex: number;
totalCharacters: number;
newFile?: boolean;
}
export interface ChatTerminalAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'runInTerminal';
codeBlockIndex: number;
languageId?: string;
}
export interface ChatCommandAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'command';
commandButton: ChatCommandButton;
}
export interface ChatFollowupAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'followUp';
followup: ChatFollowup;
}
export interface ChatBugReportAction {
// eslint-disable-next-line local/vscode-dts-string-type-literals
kind: 'bug';
}
export interface ChatEditorAction {
kind: 'editor';
accepted: boolean;
}
export interface ChatUserActionEvent {
readonly result: ChatResult;
readonly action: ChatCopyAction | ChatInsertAction | ChatTerminalAction | ChatCommandAction | ChatFollowupAction | ChatBugReportAction | ChatEditorAction;
}
export interface ChatPromptReference {
/**
* TODO Needed for now to drive the variableName-type reference, but probably both of these should go away in the future.
*/
readonly name: string;
}
}

View File

@ -0,0 +1,35 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
declare module 'vscode' {
export namespace lm {
/**
* Register a LanguageModelTool. The tool must also be registered in the package.json `languageModelTools` contribution point.
*/
export function registerTool(toolId: string, tool: LanguageModelTool): Disposable;
/**
* A list of all available tools.
*/
export const tools: ReadonlyArray<LanguageModelToolDescription>;
/**
* Invoke a tool with the given parameters.
*/
export function invokeTool(toolId: string, parameters: Object, token: CancellationToken): Thenable<string>;
}
export interface LanguageModelToolDescription {
id: string;
description: string;
parametersSchema?: JSONSchema;
displayName?: string;
}
export interface LanguageModelTool {
invoke(parameters: any, token: CancellationToken): Thenable<string>;
}
}