From afe4d1eb3e5ce914ba65bb82c02331cee5b07772 Mon Sep 17 00:00:00 2001
From: isidorn <inikolic@microsoft.com>
Date: Thu, 1 Aug 2024 13:59:37 +0200
Subject: [PATCH] use telemetry logger for counting requests, and unhelpful
 feedback, to be able to compute success metric

---
 chat-sample/src/extension.ts | 40 +++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/chat-sample/src/extension.ts b/chat-sample/src/extension.ts
index 7caeb575..7d65d000 100644
--- a/chat-sample/src/extension.ts
+++ b/chat-sample/src/extension.ts
@@ -39,14 +39,15 @@ export function activate(context: vscode.ExtensionContext) {
                     }
                 }
             } catch(err) {
-                handleError(err, stream);
+                handleError(logger, err, stream);
             }
 
             stream.button({
                 command: CAT_NAMES_COMMAND_ID,
                 title: vscode.l10n.t('Use Cat Names in Editor')
             });
-
+            
+            logger.logUsage('request', { kind: 'teach'});
             return { metadata: { command: 'teach' } };
         } else if (request.command === 'play') {
             stream.progress('Throwing away the computer science books and preparing to play with some Python code...');
@@ -66,9 +67,10 @@ export function activate(context: vscode.ExtensionContext) {
                     }
                 }
             } catch(err) {
-                handleError(err, stream);
+                handleError(logger, err, stream);
             }
 
+            logger.logUsage('request', { kind: 'play'});
             return { metadata: { command: 'play' } };
         } else {
             try {
@@ -89,9 +91,10 @@ export function activate(context: vscode.ExtensionContext) {
                     }
                 }
             } catch(err) {
-                handleError(err, stream);
+                handleError(logger, err, stream);
             }
 
+            logger.logUsage('request', { kind: ''});
             return { metadata: { command: '' } };
         }
     };
@@ -111,6 +114,29 @@ export function activate(context: vscode.ExtensionContext) {
         }
     };
 
+    const logger = vscode.env.createTelemetryLogger({
+        sendEventData(eventName, data) {
+            // Capture event telemetry
+            console.log(`Event: ${eventName}`);
+            console.log(`Data: ${JSON.stringify(data)}`);
+        },
+        sendErrorData(error, data) {
+            // Capture error telemetry
+            console.error(`Error: ${error}`);
+            console.error(`Data: ${JSON.stringify(data)}`);
+        }
+    });
+
+    context.subscriptions.push(cat.onDidReceiveFeedback((feedback: vscode.ChatResultFeedback) => {
+        if (logger.isUsageEnabled) {
+            // Log chat result feedback to be able to compute the success matric of the participant
+            // unhelpful / totalRequests is a good success metric
+            logger.logUsage('chatResultFeedback', {
+                kind: feedback.kind
+            });
+        }
+    }));
+
     context.subscriptions.push(
         cat,
         // Register the command handler for the /meow followup
@@ -170,11 +196,15 @@ export function activate(context: vscode.ExtensionContext) {
     );
 }
 
-function handleError(err: any, stream: vscode.ChatResponseStream): void {
+function handleError(logger: vscode.TelemetryLogger, err: any, stream: vscode.ChatResponseStream): void {
     // making the chat request might fail because
     // - model does not exist
     // - user consent not given
     // - quote limits exceeded
+    if (logger.isErrorsEnabled) {
+        logger.logError(err);
+    }
+    
     if (err instanceof vscode.LanguageModelError) {
         console.log(err.message, err.code, err.cause);
         if (err.cause instanceof Error && err.cause.message.includes('off_topic')) {