diff --git a/packages/vscode-extension/src/officeChat/common/declarationFinder.ts b/packages/vscode-extension/src/officeChat/common/declarationFinder.ts index 830de1ae63..f9ffff5bca 100644 --- a/packages/vscode-extension/src/officeChat/common/declarationFinder.ts +++ b/packages/vscode-extension/src/officeChat/common/declarationFinder.ts @@ -4,7 +4,7 @@ import ts = require("typescript"); import { fetchRawFileContent } from "./utils"; import { SampleData } from "./samples/sampleData"; -import { DocParagraph, DocPlainText, TSDocParser } from "@microsoft/tsdoc"; +import { DocLinkTag, DocParagraph, DocPlainText, TSDocParser } from "@microsoft/tsdoc"; export class DeclarationFinder { private static DECLARATION_FILE_NAME = "office-js.d.ts"; @@ -128,6 +128,14 @@ export class DeclarationFinder { } private getDocCommentAndSummary(node: ts.Node): { docComment: string; summary: string } { + // For the comments, we'd like to get the summary section of the comments. For example: + // /** + // * Sets multiple properties of an object at the same time. You can pass either a plain object with the appropriate properties, or another API object of the same type. + // * @param properties A JavaScript object with properties that are structured isomorphically to the properties of the object on which the method is called. + // * @param options Provides an option to suppress errors if the properties object tries to set any read-only properties. + // */ + // We expect to get the summary section of the comments, like: + // "Sets multiple properties of an object at the same time. You can pass either a plain object with the appropriate properties, or another API object of the same type." const sourceFile = this.definionFile; const commentRanges = ts.getLeadingCommentRanges(sourceFile!.text, node.pos); const comments: string | undefined = commentRanges @@ -144,9 +152,19 @@ export class DeclarationFinder { while (!summarySectionNext.done) { const node = summarySectionNext.value; if (node.kind === "PlainText") { + // Deal with the plain text in the summary section. Like: + // "Gets the first note item in this collection. Throws an `ItemNotFound` error if this collection is empty." description += (node as DocPlainText).text.trim().replace("`", "'") + " "; } + if (node.kind === "LinkTag") { + const link = node as DocLinkTag; + if (link.linkText) { + description += " " + link.linkText + " "; + } + } if (node.kind === "Paragraph") { + // dealing with comments has extra content beyond pure text (link, for example), like: + // "Contains a collection of {@link Word.NoteItem} objects." const paragraph = node as DocParagraph; const paragraphIterator = paragraph.nodes.values(); let paragraphNext = paragraphIterator.next(); @@ -156,6 +174,22 @@ export class DeclarationFinder { description += (paragraphNode as unknown as DocPlainText).text.trim().replace("`", "'") + " "; } + // dealing with links in the paragraph, like: + // "{@link Word.NoteItem}" + // It will get the Word.NoteItem from the link. + if (paragraphNode.kind === "LinkTag") { + const link = paragraphNode as DocLinkTag; + let plainText = ""; + if (link.codeDestination) { + const parts = link.codeDestination.memberReferences.map( + (memberReference) => memberReference.memberIdentifier?.identifier + ); + plainText += parts.join("."); + } else { + plainText += link.linkText || ""; + } + description += ` ${plainText} `; + } paragraphNext = paragraphIterator.next(); } } diff --git a/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts b/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts index ec2a8789e0..f23b83aa86 100644 --- a/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts +++ b/packages/vscode-extension/src/officeChat/common/samples/sampleProvider.ts @@ -9,6 +9,7 @@ import { prepareDiscription } from "../../retrievalUtil/retrievalUtil"; import { countMessagesTokens, getCopilotResponseAsString } from "../../../chat/utils"; import { getMostRelevantClassPrompt, + getMostRelevantClassUsingNameOnlyPrompt, getMostRelevantMethodPropertyPrompt, } from "../../officePrompts"; import { DeclarationFinder } from "../declarationFinder"; @@ -57,6 +58,12 @@ export class SampleProvider { }); } + /** + * Get the most relevant declarations using the language model. + * Due to the limitation of the token count, we have to split the process into a few steps. + * The first step is to get the most relevant classes based on the understanding of the code spec and the sample. + * The second step is to get the most relevant methods or properties from selected classes from previous step, based on the understanding of the code spec and the sample. + */ public async getMostRelevantDeclarationsUsingLLM( token: CancellationToken, host: string, @@ -64,30 +71,46 @@ export class SampleProvider { sample: string ): Promise> { const pickedDeclarations: Map = new Map(); - + const model: "copilot-gpt-3.5-turbo" | "copilot-gpt-4" = "copilot-gpt-4"; const t1 = performance.now(); + let countOfLLMInvoke = 0; const classSummaries = await DeclarationFinder.getInstance().getClassSummariesForHost(host); if (classSummaries.length === 0) { return pickedDeclarations; } + + // It is possible that with the increase of the number of classes, the token count of the message will exceed the limitation. So if the token count exceeds the limitation, we will use the prompt that only contains the class name rather than the class's description to reduce the token count. let sampleMessage: LanguageModelChatUserMessage = new LanguageModelChatUserMessage( getMostRelevantClassPrompt(codeSpec, classSummaries, sample) ); + let msgCount = countMessagesTokens([sampleMessage]); + if (msgCount > getTokenLimitation(model)) { + sampleMessage = new LanguageModelChatUserMessage( + getMostRelevantClassUsingNameOnlyPrompt(codeSpec, classSummaries, sample) + ); + msgCount = countMessagesTokens([sampleMessage]); + } - let copilotResponse = await getCopilotResponseAsString( - "copilot-gpt-3.5-turbo", // "copilot-gpt-3.5-turbo", // "copilot-gpt-4", - [sampleMessage], - token - ); - - let returnObject: { picked: string[] } = JSON.parse(copilotResponse); - if (returnObject.picked.length === 0) { + if (msgCount > getTokenLimitation(model)) { + console.debug( + "[getMostRelevantDeclarationsUsingLLM] The token count of the message exceeds the limitation." + ); return pickedDeclarations; } + + countOfLLMInvoke += 1; + const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token); + + const returnObject: { picked: string[] } = JSON.parse( + copilotResponse.replace("```json", "").replace("```", "").replace(/\\n/g, "") + ); const classNames: string[] = returnObject.picked.map((value) => value.replace("- ", "").trim()); if (classNames.length === 0) { + console.debug("[getMostRelevantDeclarationsUsingLLM] No relevant class found for this task."); return pickedDeclarations; + } else { + console.debug("[getMostRelevantDeclarationsUsingLLM] The relevant classes are: ", classNames); } const t2 = performance.now(); @@ -98,27 +121,35 @@ export class SampleProvider { classDeclarationPairs.push([className, methodsOrProperties]); } + const giantMethodsOrPropertiesSet: Map[] = []; + // It is possible that the token count of the message will exceed the limitatiotn. So we have to split the process into a few steps. In some cases, a single class may has huge amount of methods or properties we can't afford, we have to skip it. For example, the class "Worksheet" in Excel has 100+ methods and properties. while (classDeclarationPairs.length > 0) { let msgCount = 0; + // The following two variables are used to store the classes and methods/properties that will contains in the message send to copilot later, the token count of the message will be safe. let classNamesList: string[] = []; - const classNamesListTemp: string[] = []; let methodsOrProperties: SampleData[] = []; + // following two variables are temporary used to store the classes and methods/properties to calculate the token count. The token count of the message could exceed the limitation. + const classNamesListTemp: string[] = []; const methodsOrPropertiesTemp: SampleData[] = []; + + let groupedMethodsOrProperties: Map = new Map(); let getMoreRelevantMethodsOrPropertiesPrompt = ""; - while (msgCount < getTokenLimitation("copilot-gpt-3.5-turbo")) { - const candidate = classDeclarationPairs.pop(); + + // The while loop is used to get the classes and methods/properties that will contains in the message send to copilot later, the token count of the message will be safe. Those used classes will be removed from the classDeclarationPairs. + let candidate: [string, SampleData[]] | undefined; + while (msgCount < getTokenLimitation(model)) { + classNamesList = classNamesListTemp.map((value) => value); + methodsOrProperties = methodsOrPropertiesTemp.map((value) => value); + + candidate = classDeclarationPairs.pop(); if (!candidate) { break; } - classNamesList = classNamesListTemp.map((value) => value); + classNamesListTemp.unshift(candidate[0]); - methodsOrProperties = methodsOrPropertiesTemp.map((value) => value); methodsOrPropertiesTemp.unshift(...candidate[1]); // group the methods or properties by class name - const groupedMethodsOrProperties: Map = new Map< - string, - SampleData[] - >(); + groupedMethodsOrProperties = new Map(); for (const methodOrProperty of methodsOrPropertiesTemp) { if (!groupedMethodsOrProperties.has(methodOrProperty.definition)) { groupedMethodsOrProperties.set(methodOrProperty.definition, []); @@ -127,19 +158,23 @@ export class SampleProvider { } getMoreRelevantMethodsOrPropertiesPrompt = getMostRelevantMethodPropertyPrompt( codeSpec, - classNamesList, + classNamesListTemp, groupedMethodsOrProperties, sample ); sampleMessage = new LanguageModelChatUserMessage(getMoreRelevantMethodsOrPropertiesPrompt); msgCount = countMessagesTokens([sampleMessage]); } - if (methodsOrProperties.length === 0) { - // For class that has huge amount of methods or properties, we have to skip it. - continue; + if (msgCount > getTokenLimitation(model)) { + if (methodsOrProperties.length === 0) { + giantMethodsOrPropertiesSet.push(groupedMethodsOrProperties); + continue; + } else { + classDeclarationPairs.push(candidate as [string, SampleData[]]); + } } // group the methods or properties by class name - const groupedMethodsOrProperties: Map = new Map(); + groupedMethodsOrProperties = new Map(); for (const methodOrProperty of methodsOrProperties) { if (!groupedMethodsOrProperties.has(methodOrProperty.definition)) { groupedMethodsOrProperties.set(methodOrProperty.definition, []); @@ -147,48 +182,172 @@ export class SampleProvider { groupedMethodsOrProperties.get(methodOrProperty.definition)?.push(methodOrProperty); } - getMoreRelevantMethodsOrPropertiesPrompt = getMostRelevantMethodPropertyPrompt( + countOfLLMInvoke += 1; + const picked = await this.getMostRelevantPropertiesOrMethodsDeclaratitons( codeSpec, classNamesList, groupedMethodsOrProperties, - sample - ); - sampleMessage = new LanguageModelChatUserMessage(getMoreRelevantMethodsOrPropertiesPrompt); - copilotResponse = await getCopilotResponseAsString( - "copilot-gpt-3.5-turbo", // "copilot-gpt-3.5-turbo", // "copilot-gpt-4", - [sampleMessage], - token + sample, + methodsOrProperties, + token, + model ); + picked.forEach((value, key) => { + if (!pickedDeclarations.has(key)) { + pickedDeclarations.set(key, value); + } + }); + } - try { - returnObject = JSON.parse(copilotResponse); - } catch (error) { - console.log(copilotResponse); - } + for (const groupedMethodsOrProperties of giantMethodsOrPropertiesSet) { + for (const key of Array.from(groupedMethodsOrProperties.keys())) { + const classNamesListTemp = [key]; + let methodOrPropertyDeclarationsTemp: SampleData[] = []; + let methodOrPropertyDeclarations: SampleData[] = []; - returnObject.picked.forEach((value: string) => { - const sampleData = methodsOrProperties.find( - (sample) => - value.trim() == sample.codeSample.trim() || - value.trim().endsWith(sample.codeSample.trim()) || - sample.codeSample.trim().endsWith(value.trim()) || - value.trim().indexOf(sample.codeSample.trim()) >= 0 || - sample.codeSample.trim().indexOf(value.trim()) >= 0 - ); - if (sampleData) { - pickedDeclarations.set(sampleData.description, sampleData); + while ((groupedMethodsOrProperties.get(key) || []).length > 0) { + methodOrPropertyDeclarationsTemp = []; + do { + methodOrPropertyDeclarations = + groupedMethodsOrProperties.get(key) || ([] as SampleData[]); + if (methodOrPropertyDeclarations.length > 1) { + methodOrPropertyDeclarationsTemp.push( + methodOrPropertyDeclarations.pop() as SampleData + ); + } + + const getMoreRelevantMethodsOrPropertiesPrompt = getMostRelevantMethodPropertyPrompt( + codeSpec, + classNamesListTemp, + groupedMethodsOrProperties, + sample + ); + sampleMessage = new LanguageModelChatUserMessage( + getMoreRelevantMethodsOrPropertiesPrompt + ); + msgCount = countMessagesTokens([sampleMessage]); + } while (msgCount > getTokenLimitation(model)); + + countOfLLMInvoke += 1; + const picked = await this.getMostRelevantPropertiesOrMethodsDeclaratitons( + codeSpec, + classNamesListTemp, + groupedMethodsOrProperties, + sample, + methodOrPropertyDeclarations, + token, + model + ); + picked.forEach((value, key) => { + if (!pickedDeclarations.has(key)) { + pickedDeclarations.set(key, value); + } + }); + + groupedMethodsOrProperties.delete(key); + groupedMethodsOrProperties.set(key, methodOrPropertyDeclarationsTemp); } - }); + } } const t3 = performance.now(); console.log( - `Pick relevant classes: ${(t2 - t1) / 1000} seconds, get methods/properties: ${ - (t3 - t2) / 1000 - }.` + `Pick relevant classes: ${(t2 - t1) / 1000} seconds, get ${ + pickedDeclarations.size + } methods/properties: ${(t3 - t2) / 1000}, count of LLM invoking: ${countOfLLMInvoke}.` ); return new Promise>((resolve, reject) => { resolve(pickedDeclarations); }); } + + private async getMostRelevantPropertiesOrMethodsDeclaratitons( + codeSpec: string, + classNamesList: string[], + groupedMethodsOrProperties: Map, + sample: string, + methodsOrProperties: SampleData[], + token: CancellationToken, + model: "copilot-gpt-3.5-turbo" | "copilot-gpt-4" + ): Promise> { + const pickedDeclarations: Map = new Map(); + const getMoreRelevantMethodsOrPropertiesPrompt = getMostRelevantMethodPropertyPrompt( + codeSpec, + classNamesList, + groupedMethodsOrProperties, + sample + ); + const sampleMessage = new LanguageModelChatUserMessage( + getMoreRelevantMethodsOrPropertiesPrompt + ); + const copilotResponse = await getCopilotResponseAsString(model, [sampleMessage], token); + + let returnObject: { picked: string[] } = { picked: [] }; + try { + returnObject = JSON.parse( + copilotResponse.replace("```json", "").replace("```", "").replace(/\\n/g, "") + ); + } catch (error) { + console.log(copilotResponse); + } + + returnObject.picked.forEach((value: string) => { + // The return may contains encoded characters, we need to decode them. + const parts = value + .replace(/</g, "<") + .replace(/>/g, ">") + .split(";") + .map((part) => part.trim()) + .filter((part) => part.length > 0); + if (parts.length == 1) { + // Sometimes the return in the format of "method1;" without class name + const methodPropertyDeclaration = parts[0].trim() + ";"; + // methodPropertyDeclaration = methodPropertyDeclaration.endsWith(";") + // ? methodPropertyDeclaration + // : methodPropertyDeclaration + ";"; + const sampleData = methodsOrProperties.find( + (sample) => sample.codeSample.trim() === methodPropertyDeclaration + ); + if (sampleData) { + pickedDeclarations.set(sampleData.description, sampleData); + } else { + console.debug("[parts.length == 1]: " + methodPropertyDeclaration); + } + } else if (parts.length > 2) { + // Sometimes the return in the format of "class: className; method1; method2; ...; methodN;" + const className = parts[0].replace("class:", "").trim(); + for (let i = 1; i < parts.length - 1; i++) { + const methodPropertyDeclaration = parts[i].trim() + ";"; + // methodPropertyDeclaration = methodPropertyDeclaration.endsWith(";") + // ? methodPropertyDeclaration + // : methodPropertyDeclaration + ";"; + const sampleData = methodsOrProperties.find( + (sample) => + sample.definition.trim() === className && + sample.codeSample.trim() === methodPropertyDeclaration + ); + if (sampleData) { + pickedDeclarations.set(sampleData.description, sampleData); + } + } + } else if (parts.length === 2) { + // in the format of "class: className; methodOrPropertyDeclaration;" + const className = parts[0].replace("class:", "").trim(); + const methodPropertyDeclaration = parts[1].trim() + ";"; + // methodPropertyDeclaration = methodPropertyDeclaration.endsWith(";") + // ? methodPropertyDeclaration + // : methodPropertyDeclaration + ";"; + const sampleData = methodsOrProperties.find( + (sample) => + sample.definition.trim() === className && + sample.codeSample.trim() === methodPropertyDeclaration + ); + if (sampleData) { + pickedDeclarations.set(sampleData.description, sampleData); + } + } + }); + + return pickedDeclarations; + } } diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts b/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts index dc75e5403a..21affbd053 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeGenerator.ts @@ -25,7 +25,6 @@ import { customFunctionSystemPrompt, getUserInputBreakdownTaskUserPrompt, getUserAskPreScanningSystemPrompt, - getUserComplexAskBreakdownTaskSystemPrompt, getUserSimpleAskBreakdownTaskSystemPrompt, getGenerateCodeUserPrompt, getGenerateCodeSamplePrompt, @@ -34,6 +33,7 @@ import { } from "../../officePrompts"; import { localize } from "../../../utils/localizeUtils"; import { getTokenLimitation } from "../../consts"; +import { SampleData } from "../samples/sampleData"; export class CodeGenerator implements ISkill { name: string; @@ -122,7 +122,6 @@ export class CodeGenerator implements ISkill { spec.appendix.codeTaskBreakdown = breakdownResult.funcs; spec.appendix.codeExplanation = breakdownResult.spec; } - if (!spec.appendix.telemetryData.measurements[MeasurementCodeGenAttemptCount]) { spec.appendix.telemetryData.measurements[MeasurementCodeGenAttemptCount] = 0; } @@ -184,8 +183,8 @@ export class CodeGenerator implements ISkill { // Perform the desired operation const messages: LanguageModelChatMessage[] = [ - new LanguageModelChatSystemMessage(defaultSystemPrompt), new LanguageModelChatUserMessage(userPrompt), + new LanguageModelChatSystemMessage(defaultSystemPrompt), ]; const copilotResponse = await getCopilotResponseAsString( "copilot-gpt-3.5-turbo", // "copilot-gpt-4", // "copilot-gpt-3.5-turbo", @@ -210,7 +209,11 @@ export class CodeGenerator implements ISkill { } else { copilotRet = JSON.parse(codeSnippetRet[1].trim()); } - console.log(`The complexity score: ${copilotRet.complexity}`); + console.debug( + `Custom functions: ${copilotRet.customFunctions ? "true" : "false"}, Complexity score: ${ + copilotRet.complexity + }` + ); } catch (error) { console.error("[User task scanning] Failed to parse the response from Copilot:", error); return null; @@ -230,24 +233,21 @@ export class CodeGenerator implements ISkill { spec: string; funcs: string[]; }> { - const userPrompt = getUserInputBreakdownTaskUserPrompt(userInput); - const defaultSystemPrompt = - complexity >= 50 - ? getUserComplexAskBreakdownTaskSystemPrompt(userInput) - : getUserSimpleAskBreakdownTaskSystemPrompt(userInput); + let userPrompt = getUserSimpleAskBreakdownTaskSystemPrompt(userInput); + if (isCustomFunctions) { + userPrompt = `This is a task about Excel custom functions, pay attention if this is a regular custom functions or streaming custom functions:\n\n ${userPrompt}`; + } + userPrompt += "\nThink about that step by step."; // Perform the desired operation - const messages: LanguageModelChatMessage[] = [ - new LanguageModelChatUserMessage(userPrompt), - new LanguageModelChatSystemMessage(defaultSystemPrompt), - ]; + const messages: LanguageModelChatMessage[] = [new LanguageModelChatUserMessage(userPrompt)]; if (sampleCode.length > 0) { messages.push(new LanguageModelChatSystemMessage(getCodeSamplePrompt(sampleCode))); } const copilotResponse = await getCopilotResponseAsString( - "copilot-gpt-3.5-turbo", //"copilot-gpt-4", // "copilot-gpt-3.5-turbo", + "copilot-gpt-4", //"copilot-gpt-4", // "copilot-gpt-3.5-turbo", messages, token ); @@ -307,40 +307,73 @@ export class CodeGenerator implements ISkill { break; } - const declarations = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( - token, - host, - codeSpec, - sampleCode - ); + if (!spec.appendix.apiDeclarationsReference || !spec.appendix.apiDeclarationsReference.size) { + const declarations = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + token, + host, + codeSpec, + "" //sampleCode + ); - spec.appendix.apiDeclarationsReference = declarations; + spec.appendix.apiDeclarationsReference = declarations; + } let declarationPrompt = getGenerateCodeDeclarationPrompt(); - if (declarations.size > 0) { - declarationPrompt += Array.from(declarations.values()) - .map((declaration) => `- ${declaration.definition}`) - .join("\n"); + if (spec.appendix.apiDeclarationsReference.size > 0) { + const groupedMethodsOrProperties: Map = new Map(); + spec.appendix.apiDeclarationsReference.forEach((declaration) => { + if (!groupedMethodsOrProperties.has(declaration.definition)) { + groupedMethodsOrProperties.set(declaration.definition, []); + } + groupedMethodsOrProperties.get(declaration.definition)?.push(declaration); + }); + + let tempClassDeclaration = "\n```typescript\n"; + groupedMethodsOrProperties.forEach((methodsOrPropertiesCandidates, className) => { + tempClassDeclaration += ` +class ${className} extends OfficeExtension.ClientObject { + ${methodsOrPropertiesCandidates.map((sampleData) => sampleData.codeSample).join("\n")} +} +\n + `; + }); + tempClassDeclaration += "```\n"; + + declarationPrompt += tempClassDeclaration; + console.debug(`API declarations: \n${declarationPrompt}`); } + const model: "copilot-gpt-4" | "copilot-gpt-3.5-turbo" = "copilot-gpt-4"; + let msgCount = 0; - let samplePrompt = getGenerateCodeSamplePrompt(); + // Perform the desired operation + // The order in array is matter, don't change it unless you know what you are doing + const messages: LanguageModelChatMessage[] = [new LanguageModelChatUserMessage(userPrompt)]; if (sampleCode.length > 0) { + let samplePrompt = getGenerateCodeSamplePrompt(); samplePrompt += ` + \n \`\`\`typescript ${sampleCode} \`\`\` + + Let's think step by step. `; + messages.push(new LanguageModelChatSystemMessage(samplePrompt)); } - // Perform the desired operation - // The order in array is matter, don't change it unless you know what you are doing - const messages: LanguageModelChatMessage[] = [ - new LanguageModelChatUserMessage(userPrompt), - new LanguageModelChatSystemMessage(declarationPrompt), - new LanguageModelChatSystemMessage(samplePrompt), - new LanguageModelChatSystemMessage(referenceUserPrompt), - ]; - const model: "copilot-gpt-4" | "copilot-gpt-3.5-turbo" = "copilot-gpt-4"; - let msgCount = countMessagesTokens(messages); + // May sure for the custom functions, the reference user prompt is shown first so it has lower risk to be cut off + if (isCustomFunctions) { + messages.push( + new LanguageModelChatSystemMessage(referenceUserPrompt), + new LanguageModelChatSystemMessage(declarationPrompt) + ); + } else { + messages.push( + new LanguageModelChatSystemMessage(declarationPrompt), + new LanguageModelChatSystemMessage(referenceUserPrompt) + ); + } + // Because of the token window limitation, we have to cut off the messages if it exceeds the limitation + msgCount = countMessagesTokens(messages); while (msgCount > getTokenLimitation(model)) { messages.pop(); msgCount = countMessagesTokens(messages); diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts b/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts index a64f1ddd5e..9acf4b84c1 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeIssueCorrector.ts @@ -28,6 +28,7 @@ import { } from "../../officePrompts"; import { localize } from "../../../utils/localizeUtils"; import { getTokenLimitation } from "../../consts"; +import { SampleData } from "../samples/sampleData"; export class CodeIssueCorrector implements ISkill { static MAX_TRY_COUNT = 10; // From the observation from a small set of test, fix over 2 rounds leads to worse result, set it to a smal number so we can fail fast @@ -101,21 +102,33 @@ export class CodeIssueCorrector implements ISkill { let setDeclartionPrompt = getDeclarationsPrompt(); - if (spec.appendix.apiDeclarationsReference.size > 0) { - const codeSnippets: string[] = []; - spec.appendix.apiDeclarationsReference.forEach((sample, api) => { - console.debug(`[Code corrector] Declaration matched: ${sample.description}`); - codeSnippets.push(` -- [Description] ${sample.description}: -\`\`\`typescript -${sample.codeSample} -\`\`\`\n -`); + if (!!spec.appendix.apiDeclarationsReference && !!spec.appendix.apiDeclarationsReference.size) { + const groupedMethodsOrProperties = new Map(); + for (const methodOrProperty of spec.appendix.apiDeclarationsReference) { + if (!groupedMethodsOrProperties.has(methodOrProperty[1].definition)) { + groupedMethodsOrProperties.set(methodOrProperty[1].definition, [methodOrProperty[1]]); + } + groupedMethodsOrProperties.get(methodOrProperty[1].definition)?.push(methodOrProperty[1]); + } + + let tempClassDeclaration = ""; + groupedMethodsOrProperties.forEach((methodsOrPropertiesCandidates, className) => { + tempClassDeclaration += ` +class ${className} extends OfficeExtension.ClientObject { + ${methodsOrPropertiesCandidates.map((sampleData) => sampleData.codeSample).join("\n\n")} +} +\n\n + `; }); - if (codeSnippets.length > 0) { - setDeclartionPrompt = setDeclartionPrompt.concat(`\n${codeSnippets.join("\n")}\n\n`); - } + setDeclartionPrompt += ` + + \`\`\`typescript + ${tempClassDeclaration}; + \`\`\` + + Let's think step by step. + `; } const declarationMessage: LanguageModelChatSystemMessage | null = spec.appendix.apiDeclarationsReference.size > 0 diff --git a/packages/vscode-extension/src/officeChat/common/skills/codeIssueDetector.ts b/packages/vscode-extension/src/officeChat/common/skills/codeIssueDetector.ts index bf63f30e53..9afbaad6b0 100644 --- a/packages/vscode-extension/src/officeChat/common/skills/codeIssueDetector.ts +++ b/packages/vscode-extension/src/officeChat/common/skills/codeIssueDetector.ts @@ -465,6 +465,10 @@ export class CodeIssueDetector { while (node && !ts.isCallExpression(node)) { node = node.parent; } + + if (!node) { + return; + } const callExpression = node; if (!ts.isCallExpression(callExpression)) { diff --git a/packages/vscode-extension/src/officeChat/consts.ts b/packages/vscode-extension/src/officeChat/consts.ts index c852fd2ce9..cb92fc8ff9 100644 --- a/packages/vscode-extension/src/officeChat/consts.ts +++ b/packages/vscode-extension/src/officeChat/consts.ts @@ -12,12 +12,5 @@ export const enum OfficeChatCommand { } export function getTokenLimitation(model: "copilot-gpt-3.5-turbo" | "copilot-gpt-4"): number { - if (model === "copilot-gpt-3.5-turbo") { - return 3500; - } else if (model === "copilot-gpt-4") { - // This is strange for gt4, the limit is less than 4k - return 3500; - } - - return 3500; + return 3000; } diff --git a/packages/vscode-extension/src/officeChat/officePrompts.ts b/packages/vscode-extension/src/officeChat/officePrompts.ts index 64caf1f835..e4a07ed14b 100644 --- a/packages/vscode-extension/src/officeChat/officePrompts.ts +++ b/packages/vscode-extension/src/officeChat/officePrompts.ts @@ -188,9 +188,10 @@ export function getUserInputBreakdownTaskUserPrompt(userInput: string): string { You are an expert in Office JavaScript Add-ins, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. # Your tasks: - For this given ask: "${userInput}", that is about automate a process using Office JavaScript API. I need you help to analyze it, and give me your suggestion in the format of JSON object. + For this given task: "${userInput}", that is about automate a process using Office JavaScript API. I need you help to analyze it under the context of Office JavaScript addins and Office JavaScript APIs, and give me your suggestion in the format of JSON object. You should pay attention to the following points: + - Your language should be clear for a Office Add-ins developer to follow. + - Some of the term sounds like generic term, but they're not, they're specific to Office applications, like "Annotation", "Comment", "Range", "Table", "Chart", "Worksheet", "Workbook", "Document", "Slide", "Presentation", "Taskpane", "Custom Functions", "Shape", etc. You should keep those terms in the context of Office applications not general term. - Think about that step by step. `; } @@ -236,7 +237,7 @@ export function getUserAskPreScanningSystemPrompt(): string { `; } -export function getUserComplexAskBreakdownTaskSystemPrompt(userInput: string): string { +export function getUserSimpleAskBreakdownTaskSystemPrompt(userInput: string): string { return ` The following content written using Markdown syntax, using "Bold" style to highlight the key information. @@ -244,50 +245,37 @@ export function getUserComplexAskBreakdownTaskSystemPrompt(userInput: string): s You are an expert in Office JavaScript Add-ins, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. # Context: - The output must be a JSON object wrapped into a markdown json block, and it will contain the following keys: - - spec. value is a string. - - funcs. value is a array of string. - - # Your tasks: - Analyze the user input: ${userInput}, understand the intentions, and how Office JavaScript API could help to address that ask. Read the sample code snippet if it provided, make it as reference on steps on similar scenario. Then deduce your think result to two parts: - 1. You need to write a detail functional spec on how to step by step to achieve the user's ask, especially those parts that need to interact with Office applications. The function spec should not include code snippet or suggestion on APIs, but should include the explanation on what need to do. Let me repeat that, you should tell perform what action, rather than tell use what API. The spec It should be clear, concise, and easy to understand. Add that spec to the "spec" field of the output JSON object. - 2. According to the spec, break the ask down into a few TypeScript functions. For each function, give a one line function description, that should have detail description of the function intention. Do not break the description into multiple sub items. Add those function descriptions to the "funcs" field of the output JSON object. - - bypass step like "create a new Office Add-ins project" or "create a new Excel workbook" or "create a new Word document" or "create a new PowerPoint presentation". - - bypass step like "open the workbook" or "open the document" or "open the presentation". - - bypass step like "save the workbook" or "save the document" or "save the presentation". - - bypass step like the "generate Addins Code" or "generate xxx Code". - - bypass step like "Use the Office JavaScript Add-ins API to perform the required operations". - - bypass step like "Register the xxx function". + The input is - # Example of one line function description: - - Create a function named 'createTrendlineChart'. This function should create a trendline chart in the worksheet where dates are set as the x-value and prices as the y-value. - - # The format of output: - Beyond the JSON object. You should not add anything else to the output. - The example of output you must to follow: - { - spec: "The functional spec", - funcs: ["function1 description", "function2 description"] - } - `; -} - -export function getUserSimpleAskBreakdownTaskSystemPrompt(userInput: string): string { - return ` - The following content written using Markdown syntax, using "Bold" style to highlight the key information. - - # Role: - You are an expert in Office JavaScript Add-ins, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. + \`\`\`text + ${userInput} + \`\`\` - # Context: The output must be a JSON object wrapped into a markdown json block, and it will contain the following keys: - spec. value is a string. - - funcs. value is a array of string. However, for the simple task, the array should contain only one string. + - funcs. value is a array of string. # Your tasks: - Analyze the user input: ${userInput}, understand the intentions, and how Office JavaScript API could help to address that ask. Reference sample code snippet if it provided, deduce your think result to two parts: - 1. You need to write a detailed functional spec on how to step by step to achieve the user's ask, especially those parts that need to interact with Office applications. The function spec should not include code snippet or suggestion on APIs, but should include the explanation on what need to do. Let me repeat that, you should tell perform what action, rather than tell use what API. The spec It should be clear, concise, and easy to understand. Add that spec to the "spec" field of the output JSON object. - 2. According to the spec, suggest a name of TypeScript function. And then, give a one line function description, that should have description of the function intention, what parameters it should take, and what it should return. Do not break the description into multiple sub items. Add put the function description to the "funcs" field of the output JSON object. + Read the input, understand the intention and request of the ask, and think about how Office JavaScript API could help to address them. Then deduce your think result to two parts: + 1. Write a clear and detailed functional specification that explains how you will address the given ask, especially for parts that interact with Office applications. + - The specification should focus on actions, not APIs, and be concise and easy to understand. + - The specification should only contains content of request explicitly asked by user. + - The specification should use the term been widely used in the software developing world. + - The specification should use the term been widely used in the context of Microsoft Office application depends on the ask. + - The specification should use the term been widely used in the context of Office JavaScript Add-in development. + Add the specification to the "spec" field of the output JSON object. + 2. Based on the spec, think about how to write the code, and wrap the code into a single function, suggest a name for that TypeScript function. And provide a one-line description for that function, that includes the function's core functionality and actions in details. Do not break the description into sub-items. Add the function description to the "funcs" field of the output JSON object. + + # Example of specification: + + \`\`\`text + To retrieve the content of the initial footnote in a Word document using Office JavaScript APIs, you can follow these steps: + 1. Get the current selection in the document. + 2. Check if the selection contains any footnotes. + 3. Retrieve the first footnote in the collection. + 4. Fetch the content of the footnote. + 5. Process the retrieved content as needed. + \`\`\` # Example of one line function description: - Create a function named 'createTrendlineChart'. This function should create a trendline chart in the worksheet where dates are set as the x-value and prices as the y-value. @@ -304,9 +292,8 @@ export function getUserSimpleAskBreakdownTaskSystemPrompt(userInput: string): st export function getCodeSamplePrompt(codeSample: string): string { return ` - The following content written using Markdown syntax, using "Bold" style to highlight the key information. + Some code snippets provided below, that they may address different user scenarios. Read those code and get list of scenarios those code try to address. Find if any scenarios match the user's ask, and use the relevant code snippet or code logic as a reference in your task. - # There're some samples relevant to the user's ask, read it through and think why the sample write like that, and how it could be used in your task.: \`\`\`typescript ${codeSample} \`\`\` @@ -336,43 +323,52 @@ export function getCodeGenerateGuidance(host: string) { } export function getGenerateCodeUserPrompt( - userInput: string, + codeSpec: string, host: string, functionSpec: string[] ): string { return ` -The following content written using Markdown syntax, using "Bold" style to highlight the key information. - -# Your role: -You're a professional and senior Office JavaScript Add-ins developer with a lot of experience and know all best practice on TypeScript, JavaScript, popular algorithm, Office Add-ins API, and deep understanding on the feature of Office applications (Word, Excel, PowerPoint). You should help the user to automate a certain process or accomplish a certain task, by generate TypeScript code using Office JavaScript Add-ins. + The request is about Office application "${host}". -# Context: -This is the ask need your help to generate the code for this request: ${userInput}. -- The request is about Office Add-ins, and it is relevant to the Office application "${host}". -- It's a functional spec you should follow. **Read through those descriptions, and repeat by yourself**. Make sure you understand that before go to the task: -${functionSpec.map((spec) => `- ${spec}`).join("\n")} + # Your role: + You're a professional and senior Office JavaScript Add-ins developer with a lot of experience and know all best practice on TypeScript, JavaScript, popular algorithm, Office Add-ins API, and deep understanding on the feature of Office applications (Word, Excel, PowerPoint). You should help the user to automate a certain process or accomplish a certain task, by generate TypeScript code using Office JavaScript APIs. + + # Context: + The input is: -# Your tasks: -Generate code for each listed functions based on the user request, the generated code **MUST** include implementations of those functions listed above, and not limited to this. Code write in **TypeScript code** and **Office JavaScript Add-ins API**, while **follow the coding rule**. Do not generate code to invoke the "main" function or "entry" function if that function generated. + \`\`\`text + ${codeSpec} + \`\`\` -${getCodeGenerateGuidance(host)} + The output must be a markdown code typescript code block and it will contain the generated code, which is looks like: -# Format of output: -**You must strickly follow the format of output**. The output will only contains code without any explanation on the code or generate process. Beyond that, nothing else should be included in the output. -- The code surrounded by a pair of triple backticks, and must follow with a string "typescript". For example: -\`\`\`typescript -// The code snippet -\`\`\` + \`\`\`typescript + // The generated code + \`\`\` + + # Your tasks: + Analyze this input, focus on the mentioned asks, and generate code to fulfill the descriptions of following listed functions: + ${functionSpec.map((spec) => `- ${spec}`).join("\n")} + The generated code **MUST** include implementations of mentioned functions listed in the input. Do not generate code to invoke the "main" function or "entry" function. + You should follow the code guidance on generating the code. + ${getCodeGenerateGuidance(host)} + + # Format of output: + **You must strickly follow the format of output**. The output will only contains code without any explanation on the code or generate process. Beyond that, nothing else should be included in the output. + - The code surrounded by a pair of triple backticks, and must follow with a string "typescript". For example: + \`\`\`typescript + // The code snippet + \`\`\` -Let's think step by step. - `; + Let's think step by step. + `; } export function getGenerateCodeSamplePrompt(): string { return ` - The following content written using Markdown syntax, using "Bold" style to highlight the key information. + Sample code provided below, read and understand it. In case the sample code contains solution or code snippet to the user's request, you should use the solution or code snippet as a reference in your task. - # There're some samples relevant to the user's ask, you must read and repeat following samples before generate code. And then use the content and coding styles as your reference when you generate code: + # Sample code: `; } @@ -386,9 +382,9 @@ export function getDeclarationsPrompt(): string { export function getGenerateCodeDeclarationPrompt(): string { return ` - The following content written using Markdown syntax, using "Bold" style to highlight the key information. + The following content are some TypeScript code relevant to the user's ask, follow those TypeScript declarations when you generate the code. Make sure you understand that before go to the task: - # There're some TypeScript declarations relevant to the user's ask, you should reference those declarations when you generate code: + # Office JavaScript API declarations: `; } @@ -422,7 +418,7 @@ ${ } # Your tasks: -Fix all errors on the given code snippet then return the updated code snippet back. +Fix all errors on the given code snippet then return the updated code snippet back. If sample code is provided, read and understand it. Which it should contains approaches and code snippets on same scenarios. Use if as reference if applicable. Let's think step by step. `; @@ -672,37 +668,79 @@ export function getMostRelevantClassPrompt( classSummaries: SampleData[], sampleCode: string ) { + const formattedCodespec = codeSpec.replace(/`/g, '"').replace(/'/g, '"'); return ` # Role: You are an expert in Office JavaScript Add-ins and TypeScript, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. # Context: - You should give suggestions as an JSON object, and the output must be the JSON object and it will contain the following keys: + The input is + + \`\`\`text + ${formattedCodespec} + \`\`\` + + The output must be a JSON object and it will contain the following keys: - picked. value is a string array. - Beyond this JSON object, you should not add anything else to the output. Do not explain, do not provide additional context, do not add any other information to the output. # Your tasks: - Firstly, For the given user ask: - '${codeSpec}' - repeat and make sure you understand that. Pay attention, the user intent is right, but the spec may list incorrect API names or descriptions. You should focus on the user intent and ignore the incorrect API names or descriptions. - Second, For each strings listed below, they're descriptions of Office JavaScript API class, interface or enums. Read them carefully and think about how they could help on the task. - The last step, based on your understanding, deduce your think result to a list of Office JavaScript API class descriptions those picked from the list below. A sample code snippet may be offered below as reference, combine with the user's actual ask and the sample code, you should understand those class be used in similar scenario, then picked them up. You should pick strings from the candidates below, and put them into an array of string. Each item in the array has a priority, indicates how important this item in the task. For example if the task is about manipulate shape, then shape relevant class descriptions should have higher priority score. Order those array items in the descent directly by priority. If you don't find any relevant strings, you should return an empty array. For the array of string, it should be the value of the key 'picked' in the return object. + Understand the input, focus on the mentioned asks, and think about the coding approach under the context of Office JavaScript API. Then, pick some Office JavaScript API classes/enums/interfaces will be used in your coding approach, including class in the intermediate step (for example, the type of a intermediate result in the chaining invoke), put those picked classes/enums/interfaces into an array to return. Use the list of Office JavaScript classes below as your reference, but not limited to. Each class below contains a class name and the description of the class, those selected most likely related to your task. For the array of items, order them in the sequency will be used by the task. In general it will start from entry class, for example: + - In Excel, it could start from "Workbook", then follow with "Worksheet", "WorksheetCollection", etc. + - In Word, it could start with "Document", the follow with "Body", etc. + - In PowerPoint, it could start with "Presentation", then follow with "Slide", "SlideCollection", etc. + Return an empty list if no relevant strings are found. The list should be the value of the key 'picked' in the return object. # The candidate strings: - ${classSummaries.map((sampleData) => "- " + sampleData.definition).join("\n")} - - # Sample code snippet: - \`\`\`typescript - ${sampleCode} - \`\`\` + ${classSummaries + .map( + (sampleData) => + "- Name: '" + sampleData.definition + "', Description: '" + sampleData.description + "'." + ) + .join("\n")} # The format of output: Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not explain, do not provide additional context, do not add any other information to the output. - The example of output you must to follow: - { - "picked": ["Highest priority class", "normal priority class", "lowest priority class"] - } + For example, the candidate strings could be like:" - Name: 'Workbook', Description: 'Represents a workbook in Excel.'.", then the return object could be like: "{ 'picked': ['Workbook'] }". + `; +} + +export function getMostRelevantClassUsingNameOnlyPrompt( + codeSpec: string, + classSummaries: SampleData[], + sampleCode: string +) { + const formattedCodespec = codeSpec.replace(/`/g, '"').replace(/'/g, '"'); + return ` + # Role: + You are an expert in Office JavaScript Add-ins and TypeScript, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. + + # Context: + The input is: + + \`\`\`text + ${formattedCodespec} + \`\`\` + + The output must be a JSON object and it will contain the following keys: + - picked. value is a string array. + + Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not repeat the ask, do not ask questions, do not explain, do not provide additional context, do not add any other information to the output. + + # Your tasks: + Understand the input, focus on the mentioned asks, and think about the coding approach under the context of Office JavaScript API. Then, pick some Office JavaScript API classes/enums/interfaces will be used in your coding approach, including class in the intermediate step (for example, the type of a intermediate result in the chaining invoke), put those picked classes/enums/interfaces into an array to return. Use the list of Office JavaScript classes below as your reference, but not limited to. Each class below contains a class name, those selected most likely related to your task. For the array of items, order them in the sequency will be used by the task. In general it will start from entry class, for example: + - In Excel, it could start from "Workbook", then follow with "Worksheet", "WorksheetCollection", etc. + - In Word, it could start with "Document", the follow with "Body", etc. + - In PowerPoint, it could start with "Presentation", then follow with "Slide", "SlideCollection", etc. + + Return an empty list if no relevant strings are found. The list should be the value of the key 'picked' in the return object. + + # The list of Office JavaScript API: + ${classSummaries.map((sampleData) => "- Name: '" + sampleData.definition + "'.").join("\n")} + + # The format of output: + Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not repeat the ask, do not ask questions, do not explain, do not provide additional context, do not add any other information to the output. + For example, the candidate strings could be like:" - Name: 'Workbook'.", then the return object could be like: "{ 'picked': ['Workbook'] }". `; } @@ -716,44 +754,59 @@ export function getMostRelevantMethodPropertyPrompt( methodsOrPropertiesCandidatesByClassName.forEach((methodsOrPropertiesCandidates, className) => { tempClassDeclaration += ` class ${className} extends OfficeExtension.ClientObject { - ${methodsOrPropertiesCandidates.map((sampleData) => sampleData.codeSample).join("\n\n")} + ${methodsOrPropertiesCandidates.map((sampleData) => sampleData.codeSample).join("\n")} } -\n\n +\n `; }); + const formattedCodespec = codeSpec.replace(/`/g, '"').replace(/'/g, '"'); return ` # Role: You are an expert in Office JavaScript Add-ins, and you are familiar with scenario and the capabilities of Office JavaScript Add-ins. You need to offer the user a suggestion based on the user's ask. # Context: - You should give suggestions as an JSON object, and the output must be the JSON object and it will contain the following keys: + The input is: + + \`\`\`text + ${formattedCodespec} + \`\`\` + + The output must be a JSON object and it will contain the following keys: - picked. value is a string array. - - Beyond this JSON object, you should not add anything else to the output. Do not explain, do not provide additional context, do not add any other information to the output. + Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not repeat the ask, do not ask questions, do not explain, do not provide additional context, do not add any other information to the output. # Your tasks: - For the given description of user ask: - "${codeSpec.replace(/"/g, "'")}" - and list of Office JavaScript Add-ins API object class names: ' - ${classNamesList.join( - "," - )}', after understand the possible solution, you should able to pick some of the most relevant method and property declarations from the given list of method and property declaration below. Those picked method and property declarations should be used to complete the code it represent the user's ask. A sample code snippet may be offered below as reference, combine with the user's actual ask and the sample code, you should understand those methods and properties be used in similar scenario, then picked up declarations. You should pick the declaration from the below list, not from the given sample code. Then put the whole method or property declaration into an array of string. If you don't find any relevant declarations, you should return an empty array. For the array of string, it should be the value of the key 'picked' in the return object. - - # The method and property declarations: + Analyze each mentioned steps in the input, for any portion of those steps, and think what Office JavaScript Office API methods and properties should be used to fulfill those asks. A few Office JavaScript classes contains methods or properties declarations listed below as candidate for you. You should use them as your reference, pick those Office JavaScript API methods/properties will be used, including method or property in the intermediate step (for example, method or property in the chaining invoke), put those picked methods/properties into an array to return. Or return an empty list if no relevant strings are found. For each item in the array, it format should like "class: %name of the class%; %method or property declaration%". The list should be the value of the key 'picked' in the return object. + Pay attention to the section of "The format of output" below, and make sure you follow the format. + + # The list of Office JavaScript API: \`\`\`typescript ${tempClassDeclaration} \`\`\` - # Sample code: + # The format of output: + Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not repeat the ask, do not ask questions, do not explain, do not provide additional context, do not add any other information to the output. + For example if the given class declaration are like: + \`\`\`typescript - ${sampleCode} - \`\`\` + class NoteItem extends OfficeExtension.ClientObject { + context: RequestContext; + + readonly reference: Word.Range; + } - # The format of output: - Beyond the JSON object, You should not add anything else to the output. Do not add the markdown syntax around the JSON object. Do not explain, do not provide additional context, do not add any other information to the output. - The example of output you must to follow: + class NoteItemCollection extends OfficeExtension.ClientObject { + readonly items: Word.NoteItem[]; + + getFirst(): Word.NoteItem; + } + \`\`\` + + Then the return object could be like: + \`\`\`json { - "picked": ["getDataTable", "setData", "setPosition"] + "picked": ["class: NoteItem; readonly reference: Word.Range;", "class: NoteItemCollection; getFirst(): Word.NoteItem;"] } + \`\`\` `; } diff --git a/packages/vscode-extension/src/officeChat/utils.ts b/packages/vscode-extension/src/officeChat/utils.ts index ddeeafc071..eb782f4126 100644 --- a/packages/vscode-extension/src/officeChat/utils.ts +++ b/packages/vscode-extension/src/officeChat/utils.ts @@ -18,10 +18,14 @@ export async function purifyUserMessage( token: CancellationToken ): Promise { const userMessagePrompt = ` - Please help to rephrase the following meesage in a more accurate and professional way. Message: ${message} + Please act as a professional Office JavaScript add-in developer and expert office application user, to rephrase the following meesage in an accurate and professional manner. Message: ${message} `; const systemPrompt = ` - You should only return the rephrased message, without any explanation or additional information. + You should only return the rephrased message, without any explanation or additional information. + + There're some general terms has special meaning in the Microsoft Office or Office JavaScript add-in development, please make sure you're using the correct terms or keep it as it is. For example, "task pane" is preferred than "side panel" in Office JavaScript add-in developing, or keep "Annotation", "Comment", "Document", "Body", "Slide", "Range", "Note", etc. as they're refer to a feature in Office client. + + The rephrased message should be clear and concise for developer. `; const purifyUserMessage = [ new LanguageModelChatUserMessage(userMessagePrompt), diff --git a/packages/vscode-extension/test/officeChat/common/samples/sampleProvider.test.ts b/packages/vscode-extension/test/officeChat/common/samples/sampleProvider.test.ts index 5f9cd9c0f3..daccea8092 100644 --- a/packages/vscode-extension/test/officeChat/common/samples/sampleProvider.test.ts +++ b/packages/vscode-extension/test/officeChat/common/samples/sampleProvider.test.ts @@ -71,4 +71,258 @@ describe("SampleProvider", () => { expect(topKSamples).to.have.lengthOf(0); // Add more assertions based on what you expect the topKSamples to be }); + + it("no class available for given host sample codes LLM", async () => { + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "UnkownHost"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("token overload relevant to scenario sample codes LLM", async () => { + const sample = "a fake code sample"; + const scenario = ` + Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document. +To make your document look professionally produced, Word provides header, footer, cover page, and text box designs that complement each other. For example, you can add a matching cover page, header, and sidebar. Click Insert and then choose the elements you want from the different galleries. +Themes and styles also help keep your document coordinated. When you click Design and choose a new Theme, the pictures, charts, and SmartArt graphics change to match your new theme. When you apply styles, your headings change to match the new theme. +Save time in Word with new buttons that show up where you need them. To change the way a picture fits in your document, click it and a button for layout options appears next to it. When you work on a table, click where you want to add a row or a column, and then click the plus sign. +Reading is easier, too, in the new Reading view. You can collapse parts of the document and focus on the text you want. If you need to stop reading before you reach the end, Word remembers where you left off - even on another device. + `; + const host = "UnkownHost"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario.repeat(100), // repeat the scenario to make it longer + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("no class relevant to scenario sample codes LLM", async () => { + sandbox.stub(utils, "getCopilotResponseAsString").resolves('{"picked":[]}'); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "UnkownHost"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("token overload", async () => { + sandbox.stub(utils, "getCopilotResponseAsString").resolves('{"picked":[]}'); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(4000); + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("no methods or properties relevant to scenario sample codes LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub + .onCall(0) + .returns(Promise.resolve('{"picked":["Workbook", "Worksheet", "Range", "Chart", "Shape"]}')); + getCopilotResponseAsStringStub.onCall(1).returns(Promise.resolve('{"picked":[]}')); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("no class returned from LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub.onCall(0).returns(Promise.resolve('{"picked":[]}')); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(0); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("one methods or properties relevant to scenario sample codes LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub.onCall(0).returns(Promise.resolve('{"picked":["Shape"]}')); + getCopilotResponseAsStringStub + .onCall(1) + .returns( + Promise.resolve('{"picked":["class: Shape; readonly connectionSiteCount: number;"]}') + ); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(1); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("mutiple methods or properties relevant to scenario sample codes LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub.onCall(0).returns(Promise.resolve('{"picked":["Shape"]}')); + getCopilotResponseAsStringStub + .onCall(1) + .returns( + Promise.resolve( + '{"picked":["class: Shape; readonly connectionSiteCount: number; altTextDescription: string;"]}' + ) + ); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("return method without class to scenario sample codes LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub.onCall(0).returns(Promise.resolve('{"picked":["Shape"]}')); + getCopilotResponseAsStringStub + .onCall(1) + .returns(Promise.resolve('{"picked":["readonly connectionSiteCount: number;"]}')); + const sample = "a fake code sample"; + const scenario = "insert annotation into document"; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + expect(topKSamples).to.have.lengthOf(1); + // Add more assertions based on what you expect the topKSamples to be + }); + + it("giant class to scenario sample codes LLM", async () => { + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub + .onCall(0) + .returns( + Promise.resolve( + '{"picked":["Workbook", "Worksheet", "Range", "FunctionResult", "Functions"]}' + ) + ); + getCopilotResponseAsStringStub + .onCall(1) + .returns( + Promise.resolve( + '{"picked":["class: Workbook; readonly application: Excel.Application;", "class: Worksheet; readonly charts: Excel.ChartCollection;"]}' + ) + ); + getCopilotResponseAsStringStub + .onCall(2) + .returns( + Promise.resolve( + '{"picked":["class: Functions; arabic(text: string | Excel.Range | Excel.RangeReference | Excel.FunctionResult): FunctionResult;"]}' + ) + ); + getCopilotResponseAsStringStub + .onCall(3) + .returns(Promise.resolve('{"picked":["class: 3; method 1;"]}')); + getCopilotResponseAsStringStub + .onCall(4) + .returns(Promise.resolve('{"picked":["class: 3; method 2;"]}')); + getCopilotResponseAsStringStub + .onCall(5) + .returns(Promise.resolve('{"picked":["class: 3; method 3;"]}')); + getCopilotResponseAsStringStub + .onCall(6) + .returns(Promise.resolve('{"picked":["class: 3; method 4;"]}')); + getCopilotResponseAsStringStub + .onCall(7) + .returns(Promise.resolve('{"picked":["class: 3; method 5;"]}')); + getCopilotResponseAsStringStub + .onCall(8) + .returns(Promise.resolve('{"picked":["class: 3; method 6;"]}')); + getCopilotResponseAsStringStub + .onCall(9) + .returns(Promise.resolve('{"picked":["class: 3; method 7;"]}')); + + const sample = ""; + const scenario = + "To set up streaming custom functions with the Office JS API that fetch real-time data from the web at 10-second intervals, you should follow these steps: 1. Define a function in a JavaScript or Typescript file that fetches the data from the web. 2. Ensure this function is async and is continuously running with a call every 10 seconds. 3. In the custom functions metadata, register this function as a streaming function. 4. Test this function in Excel to confirm it behaves correctly."; + const host = "Excel"; + const topKSamples = await SampleProvider.getInstance().getMostRelevantDeclarationsUsingLLM( + null as any, + host, + scenario, + sample + ); + + expect(topKSamples).to.exist; + expect(topKSamples).to.be.an("map"); + }).timeout(10000); }); diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts index 65242ed18d..abd69747bf 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeGenerator.test.ts @@ -149,6 +149,36 @@ describe("codeGenerator", () => { chai.expect(result).to.equal(parserResult); }); + it("userAskPreScanningAsync provided json object, json detected: not custom function", async () => { + const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); + const codeGenerator = new CodeGenerator(); + + sandbox.stub(console, "log"); + sandbox.stub(console, "error"); + + const getCopilotResponseStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseStub.resolves( + JSON.stringify({ + host: "fakeHost", + shouldContinue: false, + customFunctions: false, + complexity: 1, + }) + ); + const jsonParseStub = sandbox.stub(JSON, "parse"); + const parserResult = { + host: "fakeHost", + shouldContinue: false, + customFunctions: false, + complexity: 1, + }; + jsonParseStub.returns(parserResult); + + const result = await codeGenerator.userAskPreScanningAsync(spec, fakeToken); + + chai.expect(result).to.equal(parserResult); + }); + it("userAskPreScanningAsync provided json with markdown syntax, json detected", async () => { const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); const codeGenerator = new CodeGenerator(); @@ -198,10 +228,10 @@ describe("codeGenerator", () => { const result = await codeGenerator.userAskBreakdownAsync( fakeToken, spec.appendix.complexity, - spec.appendix.isCustomFunction, + true, //isCustomFunction spec.appendix.host, spec.userInput, - "" + "Some code sample" ); chai.expect(result).to.equal(null); @@ -415,6 +445,60 @@ describe("codeGenerator", () => { chai.expect(result).to.exist; // Replace with more specific assertions }); + it("generateCode - Word", async () => { + const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); + const host = "Word"; + const codeSpec = "codeSpec"; + const isCustomFunctions = false; + const suggestedFunction = ["function1", "function2"]; + const fakeSampleCode = "fakeSampleCode"; + const codeGenerator = new CodeGenerator(); + sandbox.stub(console, "log"); + sandbox.stub(console, "debug"); + sandbox.stub(console, "error"); + const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); + getCopilotResponseAsStringStub.returns(Promise.resolve("```typescript\n// Some code\n```")); + const getMostRelevantDeclarationsUsingLLMStub = sandbox.stub( + SampleProvider.prototype, + "getMostRelevantDeclarationsUsingLLM" + ); + + const scenarioSamples = new Map(); + scenarioSamples.set( + "sample1", + new SampleData( + "Sample Name", + "https://docs.example.com", + "sample code", + "description", + "definition", + "usage" + ) + ); + getMostRelevantDeclarationsUsingLLMStub.returns(Promise.resolve(scenarioSamples)); + + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(100); + + // Act + const result = await codeGenerator.generateCode( + fakeToken, + host, + spec, + codeSpec, + isCustomFunctions, + suggestedFunction, + fakeSampleCode + ); + + // Assert + chai.expect(result).to.exist; // Replace with more specific assertions + }); + it("generateCode - Excel - invalid return", async () => { const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); const host = "Excel"; @@ -543,6 +627,150 @@ describe("codeGenerator", () => { chai.expect((await result).result).to.equal(ExecutionResultEnum.Failure); }); + it("Invoke Failure: Condition 2", async () => { + const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); + const codeGenerator = new CodeGenerator(); + sandbox.stub(console, "log"); + sandbox.stub(console, "debug"); + + spec.appendix.host = ""; + spec.appendix.complexity = 0; + spec.appendix.codeSample = ""; + spec.appendix.codeTaskBreakdown = []; + spec.appendix.codeExplanation = ""; + + sandbox.stub(codeGenerator, "userAskPreScanningAsync").resolves({ + host: "some host", + shouldContinue: true, + customFunctions: false, + complexity: 60, + }); + + sandbox.stub(codeGenerator, "userAskBreakdownAsync").resolves(null); + sandbox.stub(codeGenerator, "generateCode").resolves(null); + + const getMostRelevantDeclarationsUsingLLMStub = sandbox.stub( + SampleProvider.prototype, + "getTopKMostRelevantScenarioSampleCodesBM25" + ); + + const scenarioSamples = new Map(); + scenarioSamples.set( + "sample1", + new SampleData( + "Sample Name", + "https://docs.example.com", + "sample code", + "description", + "definition", + "usage" + ) + ); + getMostRelevantDeclarationsUsingLLMStub.returns(Promise.resolve(scenarioSamples)); + + const result = codeGenerator.invoke(model, fakeResponse, fakeToken, spec); + + chai.expect((await result).result).to.equal(ExecutionResultEnum.Failure); + }); + + it("Invoke Failure: Condition 3", async () => { + const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); + const codeGenerator = new CodeGenerator(); + sandbox.stub(console, "log"); + sandbox.stub(console, "debug"); + + spec.appendix.host = ""; + spec.appendix.complexity = 0; + spec.appendix.codeSample = ""; + spec.appendix.codeTaskBreakdown = []; + spec.appendix.codeExplanation = ""; + + sandbox.stub(codeGenerator, "userAskPreScanningAsync").resolves({ + host: "some host", + shouldContinue: true, + customFunctions: false, + complexity: 60, + }); + + sandbox.stub(codeGenerator, "userAskBreakdownAsync").resolves({ + spec: "", + funcs: ["some data"], + }); + sandbox.stub(codeGenerator, "generateCode").resolves(null); + + const getMostRelevantDeclarationsUsingLLMStub = sandbox.stub( + SampleProvider.prototype, + "getTopKMostRelevantScenarioSampleCodesBM25" + ); + + const scenarioSamples = new Map(); + scenarioSamples.set( + "sample1", + new SampleData( + "Sample Name", + "https://docs.example.com", + "sample code", + "description", + "definition", + "usage" + ) + ); + getMostRelevantDeclarationsUsingLLMStub.returns(Promise.resolve(scenarioSamples)); + + const result = codeGenerator.invoke(model, fakeResponse, fakeToken, spec); + + chai.expect((await result).result).to.equal(ExecutionResultEnum.Failure); + }); + + it("Invoke Failure: Condition 4", async () => { + const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); + const codeGenerator = new CodeGenerator(); + sandbox.stub(console, "log"); + sandbox.stub(console, "debug"); + + spec.appendix.host = ""; + spec.appendix.complexity = 0; + spec.appendix.codeSample = ""; + spec.appendix.codeTaskBreakdown = []; + spec.appendix.codeExplanation = ""; + + sandbox.stub(codeGenerator, "userAskPreScanningAsync").resolves({ + host: "some host", + shouldContinue: true, + customFunctions: false, + complexity: 60, + }); + + sandbox.stub(codeGenerator, "userAskBreakdownAsync").resolves({ + spec: "some spec", + funcs: [], + }); + sandbox.stub(codeGenerator, "generateCode").resolves(null); + + const getMostRelevantDeclarationsUsingLLMStub = sandbox.stub( + SampleProvider.prototype, + "getTopKMostRelevantScenarioSampleCodesBM25" + ); + + const scenarioSamples = new Map(); + scenarioSamples.set( + "sample1", + new SampleData( + "Sample Name", + "https://docs.example.com", + "sample code", + "description", + "definition", + "usage" + ) + ); + getMostRelevantDeclarationsUsingLLMStub.returns(Promise.resolve(scenarioSamples)); + + const result = codeGenerator.invoke(model, fakeResponse, fakeToken, spec); + + chai.expect((await result).result).to.equal(ExecutionResultEnum.Failure); + }); + it("Invoke Success", async () => { const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); const codeGenerator = new CodeGenerator(); @@ -600,7 +828,7 @@ describe("codeGenerator", () => { spec.appendix.telemetryData.measurements["CodeGenExecutionTimeInTotalSec"] = 1; spec.appendix.host = "Excel"; - spec.appendix.complexity = 50; + spec.appendix.complexity = 10; spec.appendix.shouldContinue = true; spec.appendix.codeSample = "sample code"; spec.appendix.codeTaskBreakdown = ["task1", "task2"]; diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts index fc4364968b..2741468456 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeIssueCorrector.test.ts @@ -79,7 +79,7 @@ describe("CodeIssueCorrector", () => { chai.assert.equal(codeIssueCorrector.capability, "Fix code issues"); }); - it("canInvoke returns true", () => { + it("canInvoke returns true", async () => { const corrector = new CodeIssueCorrector(); const spec = new Spec("Some user input"); spec.taskSummary = "Some task summary"; @@ -108,16 +108,31 @@ describe("CodeIssueCorrector", () => { shouldContinue: false, }; - const result = corrector.canInvoke(spec); + const result = await corrector.canInvoke(spec); chai.assert.isTrue(result); }); it("fixIssueAsync no error return codeSnippet", async () => { + const sampleCodeLong = + `Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document. + To make your document look professionally produced, Word provides header, footer, cover page, and text box designs that complement each other. For example, you can add a matching cover page, header, and sidebar. Click Insert and then choose the elements you want from the different galleries. + Themes and styles also help keep your document coordinated. When you click Design and choose a new Theme, the pictures, charts, and SmartArt graphics change to match your new theme. When you apply styles, your headings change to match the new theme. + Save time in Word with new buttons that show up where you need them. To change the way a picture fits in your document, click it and a button for layout options appears next to it. When you work on a table, click where you want to add a row or a column, and then click the plus sign. + Reading is easier, too, in the new Reading view. You can collapse parts of the document and focus on the text you want. If you need to stop reading before you reach the end, Word remembers where you left off - even on another device. + `.repeat(20); const corrector = new CodeIssueCorrector(); const fakeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { content: "some sample message", }; - + const fakeSampleCodeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { + content: sampleCodeLong, + }; + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(100); const result = await corrector.fixIssueAsync( { isCancellationRequested: false, @@ -133,17 +148,27 @@ describe("CodeIssueCorrector", () => { "additional info", // additionalInfo "copilot-gpt-3.5-turbo", // model fakeLanguageModelChatSystemMessage, - fakeLanguageModelChatSystemMessage + fakeSampleCodeLanguageModelChatSystemMessage ); chai.assert.equal(result, "original code snippet"); }); it("fixIssueAsync error with the LLM output and Excel host, isCustomFunctions false", async () => { + const sampleCodeLong = + `Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document. + To make your document look professionally produced, Word provides header, footer, cover page, and text box designs that complement each other. For example, you can add a matching cover page, header, and sidebar. Click Insert and then choose the elements you want from the different galleries. + Themes and styles also help keep your document coordinated. When you click Design and choose a new Theme, the pictures, charts, and SmartArt graphics change to match your new theme. When you apply styles, your headings change to match the new theme. + Save time in Word with new buttons that show up where you need them. To change the way a picture fits in your document, click it and a button for layout options appears next to it. When you work on a table, click where you want to add a row or a column, and then click the plus sign. + Reading is easier, too, in the new Reading view. You can collapse parts of the document and focus on the text you want. If you need to stop reading before you reach the end, Word remembers where you left off - even on another device. + `.repeat(20); const corrector = new CodeIssueCorrector(); const fakeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { content: "some sample message", }; + const fakeSampleCodeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { + content: sampleCodeLong, + }; const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); getCopilotResponseAsStringStub.returns( @@ -151,8 +176,13 @@ describe("CodeIssueCorrector", () => { ); sandbox.stub(console, "log"); sandbox.stub(console, "error"); - sandbox.stub(utils, "countMessagesTokens").returns(100); - sandbox.stub(utils, "countMessageTokens").returns(100); + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(100); + // sandbox.stub(utils, "countMessageTokens").returns(100); sandbox.stub(RegExp.prototype, "exec").returns(null); const result = await corrector.fixIssueAsync( @@ -170,18 +200,27 @@ describe("CodeIssueCorrector", () => { "additional info", // additionalInfo "copilot-gpt-3.5-turbo", // model fakeLanguageModelChatSystemMessage, - fakeLanguageModelChatSystemMessage + fakeSampleCodeLanguageModelChatSystemMessage ); chai.assert.equal(result, null); }); it("fixIssueAsync error with the LLM output and Excel host, isCustomFunctions true", async () => { + const sampleCodeLong = + `Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document. + To make your document look professionally produced, Word provides header, footer, cover page, and text box designs that complement each other. For example, you can add a matching cover page, header, and sidebar. Click Insert and then choose the elements you want from the different galleries. + Themes and styles also help keep your document coordinated. When you click Design and choose a new Theme, the pictures, charts, and SmartArt graphics change to match your new theme. When you apply styles, your headings change to match the new theme. + Save time in Word with new buttons that show up where you need them. To change the way a picture fits in your document, click it and a button for layout options appears next to it. When you work on a table, click where you want to add a row or a column, and then click the plus sign. + Reading is easier, too, in the new Reading view. You can collapse parts of the document and focus on the text you want. If you need to stop reading before you reach the end, Word remembers where you left off - even on another device. + `.repeat(20); const corrector = new CodeIssueCorrector(); - const fakeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { content: "some sample message", }; + const fakeSampleCodeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { + content: sampleCodeLong, + }; const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); getCopilotResponseAsStringStub.returns( @@ -189,8 +228,13 @@ describe("CodeIssueCorrector", () => { ); sandbox.stub(console, "log"); sandbox.stub(console, "error"); - sandbox.stub(utils, "countMessagesTokens").returns(100); - sandbox.stub(utils, "countMessageTokens").returns(100); + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(100); + // sandbox.stub(utils, "countMessageTokens").returns(100); sandbox.stub(RegExp.prototype, "exec").returns(null); const result = await corrector.fixIssueAsync( @@ -208,17 +252,27 @@ describe("CodeIssueCorrector", () => { "additional info", // additionalInfo "copilot-gpt-3.5-turbo", // model fakeLanguageModelChatSystemMessage, // sampleMessage - fakeLanguageModelChatSystemMessage + fakeSampleCodeLanguageModelChatSystemMessage ); chai.assert.equal(result, null); }); it("fixIssueAsync error with the LLM output and other host", async () => { + const sampleCodeLong = + `Video provides a powerful way to help you prove your point. When you click Online Video, you can paste in the embed code for the video you want to add. You can also type a keyword to search online for the video that best fits your document. + To make your document look professionally produced, Word provides header, footer, cover page, and text box designs that complement each other. For example, you can add a matching cover page, header, and sidebar. Click Insert and then choose the elements you want from the different galleries. + Themes and styles also help keep your document coordinated. When you click Design and choose a new Theme, the pictures, charts, and SmartArt graphics change to match your new theme. When you apply styles, your headings change to match the new theme. + Save time in Word with new buttons that show up where you need them. To change the way a picture fits in your document, click it and a button for layout options appears next to it. When you work on a table, click where you want to add a row or a column, and then click the plus sign. + Reading is easier, too, in the new Reading view. You can collapse parts of the document and focus on the text you want. If you need to stop reading before you reach the end, Word remembers where you left off - even on another device. + `.repeat(20); const corrector = new CodeIssueCorrector(); const fakeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { content: "some sample message", }; + const fakeSampleCodeLanguageModelChatSystemMessage: LanguageModelChatSystemMessage = { + content: sampleCodeLong, + }; const getCopilotResponseAsStringStub = sandbox.stub(utils, "getCopilotResponseAsString"); getCopilotResponseAsStringStub.returns( @@ -226,8 +280,13 @@ describe("CodeIssueCorrector", () => { ); sandbox.stub(console, "log"); sandbox.stub(console, "error"); - sandbox.stub(utils, "countMessagesTokens").returns(100); - sandbox.stub(utils, "countMessageTokens").returns(100); + sandbox + .stub(utils, "countMessagesTokens") + .onFirstCall() + .returns(4000) + .onSecondCall() + .returns(100); + // sandbox.stub(utils, "countMessageTokens").returns(100); sandbox.stub(RegExp.prototype, "exec").returns(null); const result = await corrector.fixIssueAsync( @@ -245,7 +304,7 @@ describe("CodeIssueCorrector", () => { "additional info", // additionalInfo "copilot-gpt-3.5-turbo", // model fakeLanguageModelChatSystemMessage, - fakeLanguageModelChatSystemMessage + fakeSampleCodeLanguageModelChatSystemMessage ); chai.assert.equal(result, null); @@ -338,6 +397,11 @@ describe("CodeIssueCorrector", () => { const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); spec.appendix.complexity = 10; + spec.appendix.codeSample = "some code sample"; + spec.appendix.apiDeclarationsReference.set( + "definition", + new SampleData("key1", "docLink", "sample", "description", "definition", "usage") + ); const result = await corrector.invoke(model, fakeResponse, fakeToken, spec); @@ -436,6 +500,11 @@ describe("CodeIssueCorrector", () => { const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); spec.appendix.complexity = 80; + spec.appendix.codeSample = "some code sample"; + spec.appendix.apiDeclarationsReference.set( + "definition", + new SampleData("key1", "docLink", "sample", "description", "definition", "usage") + ); sandbox.stub(corrector, "fixIssueAsync").returns(Promise.resolve(null)); const result = await corrector.invoke(model, fakeResponse, fakeToken, spec); @@ -486,12 +555,11 @@ describe("CodeIssueCorrector", () => { sandbox.stub(console, "debug"); const { spec, model, fakeResponse, fakeToken } = invokeParametersInit(); - spec.appendix.complexity = 80; const fixIssueStub = sandbox .stub(corrector, "fixIssueAsync") - .returns(Promise.resolve("some more code")); + .returns(Promise.resolve("some more code; await main(); ")); fixIssueStub.onCall(0).returns(Promise.resolve("less")); const detectorInstance = CodeIssueDetector.getInstance(); const detectIssuesStub = sandbox.stub(detectorInstance, "detectIssuesAsync"); diff --git a/packages/vscode-extension/test/officeChat/common/skills/codeIssueDetector.test.ts b/packages/vscode-extension/test/officeChat/common/skills/codeIssueDetector.test.ts index b47de669e9..3bbebd4859 100644 --- a/packages/vscode-extension/test/officeChat/common/skills/codeIssueDetector.test.ts +++ b/packages/vscode-extension/test/officeChat/common/skills/codeIssueDetector.test.ts @@ -677,6 +677,43 @@ describe("File: codeIssueDetector", () => { Reflect.set(detector, "program", backupProgram); }); + it("error treatment: Argument Count Mismatch 4", async () => { + const detector = CodeIssueDetector.getInstance(); + const backupProgram = Reflect.get(detector, "program"); + + sandbox.stub(ts, "getPreEmitDiagnostics").returns([ + { + file: { + parent: null, + text: "text test", + getStart: () => 0, + getEnd: () => 1, + getLineStarts: () => 1, + getLineEndOfPosition: (x: number) => x, + getLineAndCharacterOfPosition: () => ({ line: 1, character: 1 }), + }, + start: false, + } as any, + ]); + sandbox.stub(ts, "getPositionOfLineAndCharacter").returns(0); + sandbox.stub(ts, "flattenDiagnosticMessageText").returns("arguments, but got 1"); + Reflect.set(detector, "program", { + getTypeChecker: () => ({ + getSymbolAtLocation: () => ({ getDeclarations: () => [1, 2] }), + getSignatureFromDeclaration: () => ({ + parameters: [1, 2], + getDeclaration: () => ({ getText: () => "text" }), + }), + }), + }); + const isCallExpressionStub = sandbox.stub(ts, "isCallExpression"); + isCallExpressionStub.onCall(0).returns(false); + + const result = detector.getCompilationErrorsAsync("word", false, telemetryData); + chai.assert.isDefined(result); + Reflect.set(detector, "program", backupProgram); + }); + it("error treatment: Argument Type Mismatch", async () => { const detector = CodeIssueDetector.getInstance(); const backupProgram = Reflect.get(detector, "program");