diff --git a/ts/packages/actionGrammar/src/grammarCompiler.ts b/ts/packages/actionGrammar/src/grammarCompiler.ts index 3496bcc74..0bcc4769d 100644 --- a/ts/packages/actionGrammar/src/grammarCompiler.ts +++ b/ts/packages/actionGrammar/src/grammarCompiler.ts @@ -1,52 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { Rule, RuleDefinition, ValueNode } from "./grammarParser.js"; - -type StringPart = { - type: "string"; - - value: string[]; - - /* TODO: cache the regexp? - regexp?: RegExp; - regexpWithPendingWildcards?: RegExp; - */ -}; - -type VarStringPart = { - type: "wildcard"; - variable: string; - optional?: boolean | undefined; - - typeName: string; // Do we need this? -}; - -type VarNumberPart = { - type: "number"; - variable: string; - optional?: boolean | undefined; -}; - -type RulesPart = { - type: "rules"; - - rules: GrammarRule[]; - name?: string; // Do we need this? - - variable?: string; - optional?: boolean | undefined; -}; - -type GrammarPart = StringPart | VarStringPart | VarNumberPart | RulesPart; -export type GrammarRule = { - parts: GrammarPart[]; - value?: ValueNode | undefined; -}; - -export type Grammar = { - rules: GrammarRule[]; -}; +import { + Grammar, + GrammarPart, + GrammarRule, + StringPart, +} from "./grammarTypes.js"; +import { Rule, RuleDefinition } from "./grammarRuleParser.js"; type DefinitionMap = Map< string, diff --git a/ts/packages/actionGrammar/src/grammarDeserializer.ts b/ts/packages/actionGrammar/src/grammarDeserializer.ts new file mode 100644 index 000000000..bdce57217 --- /dev/null +++ b/ts/packages/actionGrammar/src/grammarDeserializer.ts @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + Grammar, + GrammarJson, + GrammarPart, + GrammarPartJson, + GrammarRule, + GrammarRuleJson, +} from "./grammarTypes.js"; + +export function grammarFromJson(json: GrammarJson): Grammar { + const start = json[0]; + const indexToRules: Map = new Map(); + function grammarRuleFromJson(r: GrammarRuleJson, json: GrammarJson) { + return { + parts: r.parts.map((p) => grammarPartFromJson(p, json)), + value: r.value, + }; + } + function grammarPartFromJson( + p: GrammarPartJson, + json: GrammarJson, + ): GrammarPart { + switch (p.type) { + case "string": + case "wildcard": + case "number": + return p; + case "rules": + let rules = indexToRules.get(p.index); + if (rules === undefined) { + rules = []; + indexToRules.set(p.index, rules); + for (const r of json[p.index]) { + rules.push(grammarRuleFromJson(r, json)); + } + } + return { + type: "rules", + name: p.name, + rules, + variable: p.variable, + optional: p.optional, + }; + } + } + + return { + rules: start.map((r) => grammarRuleFromJson(r, json)), + }; +} diff --git a/ts/packages/actionGrammar/src/grammarLoader.ts b/ts/packages/actionGrammar/src/grammarLoader.ts index 1482d4939..ff8010c2c 100644 --- a/ts/packages/actionGrammar/src/grammarLoader.ts +++ b/ts/packages/actionGrammar/src/grammarLoader.ts @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { compileGrammar, Grammar } from "./grammarCompiler.js"; -import { parseGrammar } from "./grammarParser.js"; +import { compileGrammar } from "./grammarCompiler.js"; +import { parseGrammarRules } from "./grammarRuleParser.js"; +import { Grammar } from "./grammarTypes.js"; -export function loadGrammar(fileName: string, content: string): Grammar { - const definitions = parseGrammar(fileName, content); +export function loadGrammarRules(fileName: string, content: string): Grammar { + const definitions = parseGrammarRules(fileName, content); const grammar = compileGrammar(definitions); return grammar; } diff --git a/ts/packages/actionGrammar/src/grammarMatcher.ts b/ts/packages/actionGrammar/src/grammarMatcher.ts index f408a7768..06b0aac9a 100644 --- a/ts/packages/actionGrammar/src/grammarMatcher.ts +++ b/ts/packages/actionGrammar/src/grammarMatcher.ts @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { ValueNode } from "./grammarParser.js"; +import { ValueNode } from "./grammarRuleParser.js"; import registerDebug from "debug"; // REVIEW: switch to RegExp.escape() when it becomes available. import escapeMatch from "regexp.escape"; -import { Grammar, GrammarRule } from "./grammarCompiler.js"; +import { Grammar, GrammarRule } from "./grammarTypes.js"; const debugMatch = registerDebug("typeagent:grammar:match"); @@ -31,6 +31,7 @@ type MatchedValue = type MatchedValueNode = { valueId: number; value: MatchedValue; + wildcard: boolean; prev: MatchedValueNode | undefined; }; @@ -69,22 +70,35 @@ type MatchState = { | undefined; }; -function getMatchedValue( - valueId: ValueIdNode, +function getMatchedValueNode( + valueId: number, values: MatchedValueNode | undefined, -): MatchedValue | undefined { +): MatchedValueNode { let v: MatchedValueNode | undefined = values; - while (v !== undefined && v.valueId !== valueId.valueId) { + while (v !== undefined && v.valueId !== valueId) { v = v.prev; } - return v?.value; + if (v === undefined) { + throw new Error(`Internal error: Missing value for ${valueId}`); + } + return v; } +type GrammarMatchStat = { + matchedValueCount: number; + wildcardCharCount: number; + entityWildcardPropertyNames: string[]; +}; +export type GrammarMatchResult = GrammarMatchStat & { + match: unknown; +}; + function createValue( + stat: GrammarMatchStat, node: ValueNode | undefined, valueIds: ValueIdNode | undefined, values: MatchedValueNode | undefined, -): any { +): unknown { if (node === undefined) { if (valueIds === undefined) { throw new Error("Internal error: default matched values"); @@ -94,9 +108,10 @@ function createValue( `Internal error: No value definitions for multiple values`, ); } - const value = getMatchedValue(valueIds, values); + const valueNode = getMatchedValueNode(valueIds.valueId, values); + const value = valueNode.value; if (typeof value === "object") { - return createValue(value.node, value.valueIds, values); + return createValue(stat, value.node, value.valueIds, values); } return value; } @@ -108,14 +123,14 @@ function createValue( const obj: Record = {}; for (const [k, v] of Object.entries(node.value)) { - obj[k] = createValue(v, valueIds, values); + obj[k] = createValue(stat, v, valueIds, values); } return obj; } case "array": { const arr: any[] = []; for (const v of node.value) { - arr.push(createValue(v, valueIds, values)); + arr.push(createValue(stat, v, valueIds, values)); } return arr; } @@ -129,10 +144,26 @@ function createValue( `Internal error: No value for variable '${node.name}. Values: ${JSON.stringify(valueIds)}'`, ); } - const value = getMatchedValue(v, values); + const valueNode = getMatchedValueNode(v.valueId, values); + const value = valueNode.value; if (typeof value === "object") { - return createValue(value.node, value.valueIds, values); + return createValue(stat, value.node, value.valueIds, values); + } + + // undefined means optional, don't count + if (value !== undefined) { + stat.matchedValueCount++; } + + if (valueNode.wildcard) { + if (typeof value !== "string") { + throw new Error( + `Internal error: Wildcard has non-string value for variable '${node.name}'`, + ); + } + stat.wildcardCharCount += value.length; + } + return value; } } @@ -162,14 +193,14 @@ function createCaptureWildcardState( newIndex: number, ) { const { start: wildcardStart, valueId } = state.pendingWildcard!; - const wildcard = captureWildcard(request, wildcardStart, wildcardEnd); - if (wildcard === undefined) { + const wildcardStr = captureWildcard(request, wildcardStart, wildcardEnd); + if (wildcardStr === undefined) { return undefined; } const newState = { ...state }; newState.index = newIndex; newState.pendingWildcard = undefined; - addValueWithId(newState, valueId, wildcard); + addValueWithId(newState, valueId, wildcardStr, true); return newState; } @@ -183,10 +214,12 @@ function addValueWithId( state: MatchState, valueId: number, matchedValue: MatchedValue, + wildcard: boolean, ) { state.values = { valueId, value: matchedValue, + wildcard, prev: state.values, }; } @@ -197,13 +230,13 @@ function addValue( matchedValue: MatchedValue, ) { const valueId = addValueId(state, name); - addValueWithId(state, valueId, matchedValue); + addValueWithId(state, valueId, matchedValue, false); } function finalizeRule( state: MatchState, request: string, - results: any[], + results: GrammarMatchResult[], pending: MatchState[], ) { const nested = state.nested; @@ -245,7 +278,7 @@ function finalizeRule( return; } state.index = request.length; - addValueWithId(state, state.pendingWildcard.valueId, value); + addValueWithId(state, state.pendingWildcard.valueId, value, true); } if (state.index < request.length) { // Detect trailing separators @@ -267,11 +300,23 @@ function finalizeRule( debugMatch( `Matched at end of input. Matched ids: ${JSON.stringify(state.valueIds)}, values: ${JSON.stringify(state.values)}'`, ); - results.push(createValue(state.rule.value, state.valueIds, state.values)); + + const matchResult: GrammarMatchResult = { + match: undefined, + matchedValueCount: 0, + wildcardCharCount: 0, + entityWildcardPropertyNames: [], + }; + matchResult.match = createValue( + matchResult, + state.rule.value, + state.valueIds, + state.values, + ); + results.push(matchResult); } -type MatchResult = any; -function matchRules(grammar: Grammar, request: string): MatchResult[] { +function matchRules(grammar: Grammar, request: string): GrammarMatchResult[] { const pending: MatchState[] = grammar.rules.map((r, i) => ({ name: `[${i}]`, rule: r, @@ -279,7 +324,7 @@ function matchRules(grammar: Grammar, request: string): MatchResult[] { index: 0, nextValueId: 0, })); - const results: MatchResult[] = []; + const results: GrammarMatchResult[] = []; while (pending.length > 0) { const state = pending.shift()!; const { rule, partIndex } = state; diff --git a/ts/packages/actionGrammar/src/grammarParser.ts b/ts/packages/actionGrammar/src/grammarRuleParser.ts similarity index 99% rename from ts/packages/actionGrammar/src/grammarParser.ts rename to ts/packages/actionGrammar/src/grammarRuleParser.ts index 522bbbe2a..e1162d3f3 100644 --- a/ts/packages/actionGrammar/src/grammarParser.ts +++ b/ts/packages/actionGrammar/src/grammarRuleParser.ts @@ -47,11 +47,11 @@ const debugParse = registerDebug("typeagent:grammar:parse"); * ::= "//" [^\n]* "\n" * ::= "/*" .* "*\/" */ -export function parseGrammar( +export function parseGrammarRules( fileName: string, content: string, ): RuleDefinition[] { - const parser = new CacheGrammarParser(fileName, content); + const parser = new GrammarRuleParser(fileName, content); const definitions = parser.parse(); debugParse(JSON.stringify(definitions, undefined, 2)); return definitions; @@ -147,7 +147,7 @@ export function isExpressionSpecialChar(char: string) { return expressionsSpecialChar.includes(char); } -class CacheGrammarParser { +class GrammarRuleParser { private curr: number = 0; constructor( private readonly fileName: string, diff --git a/ts/packages/actionGrammar/src/grammarWriter.ts b/ts/packages/actionGrammar/src/grammarRuleWriter.ts similarity index 97% rename from ts/packages/actionGrammar/src/grammarWriter.ts rename to ts/packages/actionGrammar/src/grammarRuleWriter.ts index b6b2760b2..996ba10a7 100644 --- a/ts/packages/actionGrammar/src/grammarWriter.ts +++ b/ts/packages/actionGrammar/src/grammarRuleWriter.ts @@ -8,9 +8,9 @@ import { Rule, RuleDefinition, ValueNode, -} from "./grammarParser.js"; +} from "./grammarRuleParser.js"; -export function writeGrammar(grammar: RuleDefinition[]): string { +export function writeGrammarRules(grammar: RuleDefinition[]): string { const result: string[] = []; for (const def of grammar) { writeRuleDefinition(result, def); diff --git a/ts/packages/actionGrammar/src/grammarSerializer.ts b/ts/packages/actionGrammar/src/grammarSerializer.ts new file mode 100644 index 000000000..6ebb061d5 --- /dev/null +++ b/ts/packages/actionGrammar/src/grammarSerializer.ts @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + Grammar, + GrammarJson, + GrammarPart, + GrammarPartJson, + GrammarRule, + GrammarRuleJson, + GrammarRules, +} from "./grammarTypes.js"; + +export function grammarToJson(grammar: Grammar): GrammarJson { + const json: GrammarRules[] = []; + const rulesToIndex: Map = new Map(); + let nextIndex = 1; + + function grammarPartToJson(p: GrammarPart): GrammarPartJson { + switch (p.type) { + case "string": + case "wildcard": + case "number": + return p; + case "rules": { + let index = rulesToIndex.get(p.rules); + if (index === undefined) { + index = nextIndex++; + rulesToIndex.set(p.rules, index); + json[index] = p.rules.map(grammarRuleToJson); + } + + return { + name: p.name, + type: "rules", + index, + variable: p.variable, + optional: p.optional, + }; + } + } + } + + function grammarRuleToJson(r: GrammarRule): GrammarRuleJson { + return { + parts: r.parts.map(grammarPartToJson), + value: r.value, + }; + } + + rulesToIndex.set(grammar.rules, 0); + json[0] = grammar.rules.map(grammarRuleToJson); + return json; +} diff --git a/ts/packages/actionGrammar/src/grammarTypes.ts b/ts/packages/actionGrammar/src/grammarTypes.ts new file mode 100644 index 000000000..f93d46809 --- /dev/null +++ b/ts/packages/actionGrammar/src/grammarTypes.ts @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ValueNode } from "./grammarRuleParser.js"; + +/** + * In memory types + */ +export type StringPart = { + type: "string"; + + value: string[]; + + /* TODO: cache the regexp? + regexp?: RegExp; + regexpWithPendingWildcards?: RegExp; + */ +}; + +export type VarStringPart = { + type: "wildcard"; + variable: string; + optional?: boolean | undefined; + + typeName: string; // Not needed at runtime? +}; + +export type VarNumberPart = { + type: "number"; + variable: string; + optional?: boolean | undefined; +}; + +export type RulesPart = { + type: "rules"; + + rules: GrammarRule[]; + name?: string | undefined; // For debugging + + variable?: string | undefined; + optional?: boolean | undefined; +}; + +export type GrammarPart = + | StringPart + | VarStringPart + | VarNumberPart + | RulesPart; +export type GrammarRule = { + parts: GrammarPart[]; + value?: ValueNode | undefined; +}; + +export type Grammar = { + rules: GrammarRule[]; +}; + +/** + * Serialized types + */ +export type StringPartJson = { + type: "string"; + value: string[]; +}; + +export type VarStringPartJson = { + type: "wildcard"; + variable: string; + typeName: string; +}; + +export type VarNumberPartJson = { + type: "number"; + variable: string; +}; + +export type RulePartJson = { + type: "rules"; + name?: string | undefined; + index: number; + variable?: string | undefined; + optional?: boolean | undefined; +}; + +export type GrammarPartJson = + | StringPartJson + | VarStringPartJson + | VarNumberPartJson + | RulePartJson; + +export type GrammarRuleJson = { + parts: GrammarPartJson[]; + value?: ValueNode | undefined; +}; +export type GrammarRules = GrammarRuleJson[]; +export type GrammarJson = GrammarRules[]; diff --git a/ts/packages/actionGrammar/src/index.ts b/ts/packages/actionGrammar/src/index.ts index 60dd3bcd2..899e87e28 100644 --- a/ts/packages/actionGrammar/src/index.ts +++ b/ts/packages/actionGrammar/src/index.ts @@ -1,5 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -export { loadGrammar } from "./grammarLoader.js"; -export { matchGrammar } from "./grammarMatcher.js"; +export type { GrammarJson, Grammar } from "./grammarTypes.js"; +export { grammarFromJson } from "./grammarDeserializer.js"; +export { grammarToJson } from "./grammarSerializer.js"; +export { loadGrammarRules } from "./grammarLoader.js"; +export { matchGrammar, GrammarMatchResult } from "./grammarMatcher.js"; diff --git a/ts/packages/actionGrammar/src/indexRules.ts b/ts/packages/actionGrammar/src/indexRules.ts index 89aeed72f..e4568aad9 100644 --- a/ts/packages/actionGrammar/src/indexRules.ts +++ b/ts/packages/actionGrammar/src/indexRules.ts @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -export { ValueNode, Expr, Rule, RuleDefinition } from "./grammarParser.js"; -export { writeGrammar } from "./grammarWriter.js"; +export { ValueNode, Expr, Rule, RuleDefinition } from "./grammarRuleParser.js"; +export { writeGrammarRules } from "./grammarRuleWriter.js"; diff --git a/ts/packages/actionGrammar/test/grammarMatcher.spec.ts b/ts/packages/actionGrammar/test/grammarMatcher.spec.ts index 25c4db6db..c60465eef 100644 --- a/ts/packages/actionGrammar/test/grammarMatcher.spec.ts +++ b/ts/packages/actionGrammar/test/grammarMatcher.spec.ts @@ -1,10 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { loadGrammar } from "../src/grammarLoader.js"; +import { loadGrammarRules } from "../src/grammarLoader.js"; import { matchGrammar } from "../src/grammarMatcher.js"; +import { Grammar } from "../src/grammarTypes.js"; import { escapedSpaces, spaces } from "./testUtils.js"; +function testMatchGrammar(grammar: Grammar, request: string) { + return matchGrammar(grammar, request)?.map((m) => m.match); +} describe("Grammar Matcher", () => { describe("Basic Matched Values", () => { const values = [ @@ -19,196 +23,200 @@ describe("Grammar Matcher", () => { ]; it.each(values)("matched value - '%j'", (v) => { const g = `@ = hello world -> ${JSON.stringify(v)}`; - const grammar = loadGrammar("test.grammar", g); - const match = matchGrammar(grammar, "hello world"); + const grammar = loadGrammarRules("test.grammar", g); + const match = testMatchGrammar(grammar, "hello world"); expect(match).toStrictEqual([v]); }); }); describe("Basic Match", () => { const g = `@ = hello world -> true`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); it("space prefix", () => { - expect(matchGrammar(grammar, `${spaces}hello world`)).toStrictEqual( - [true], - ); + expect( + testMatchGrammar(grammar, `${spaces}hello world`), + ).toStrictEqual([true]); }); it("space infix", () => { - expect(matchGrammar(grammar, `hello${spaces}world`)).toStrictEqual([ - true, - ]); + expect( + testMatchGrammar(grammar, `hello${spaces}world`), + ).toStrictEqual([true]); }); it("space suffix", () => { - expect(matchGrammar(grammar, `hello world${spaces}`)).toStrictEqual( - [true], - ); + expect( + testMatchGrammar(grammar, `hello world${spaces}`), + ).toStrictEqual([true]); }); it("ignore case", () => { - expect(matchGrammar(grammar, `HELLO WORLD`)).toStrictEqual([true]); + expect(testMatchGrammar(grammar, `HELLO WORLD`)).toStrictEqual([ + true, + ]); }); }); describe("Escaped Match", () => { it("escaped space prefix", () => { const g = `@ = ${escapedSpaces}hello world -> true`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, `${spaces}hello world`)).toStrictEqual( - [true], - ); + const grammar = loadGrammarRules("test.grammar", g); + expect( + testMatchGrammar(grammar, `${spaces}hello world`), + ).toStrictEqual([true]); }); it("escaped space prefix - alt space", () => { const g = `@ = ${escapedSpaces}hello world -> true`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, `${spaces}hello\tworld`), + testMatchGrammar(grammar, `${spaces}hello\tworld`), ).toStrictEqual([true]); }); it("escaped space prefix - extra space", () => { const g = `@ = ${escapedSpaces}hello world -> true`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, ` ${spaces}hello \nworld`), + testMatchGrammar(grammar, ` ${spaces}hello \nworld`), ).toStrictEqual([true]); }); it("escaped space - not match", () => { const g = `@ = ${escapedSpaces}hello world -> true`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, `${spaces} hello world`), + testMatchGrammar(grammar, `${spaces} hello world`), ).toStrictEqual([]); }); it("escaped space infix", () => { const g = `@ = hello${escapedSpaces} world -> true`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, `hello${spaces} world`)).toStrictEqual( - [true], - ); + const grammar = loadGrammarRules("test.grammar", g); + expect( + testMatchGrammar(grammar, `hello${spaces} world`), + ).toStrictEqual([true]); }); it("escaped space infix - extra space", () => { const g = `@ = hello${escapedSpaces} world -> true`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, `hello${spaces} \r\vworld`), + testMatchGrammar(grammar, `hello${spaces} \r\vworld`), ).toStrictEqual([true]); }); it("escaped space infix - not match", () => { const g = `@ = hello${escapedSpaces} world -> true`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, `hello${spaces}world`)).toStrictEqual( - [], - ); + const grammar = loadGrammarRules("test.grammar", g); + expect( + testMatchGrammar(grammar, `hello${spaces}world`), + ).toStrictEqual([]); expect( - matchGrammar(grammar, `hello ${spaces} world`), + testMatchGrammar(grammar, `hello ${spaces} world`), ).toStrictEqual([]); }); it("escaped space infix - no space", () => { const g = `@ = hello${escapedSpaces}world -> true`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, `hello${spaces}world`)).toStrictEqual([ - true, - ]); + const grammar = loadGrammarRules("test.grammar", g); + expect( + testMatchGrammar(grammar, `hello${spaces}world`), + ).toStrictEqual([true]); expect( - matchGrammar(grammar, `hello ${spaces} world`), + testMatchGrammar(grammar, `hello ${spaces} world`), ).toStrictEqual([]); }); }); describe("Variable Match", () => { it("simple variable - explicit string", () => { const g = `@ = $(x:string) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "value")).toStrictEqual(["value"]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "value")).toStrictEqual(["value"]); }); it("simple variable - explicit type name", () => { const g = `@ = $(x:TrackName) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "value")).toStrictEqual(["value"]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "value")).toStrictEqual(["value"]); }); it("simple variable - implicit string", () => { const g = `@ = $(x) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "value")).toStrictEqual(["value"]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "value")).toStrictEqual(["value"]); }); it("simple variable - simple integer", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "1234")).toStrictEqual([1234]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "1234")).toStrictEqual([1234]); }); it("simple variable - minus integer", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "-1234")).toStrictEqual([-1234]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "-1234")).toStrictEqual([-1234]); }); it("simple variable - plus integer", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "+1234")).toStrictEqual([1234]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "+1234")).toStrictEqual([1234]); }); it("simple variable - octal", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "0o123")).toStrictEqual([0o123]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "0o123")).toStrictEqual([0o123]); }); it("simple variable - binary", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "0b0101")).toStrictEqual([0b101]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "0b0101")).toStrictEqual([0b101]); }); it("simple variable - hex", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "0x123")).toStrictEqual([0x123]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "0x123")).toStrictEqual([0x123]); }); it("simple variable - float", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "10.123")).toStrictEqual([10.123]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "10.123")).toStrictEqual([10.123]); }); it("simple variable - negative float", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "-02120.123")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "-02120.123")).toStrictEqual([ -2120.123, ]); }); it("simple variable - float with exponent", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "45.678e-9")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "45.678e-9")).toStrictEqual([ 45.678e-9, ]); }); it("simple variable - optional", () => { const g = `@ = hello $(x:number)? -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "hello")).toStrictEqual([undefined]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "hello")).toStrictEqual([ + undefined, + ]); }); it("space around variable - string", () => { const g = `@ = hello $(x) world -> $(x)`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, `hello${spaces}value${spaces}world`), + testMatchGrammar(grammar, `hello${spaces}value${spaces}world`), ).toStrictEqual(["value"]); }); it("space around variable - number", () => { const g = `@ = hello $(x:number) world -> $(x)`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, `hello${spaces}123${spaces}world`), + testMatchGrammar(grammar, `hello${spaces}123${spaces}world`), ).toStrictEqual([123]); }); it("no space around variable - number and string not separated", () => { const g = `@ = $(x:number) $(y: string)-> { n: $(x), s: $(y) }`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "1234b")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "1234b")).toStrictEqual([ { n: 1234, s: "b" }, ]); }); it("no space around variable - number and term not separated", () => { const g = `@ = $(x:number)\\-$(y:number)pm -> { a: $(x), b: $(y) }`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "1-2pm")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "1-2pm")).toStrictEqual([ { a: 1, b: 2 }, ]); }); @@ -216,8 +224,8 @@ describe("Grammar Matcher", () => { const g = ` @ = $(x:number) -> $(x) @ = $(x) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "13.348")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "13.348")).toStrictEqual([ 13.348, "13.348", ]); @@ -228,8 +236,8 @@ describe("Grammar Matcher", () => { @ = hello -> "hello" @ = world -> "world" `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "hello world")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "hello world")).toStrictEqual([ { hello: "hello", world: "world" }, ]); }); @@ -239,8 +247,8 @@ describe("Grammar Matcher", () => { @ = hello -> "first" @ = hello -> "second" `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "hello")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "hello")).toStrictEqual([ "first", "second", ]); @@ -251,8 +259,8 @@ describe("Grammar Matcher", () => { @ = hello -> "first" @ = hello -> "second" `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "a hello world")).toStrictEqual([ + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "a hello world")).toStrictEqual([ "first", "second", ]); @@ -262,9 +270,9 @@ describe("Grammar Matcher", () => { const g = ` @ = hello $(x) world `; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, "hello this is a test world"), + testMatchGrammar(grammar, "hello this is a test world"), ).toStrictEqual(["this is a test"]); }); it("wildcard at end of nested rule", () => { @@ -272,9 +280,9 @@ describe("Grammar Matcher", () => { @ = world @ = hello $(x) `; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, "hello this is a test world"), + testMatchGrammar(grammar, "hello this is a test world"), ).toStrictEqual(["this is a test"]); }); @@ -283,17 +291,17 @@ describe("Grammar Matcher", () => { @ = hello $(x) @ = world `; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, "hello this is a test world"), + testMatchGrammar(grammar, "hello this is a test world"), ).toStrictEqual(["this is a test"]); }); it("wildcard alternates", () => { const g = `@ = $(x) by $(y) -> [$(x), $(y)]`; - const grammar = loadGrammar("test.grammar", g); + const grammar = loadGrammarRules("test.grammar", g); expect( - matchGrammar(grammar, "song by the sea by Bach"), + testMatchGrammar(grammar, "song by the sea by Bach"), ).toStrictEqual([ ["song", "the sea by Bach"], ["song by the sea", "Bach"], @@ -305,8 +313,8 @@ describe("Grammar Matcher", () => { const g = ` @ = hello world -> true `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "helloworld")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "helloworld")).toStrictEqual([]); }); it("sub-string expr not separated", () => { const g = ` @@ -314,47 +322,49 @@ describe("Grammar Matcher", () => { @ = hello @ = world `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "helloworld")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "helloworld")).toStrictEqual([]); }); it("trailing text", () => { const g = ` @ = hello world -> true `; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "hello world more")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "hello world more")).toStrictEqual( + [], + ); }); it("number variable - minus octal", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "-0o123")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "-0o123")).toStrictEqual([]); }); it("number variable - plus octal", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "+0o123")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "+0o123")).toStrictEqual([]); }); it("number variable - minus binary", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "-0b101")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "-0b101")).toStrictEqual([]); }); it("number variable - plus binary", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "+0b0101")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "+0b0101")).toStrictEqual([]); }); it("number variable - minus octal", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "-0x123")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "-0x123")).toStrictEqual([]); }); it("number variable - plus octal", () => { const g = `@ = $(x:number) -> $(x)`; - const grammar = loadGrammar("test.grammar", g); - expect(matchGrammar(grammar, "+0x123")).toStrictEqual([]); + const grammar = loadGrammarRules("test.grammar", g); + expect(testMatchGrammar(grammar, "+0x123")).toStrictEqual([]); }); }); }); diff --git a/ts/packages/actionGrammar/test/grammarParser.spec.ts b/ts/packages/actionGrammar/test/grammarRuleParser.spec.ts similarity index 87% rename from ts/packages/actionGrammar/test/grammarParser.spec.ts rename to ts/packages/actionGrammar/test/grammarRuleParser.spec.ts index 2b4196f58..bb6882b82 100644 --- a/ts/packages/actionGrammar/test/grammarParser.spec.ts +++ b/ts/packages/actionGrammar/test/grammarRuleParser.spec.ts @@ -1,14 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { parseGrammar } from "../src/grammarParser.js"; +import { parseGrammarRules } from "../src/grammarRuleParser.js"; import { escapedSpaces, spaces } from "./testUtils.js"; -describe("Grammar Parser", () => { +describe("Grammar Rule Parser", () => { describe("Basic Rule Definitions", () => { it("a simple rule with string expression", () => { const grammar = "@ = hello world"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toEqual([ { @@ -29,7 +29,7 @@ describe("Grammar Parser", () => { it("a rule with multiple alternatives", () => { const grammar = "@ = hello | hi | hey"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toHaveLength(1); expect(result[0].name).toBe("greeting"); @@ -50,7 +50,7 @@ describe("Grammar Parser", () => { it("a rule with value mapping", () => { const grammar = '@ = hello -> "greeting"'; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toHaveLength(1); expect(result[0].rules[0].value).toEqual({ @@ -64,7 +64,7 @@ describe("Grammar Parser", () => { @ = hello @ = goodbye `; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toHaveLength(2); expect(result[0].name).toBe("greeting"); @@ -73,7 +73,7 @@ describe("Grammar Parser", () => { it("rule with rule reference", () => { const grammar = "@ = world"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions).toHaveLength(2); expect(result[0].rules[0].expressions[0]).toEqual({ @@ -90,7 +90,7 @@ describe("Grammar Parser", () => { describe("Expression Parsing", () => { it("variable expressions with default type", () => { const grammar = "@ = $(name)"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "variable", @@ -102,7 +102,7 @@ describe("Grammar Parser", () => { it("variable expressions with specified type", () => { const grammar = "@ = $(count:number)"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "variable", @@ -114,7 +114,7 @@ describe("Grammar Parser", () => { it("variable expressions with rule reference", () => { const grammar = "@ = $(item:)"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "variable", @@ -126,7 +126,7 @@ describe("Grammar Parser", () => { it("variable expressions - optional", () => { const grammar = "@ = $(item:)?"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "variable", @@ -139,7 +139,7 @@ describe("Grammar Parser", () => { it("group expressions", () => { const grammar = "@ = (hello | hi) world"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions).toHaveLength(2); expect(result[0].rules[0].expressions[0]).toEqual({ @@ -159,7 +159,7 @@ describe("Grammar Parser", () => { it("optional group expressions", () => { const grammar = "@ = (please)? help"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "rules", @@ -175,7 +175,7 @@ describe("Grammar Parser", () => { it("complex expressions with multiple components", () => { const grammar = "@ = $(action) the $(adverb:string)"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions).toHaveLength(4); expect(result[0].rules[0].expressions[0].type).toBe("variable"); @@ -191,7 +191,7 @@ describe("Grammar Parser", () => { it("should handle escaped characters in string expressions", () => { const grammar = "@ = hello\\0world"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "string", @@ -205,8 +205,8 @@ describe("Grammar Parser", () => { const grammar1 = "@ = test -> true"; const grammar2 = "@ = test -> false"; - const result1 = parseGrammar("test.grammar", grammar1); - const result2 = parseGrammar("test.grammar", grammar2); + const result1 = parseGrammarRules("test.grammar", grammar1); + const result2 = parseGrammarRules("test.grammar", grammar2); expect(result1[0].rules[0].value).toEqual({ type: "literal", @@ -220,7 +220,7 @@ describe("Grammar Parser", () => { it("float literal values", () => { const grammar = "@ = test -> 42.5"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -230,7 +230,7 @@ describe("Grammar Parser", () => { it("integer literal values", () => { const grammar = "@ = test -> 12"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -240,7 +240,7 @@ describe("Grammar Parser", () => { it("integer hex literal values", () => { const grammar = "@ = test -> 0xC"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -250,7 +250,7 @@ describe("Grammar Parser", () => { it("integer oct literal values", () => { const grammar = "@ = test -> 0o14"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -260,7 +260,7 @@ describe("Grammar Parser", () => { it("integer binary literal values", () => { const grammar = "@ = test -> 0b1100"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -272,8 +272,8 @@ describe("Grammar Parser", () => { const grammar1 = '@ = test -> "hello world"'; const grammar2 = "@ = test -> 'hello world'"; - const result1 = parseGrammar("test.grammar", grammar1); - const result2 = parseGrammar("test.grammar", grammar2); + const result1 = parseGrammarRules("test.grammar", grammar1); + const result2 = parseGrammarRules("test.grammar", grammar2); expect(result1[0].rules[0].value).toEqual({ type: "literal", @@ -287,7 +287,7 @@ describe("Grammar Parser", () => { it("string values with escape sequences", () => { const grammar = '@ = test -> "hello\\tworld\\n"'; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "literal", @@ -297,7 +297,7 @@ describe("Grammar Parser", () => { it("array values", () => { const grammar = '@ = test -> [1, "hello", true]'; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "array", @@ -311,7 +311,7 @@ describe("Grammar Parser", () => { it("empty array values", () => { const grammar = "@ = test -> []"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "array", @@ -321,7 +321,7 @@ describe("Grammar Parser", () => { it("object values", () => { const grammar = '@ = test -> {type: "greeting", count: 1}'; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -334,7 +334,7 @@ describe("Grammar Parser", () => { it("empty object values", () => { const grammar = "@ = test -> {}"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -345,7 +345,7 @@ describe("Grammar Parser", () => { it("object values with single quote properties", () => { const grammar = "@ = test -> {'type': \"greeting\", 'count': 1}"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -359,7 +359,7 @@ describe("Grammar Parser", () => { it("object values with double quote properties", () => { const grammar = '@ = test -> {"type": "greeting", "count": 1}'; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -372,7 +372,7 @@ describe("Grammar Parser", () => { it("variable reference values", () => { const grammar = "@ = $(name) -> $(name)"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "variable", @@ -383,7 +383,7 @@ describe("Grammar Parser", () => { it("nested object and array values", () => { const grammar = "@ = test -> {items: [1, 2], meta: {count: 2}}"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -420,7 +420,7 @@ describe("Grammar Parser", () => { } `; - const result = parseGrammar("nested.grammar", grammar); + const result = parseGrammarRules("nested.grammar", grammar); expect(result).toEqual([ { @@ -559,7 +559,7 @@ describe("Grammar Parser", () => { @ = \\@ \\| \\( \\) -> "escaped" `; - const result = parseGrammar("unicode.grammar", grammar); + const result = parseGrammarRules("unicode.grammar", grammar); expect(result).toHaveLength(3); expect(result[0].rules[0].expressions[0]).toEqual({ @@ -582,7 +582,7 @@ describe("Grammar Parser", () => { const spaces = " \t\v\f\u00a0\ufeff\n\r\u2028\u2029\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F\u3000"; const grammar = `${spaces}@${spaces}${spaces}=${spaces}hello${spaces}world${spaces}->${spaces}true${spaces}`; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toEqual([ { @@ -607,7 +607,7 @@ describe("Grammar Parser", () => { it("should keep escaped whitespace in expression", () => { const grammar = `@=${escapedSpaces}hello${escapedSpaces}world${escapedSpaces}->true`; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toEqual([ { @@ -638,7 +638,7 @@ describe("Grammar Parser", () => { @ = hello // End of line comment // Another comment `; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result).toHaveLength(1); expect(result[0].name).toBe("greeting"); @@ -652,7 +652,7 @@ describe("Grammar Parser", () => { */ @ = hello /* inline comment */ world `; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "string", @@ -662,7 +662,7 @@ describe("Grammar Parser", () => { it("should handle mixed whitespace types", () => { const grammar = "@\t=\r\nhello\n\t world"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "string", @@ -672,7 +672,7 @@ describe("Grammar Parser", () => { it("should collapse multiple whitespace in strings to single space", () => { const grammar = "@ = hello world\t\t\ttest"; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].expressions[0]).toEqual({ type: "string", @@ -688,7 +688,7 @@ describe("Grammar Parser", () => { count: 1 } `; - const result = parseGrammar("test.grammar", grammar); + const result = parseGrammarRules("test.grammar", grammar); expect(result[0].rules[0].value).toEqual({ type: "object", @@ -703,112 +703,112 @@ describe("Grammar Parser", () => { describe("Error Handling", () => { it("should throw error for missing @ at start of rule", () => { const grammar = " = hello"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "'@' expected", ); }); it("should throw error for malformed rule name", () => { const grammar = "@greeting = hello"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "'<' expected", ); }); it("should throw error for missing equals sign", () => { const grammar = "@ hello"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "'=' expected", ); }); it("should throw error for unterminated string literal", () => { const grammar = '@ = test -> "unterminated'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Unterminated string literal", ); }); it("should throw error for unterminated variable", () => { const grammar = "@ = $(name"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "')' expected", ); }); it("should throw error for unterminated group", () => { const grammar = "@ = (hello"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "')' expected", ); }); it("should throw error for invalid escape sequence", () => { const grammar = '@ = test -> "invalid\\'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Missing escaped character.", ); }); it("should throw error for invalid hex escape", () => { const grammar = '@ = test -> "\\xZZ"'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Invalid hex escape sequence", ); }); it("should throw error for invalid unicode escape", () => { const grammar = '@ = test -> "\\uZZZZ"'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Invalid Unicode escape sequence", ); }); it("should throw error for unterminated array", () => { const grammar = "@ = test -> [1, 2"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Unexpected end of file in array value", ); }); it("should throw error for unterminated object", () => { const grammar = '@ = test -> {type: "test"'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Unexpected end of file in object value", ); }); it("should throw error for missing colon in object", () => { const grammar = '@ = test -> {type "test"}'; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "':' expected", ); }); it("should throw error for invalid number", () => { const grammar = "@ = test -> abc123"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Invalid literal", ); }); it("should throw error for infinity values", () => { const grammar = "@ = test -> Infinity"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Infinity values are not allowed", ); }); it("should throw error for unescaped special characters", () => { const grammar = "@ = hello-world"; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Special character needs to be escaped", ); }); it("should throw error for empty expression", () => { const grammar = "@ = "; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( "Empty expression", ); }); @@ -818,7 +818,7 @@ describe("Grammar Parser", () => { @ = hello @invalid = world `; - expect(() => parseGrammar("test.grammar", grammar)).toThrow( + expect(() => parseGrammarRules("test.grammar", grammar)).toThrow( /test\.grammar:\d+:\d+:/, ); }); @@ -842,7 +842,7 @@ describe("Grammar Parser", () => { } `; - const result = parseGrammar("deeply-nested.grammar", grammar); + const result = parseGrammarRules("deeply-nested.grammar", grammar); const value = result[0].rules[0].value as any; expect(value.type).toBe("object"); @@ -881,7 +881,7 @@ describe("Grammar Parser", () => { } `; - const result = parseGrammar("conversation.grammar", grammar); + const result = parseGrammarRules("conversation.grammar", grammar); expect(result).toHaveLength(3); diff --git a/ts/packages/actionGrammar/test/grammarWriter.spec.ts b/ts/packages/actionGrammar/test/grammarRuleWriter.spec.ts similarity index 88% rename from ts/packages/actionGrammar/test/grammarWriter.spec.ts rename to ts/packages/actionGrammar/test/grammarRuleWriter.spec.ts index 7eddd7c32..1a228ccb5 100644 --- a/ts/packages/actionGrammar/test/grammarWriter.spec.ts +++ b/ts/packages/actionGrammar/test/grammarRuleWriter.spec.ts @@ -1,18 +1,21 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { parseGrammar, expressionsSpecialChar } from "../src/grammarParser.js"; -import { writeGrammar } from "../src/grammarWriter.js"; +import { + parseGrammarRules, + expressionsSpecialChar, +} from "../src/grammarRuleParser.js"; +import { writeGrammarRules } from "../src/grammarRuleWriter.js"; import { escapedSpaces, spaces } from "./testUtils.js"; function validateRoundTrip(grammar: string) { - const rules = parseGrammar("orig", grammar); - const str = writeGrammar(rules); - const parsed = parseGrammar("test", str); + const rules = parseGrammarRules("orig", grammar); + const str = writeGrammarRules(rules); + const parsed = parseGrammarRules("test", str); expect(parsed).toStrictEqual(rules); } -describe("Grammar Writer", () => { +describe("Grammar Rule Writer", () => { it("simple", () => { validateRoundTrip(`@ = hello world`); }); diff --git a/ts/packages/actionGrammar/test/grammarSerialization.spec.ts b/ts/packages/actionGrammar/test/grammarSerialization.spec.ts new file mode 100644 index 000000000..e2a7e3374 --- /dev/null +++ b/ts/packages/actionGrammar/test/grammarSerialization.spec.ts @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { grammarFromJson } from "../src/grammarDeserializer.js"; +import { loadGrammarRules } from "../src/grammarLoader.js"; +import { grammarToJson } from "../src/grammarSerializer.js"; + +describe("Grammar Serialization", () => { + it("Round trip", () => { + const grammarText = ` + @ = hello $(x:number) -> { greeting: $(x) } + @ = one | two | three | $(y:string) | maybe + `; + const grammar = loadGrammarRules("test", grammarText); + const serialized = grammarToJson(grammar); + const deserialized = grammarFromJson(serialized); + expect(deserialized).toEqual(grammar); + const serialized2 = grammarToJson(deserialized); + expect(serialized2).toEqual(serialized); + }); +}); diff --git a/ts/packages/actionGrammarCompiler/LICENSE b/ts/packages/actionGrammarCompiler/LICENSE new file mode 100644 index 000000000..9e841e7a2 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/ts/packages/actionGrammarCompiler/README.md b/ts/packages/actionGrammarCompiler/README.md new file mode 100644 index 000000000..a4117f6c8 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/README.md @@ -0,0 +1,12 @@ +# action-schema-compiler + +A command line tool to preprocess action schema authored in TypeScript, save the result in ParsedActionSchemaGroup JSON format. +Dispatcher can load the result instead of having the typescript compiler parse the TypeScript schema at runtime. + +## Trademarks + +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow +[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). +Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. +Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/ts/packages/actionGrammarCompiler/bin/dev.cmd b/ts/packages/actionGrammarCompiler/bin/dev.cmd new file mode 100644 index 000000000..214818453 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/bin/dev.cmd @@ -0,0 +1,6 @@ +:: Copyright (c) Microsoft Corporation. +:: Licensed under the MIT License. + +@echo off + +node --loader ts-node/esm --no-warnings=ExperimentalWarning "%~dp0\dev" %* diff --git a/ts/packages/actionGrammarCompiler/bin/dev.js b/ts/packages/actionGrammarCompiler/bin/dev.js new file mode 100755 index 000000000..fb7531202 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/bin/dev.js @@ -0,0 +1,10 @@ +#!/usr/bin/env -S node --loader ts-node/esm --no-warnings=ExperimentalWarning +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +async function main() { + const { execute } = await import("@oclif/core"); + await execute({ development: true, dir: import.meta.url }); +} + +await main(); diff --git a/ts/packages/actionGrammarCompiler/bin/run.cmd b/ts/packages/actionGrammarCompiler/bin/run.cmd new file mode 100644 index 000000000..656e451fe --- /dev/null +++ b/ts/packages/actionGrammarCompiler/bin/run.cmd @@ -0,0 +1,6 @@ +:: Copyright (c) Microsoft Corporation. +:: Licensed under the MIT License. + +@echo off + +node "%~dp0\run" %* diff --git a/ts/packages/actionGrammarCompiler/bin/run.js b/ts/packages/actionGrammarCompiler/bin/run.js new file mode 100755 index 000000000..a5e3a0241 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/bin/run.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +async function main() { + const { execute } = await import("@oclif/core"); + await execute({ dir: import.meta.url }); +} + +await main(); diff --git a/ts/packages/actionGrammarCompiler/package.json b/ts/packages/actionGrammarCompiler/package.json new file mode 100644 index 000000000..d2a969ff6 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/package.json @@ -0,0 +1,50 @@ +{ + "name": "action-grammar-compiler", + "version": "0.0.1", + "description": "Action Grammar compiler", + "homepage": "https://github.com/microsoft/TypeAgent#readme", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/TypeAgent.git", + "directory": "ts/packages/actionGrammarCompiler" + }, + "license": "MIT", + "author": "Microsoft", + "type": "module", + "main": "", + "bin": { + "agc": "./bin/run.js", + "agc-dev": "./bin/dev.js" + }, + "files": [ + "dist", + "!dist/test" + ], + "scripts": { + "build": "npm run tsc", + "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log", + "prettier": "prettier --check . --ignore-path ../../.prettierignore", + "prettier:fix": "prettier --write . --ignore-path ../../prettierignore", + "tsc": "tsc -b" + }, + "oclif": { + "bin": "agc", + "commands": { + "strategy": "single", + "target": "./dist/index.js" + }, + "dirname": "agc", + "plugins": [ + "@oclif/plugin-help" + ] + }, + "dependencies": { + "@oclif/core": "^4.2.10", + "@oclif/plugin-help": "^6", + "action-grammar": "workspace:*" + }, + "devDependencies": { + "rimraf": "^6.0.1", + "typescript": "~5.4.5" + } +} diff --git a/ts/packages/actionGrammarCompiler/src/index.ts b/ts/packages/actionGrammarCompiler/src/index.ts new file mode 100644 index 000000000..325ae1b45 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/src/index.ts @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Command, Flags } from "@oclif/core"; +import path from "node:path"; +import fs from "node:fs"; +import { grammarToJson, loadGrammarRules } from "action-grammar"; + +export default class Compile extends Command { + static description = "Compile action schema files"; + + static flags = { + input: Flags.file({ + description: "Input action schema definition in typescript", + required: true, + exists: true, + char: "i", + }), + output: Flags.string({ + description: "Output file for parsed action schema group", + required: true, + char: "o", + }), + }; + + async run(): Promise { + const { flags } = await this.parse(Compile); + + const name = path.basename(flags.input); + + const grammar = loadGrammarRules( + name, + fs.readFileSync(flags.input, "utf-8"), + ); + + const outputDir = path.dirname(flags.output); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + fs.writeFileSync(flags.output, JSON.stringify(grammarToJson(grammar))); + console.log(`Action grammar written: ${flags.output}`); + } +} diff --git a/ts/packages/actionGrammarCompiler/src/tsconfig.json b/ts/packages/actionGrammarCompiler/src/tsconfig.json new file mode 100644 index 000000000..4a46f68f2 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/src/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "rootDir": ".", + "outDir": "../dist" + }, + "include": ["./**/*"], + "ts-node": { + "esm": true + } +} diff --git a/ts/packages/actionGrammarCompiler/tsconfig.json b/ts/packages/actionGrammarCompiler/tsconfig.json new file mode 100644 index 000000000..b6e1577e4 --- /dev/null +++ b/ts/packages/actionGrammarCompiler/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true + }, + "include": [], + "references": [{ "path": "./src" }], + "ts-node": { + "esm": true + } +} diff --git a/ts/packages/actionSchema/src/index.ts b/ts/packages/actionSchema/src/index.ts index e1e8057b9..708d9728f 100644 --- a/ts/packages/actionSchema/src/index.ts +++ b/ts/packages/actionSchema/src/index.ts @@ -13,6 +13,7 @@ export { ActionSchemaObject, ActionSchemaUnion, SchemaObjectFields, + ActionSchemaEntityTypeDefinition, } from "./type.js"; export { parseActionSchemaSource } from "./parser.js"; diff --git a/ts/packages/agentRpc/src/types.ts b/ts/packages/agentRpc/src/types.ts index bd6be0875..e277a9e1c 100644 --- a/ts/packages/agentRpc/src/types.ts +++ b/ts/packages/agentRpc/src/types.ts @@ -21,8 +21,8 @@ import { TemplateSchema, TypeAgentAction, CompletionGroup, + ResolveEntityResult, } from "@typeagent/agent-sdk"; -import { ResolveEntityResult } from "../../agentSdk/dist/agentInterface.js"; import { AgentInterfaceFunctionName } from "./server.js"; export type AgentContextCallFunctions = { diff --git a/ts/packages/agentSdk/src/agentInterface.ts b/ts/packages/agentSdk/src/agentInterface.ts index dda5bf579..cb9a4016e 100644 --- a/ts/packages/agentSdk/src/agentInterface.ts +++ b/ts/packages/agentSdk/src/agentInterface.ts @@ -48,10 +48,13 @@ export type SchemaTypeNames = { }; export type SchemaFormat = "ts" | "pas"; +export type GrammarFormat = "ag"; + export type SchemaManifest = { description: string; schemaType: string | SchemaTypeNames; // string if there are only action schemas schemaFile: string | { format: SchemaFormat; content: string }; + grammarFile?: string | { format: GrammarFormat; content: string }; injected?: boolean; // whether the translator is injected into other domains, default is false cached?: boolean; // whether the translator's action should be cached, default is true streamingActions?: string[]; diff --git a/ts/packages/agentSdk/src/index.ts b/ts/packages/agentSdk/src/index.ts index 74e52ae4f..b4846aba0 100644 --- a/ts/packages/agentSdk/src/index.ts +++ b/ts/packages/agentSdk/src/index.ts @@ -4,6 +4,7 @@ export { AppAgentManifest, ActionManifest, + GrammarFormat, SchemaFormat, SchemaManifest, AppAgent, diff --git a/ts/packages/agents/montage/src/agent/montageActionHandler.ts b/ts/packages/agents/montage/src/agent/montageActionHandler.ts index 24d94ec7d..9d91eccbd 100644 --- a/ts/packages/agents/montage/src/agent/montageActionHandler.ts +++ b/ts/packages/agents/montage/src/agent/montageActionHandler.ts @@ -8,6 +8,7 @@ import { ActionResult, TypeAgentAction, AppAgentInitSettings, + ResolveEntityResult, } from "@typeagent/agent-sdk"; import { ChildProcess, fork } from "child_process"; import { fileURLToPath } from "node:url"; @@ -36,7 +37,6 @@ import { import registerDebug from "debug"; import { createSemanticMap } from "typeagent"; import { openai, TextEmbeddingModel } from "aiclient"; -import { ResolveEntityResult } from "../../../../agentSdk/dist/agentInterface.js"; const debug = registerDebug("typeagent:agent:montage"); diff --git a/ts/packages/agents/player/package.json b/ts/packages/agents/player/package.json index c5446f26e..f6026b79b 100644 --- a/ts/packages/agents/player/package.json +++ b/ts/packages/agents/player/package.json @@ -17,8 +17,9 @@ "./agent/handlers": "./dist/agent/playerHandlers.js" }, "scripts": { + "agc": "agc -i ./src/agent/playerGrammar.agr -o ./dist/agent/playerGrammar.ag.json", "asc": "asc -i ./src/agent/playerSchema.ts -o ./dist/agent/playerSchema.pas.json -t PlayerActions -e PlayerEntities", - "build": "concurrently npm:tsc npm:asc", + "build": "concurrently npm:tsc npm:asc npm:agc", "clean": "rimraf --glob dist *.tsbuildinfo *.done.build.log", "tsc": "tsc -p src" }, @@ -36,6 +37,7 @@ "@types/debug": "^4.1.12", "@types/express": "^4.17.17", "@types/spotify-api": "^0.0.25", + "action-grammar-compiler": "workspace:*", "action-schema-compiler": "workspace:*", "concurrently": "^9.1.2", "rimraf": "^6.0.1", @@ -43,6 +45,19 @@ }, "fluidBuild": { "tasks": { + "agc": { + "dependsOn": [ + "action-grammar-compiler#tsc" + ], + "files": { + "inputGlobs": [ + "src/agent/playerGrammar.agr" + ], + "outputGlobs": [ + "dist/agent/playerGrammar.ag.json" + ] + } + }, "asc": { "dependsOn": [ "action-schema-compiler#tsc" diff --git a/ts/packages/agents/player/src/agent/playerGrammar.agr b/ts/packages/agents/player/src/agent/playerGrammar.agr new file mode 100644 index 000000000..d760c94b9 --- /dev/null +++ b/ts/packages/agents/player/src/agent/playerGrammar.agr @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +@ = + | + +@ = pause ((the)? music)? -> { actionName: "pause" } +@ = resume ((the)? music)? -> { actionName: "resume" } \ No newline at end of file diff --git a/ts/packages/agents/player/src/agent/playerManifest.json b/ts/packages/agents/player/src/agent/playerManifest.json index 387769a33..e99c7bfcb 100644 --- a/ts/packages/agents/player/src/agent/playerManifest.json +++ b/ts/packages/agents/player/src/agent/playerManifest.json @@ -4,6 +4,7 @@ "schema": { "description": "Music Player agent that lets you search for, play and control music.", "schemaFile": "../../dist/agent/playerSchema.pas.json", + "grammarFile": "../../dist/agent/playerGrammar.ag.json", "originalSchemaFile": "playerSchema.ts", "schemaType": { "action": "PlayerActions", diff --git a/ts/packages/cache/src/cache/cache.ts b/ts/packages/cache/src/cache/cache.ts index af0f8881f..0ab8a7e05 100644 --- a/ts/packages/cache/src/cache/cache.ts +++ b/ts/packages/cache/src/cache/cache.ts @@ -15,14 +15,22 @@ import { doCacheAction, isValidActionSchemaFileHash, } from "../explanation/schemaInfoProvider.js"; -import { ConstructionStore, ConstructionStoreImpl } from "./store.js"; +import { + ConstructionStore, + ConstructionStoreImpl, +} from "./constructionStore.js"; import { ExplainerFactory } from "./factory.js"; -import { NamespaceKeyFilter } from "../constructions/constructionCache.js"; +import { + MatchOptions, + NamespaceKeyFilter, +} from "../constructions/constructionCache.js"; import { ExplainWorkQueue, ExplanationOptions, ProcessExplanationResult, } from "./explainWorkQueue.js"; +import { GrammarStoreImpl } from "./grammarStore.js"; +import { GrammarStore, MatchResult } from "./types.js"; export type ProcessRequestActionResult = { explanationResult: ProcessExplanationResult; @@ -51,20 +59,27 @@ function getFailedResult(message: string): ProcessRequestActionResult { }; } -// Construction namespace policy +export function getSchemaNamespaceKey( + name: string, + activityName: string | undefined, + schemaInfoProvider: SchemaInfoProvider | undefined, +) { + return `${name},${schemaInfoProvider?.getActionSchemaFileHash(name) ?? ""},${activityName ?? ""}`; +} + +// Namespace policy. Combines schema name, file hash, and activity name to indicate enabling/disabling of matching. export function getSchemaNamespaceKeys( schemaNames: string[], activityName: string | undefined, schemaInfoProvider: SchemaInfoProvider | undefined, ) { // Current namespace keys policy is just combining schema name its file hash - return schemaNames.map( - (name) => - `${name},${schemaInfoProvider?.getActionSchemaFileHash(name) ?? ""},${activityName ?? ""}`, + return schemaNames.map((name) => + getSchemaNamespaceKey(name, activityName, schemaInfoProvider), ); } -function splitSchemaNamespaceKey(namespaceKey: string): { +export function splitSchemaNamespaceKey(namespaceKey: string): { schemaName: string; hash: string | undefined; activityName: string | undefined; @@ -79,6 +94,7 @@ function splitSchemaNamespaceKey(namespaceKey: string): { export class AgentCache { private _constructionStore: ConstructionStoreImpl; + private _grammarStore: GrammarStoreImpl; private readonly explainWorkQueue: ExplainWorkQueue; // Function to return whether the namespace key matches to the current schema file's hash. private readonly namespaceKeyFilter?: NamespaceKeyFilter; @@ -91,6 +107,7 @@ export class AgentCache { cacheOptions?: CacheOptions, logger?: Telemetry.Logger, ) { + this._grammarStore = new GrammarStoreImpl(schemaInfoProvider); this._constructionStore = new ConstructionStoreImpl( explainerName, cacheOptions, @@ -117,6 +134,9 @@ export class AgentCache { } } + public get grammarStore(): GrammarStore { + return this._grammarStore; + } public get constructionStore(): ConstructionStore { return this._constructionStore; } @@ -170,27 +190,24 @@ export class AgentCache { options?.namespaceSuffix, ); - const store = this._constructionStore; - if (store.isEnabled()) { - // Make sure that we don't already have a construction that will match (but reject because of options) - const matchResult = store.match(requestAction.request, { - rejectReferences: false, - history: requestAction.history, - namespaceKeys, - }); + // Make sure that we don't already have match (but rejected because of options) + const matchResult = this.match(requestAction.request, { + rejectReferences: false, + history: requestAction.history, + namespaceKeys, + }); - const actions = executableActions.map((e) => e.action); - for (const match of matchResult) { - if ( - equalNormalizedObject( - match.match.actions.map((e) => e.action), - actions, - ) - ) { - return getFailedResult( - `Existing construction matches the request but rejected.`, - ); - } + const actions = executableActions.map((e) => e.action); + for (const match of matchResult) { + if ( + equalNormalizedObject( + match.match.actions.map((e) => e.action), + actions, + ) + ) { + return getFailedResult( + `Existing construction matches the request but rejected.`, + ); } } @@ -216,6 +233,7 @@ export class AgentCache { elapsedMs, }); + const store = this._constructionStore; const generateConstruction = cache && store.isEnabled(); if (generateConstruction && explanation.success) { const construction = explanation.construction; @@ -285,4 +303,30 @@ export class AgentCache { ignoreSourceHash, ); } + + public isEnabled(): boolean { + return ( + this._grammarStore.isEnabled() || + this._constructionStore.isEnabled() + ); + } + + public match(request: string, options?: MatchOptions): MatchResult[] { + const store = this._constructionStore; + if (store.isEnabled()) { + const constructionMatches = store.match(request, options); + if (constructionMatches.length > 0) { + // TODO: Move this in the construction store + return constructionMatches.map((m) => { + const { construction, ...rest } = m; + return rest; + }); + } + } + const grammarStore = this._grammarStore; + if (grammarStore.isEnabled()) { + return this._grammarStore.match(request, options); + } + throw new Error("AgentCache is disabled"); + } } diff --git a/ts/packages/cache/src/cache/store.ts b/ts/packages/cache/src/cache/constructionStore.ts similarity index 96% rename from ts/packages/cache/src/cache/store.ts rename to ts/packages/cache/src/cache/constructionStore.ts index 814ab9014..32c7eb897 100644 --- a/ts/packages/cache/src/cache/store.ts +++ b/ts/packages/cache/src/cache/constructionStore.ts @@ -5,7 +5,10 @@ import fs from "node:fs"; import path from "node:path"; import chalk from "chalk"; import registerDebug from "debug"; -import { Construction, MatchResult } from "../constructions/constructions.js"; +import { + Construction, + ConstructionMatchResult, +} from "../constructions/constructions.js"; import { ExplanationData } from "../explanation/explanationData.js"; import { importConstructions } from "../constructions/importConstructions.js"; import { CacheConfig, CacheOptions } from "./cache.js"; @@ -20,6 +23,7 @@ import { PrintOptions, printConstructionCache, } from "../constructions/constructionPrint.js"; +import { sortMatches } from "./sortMatches.js"; const debugConstMatch = registerDebug("typeagent:const:match"); @@ -94,7 +98,7 @@ export interface ConstructionStore { delete(schemaName: string, id: number): Promise; // Usage - match(request: string, options?: MatchOptions): MatchResult[]; + match(request: string, options?: MatchOptions): ConstructionMatchResult[]; // Completion getPrefix(namespaceKeys?: string[]): string[]; @@ -370,12 +374,13 @@ export class ConstructionStoreImpl implements ConstructionStore { if (matches.length === 0 && this.builtInCache !== undefined) { matches = this.builtInCache.match(request, options); } + const sortedMatches = sortMatches(matches); if (debugConstMatch.enabled) { debugConstMatch( - `Found ${matches.length} construction(s) for '${request}':`, + `Found ${sortedMatches.length} construction(s) for '${request}':`, ); - for (let i = 0; i < matches.length; i++) { - const match = matches[i]; + for (let i = 0; i < sortedMatches.length; i++) { + const match = sortedMatches[i]; const actionStr = chalk.green(match.match.actions); const constructionStr = chalk.grey(`(${match.construction})`); const message = [ @@ -389,7 +394,7 @@ export class ConstructionStoreImpl implements ConstructionStore { debugConstMatch(message.join("\n")); } } - return matches; + return sortedMatches; } public async prune(filter: (namespaceKey: string) => boolean) { diff --git a/ts/packages/cache/src/cache/grammarStore.ts b/ts/packages/cache/src/cache/grammarStore.ts new file mode 100644 index 000000000..f5002d4c3 --- /dev/null +++ b/ts/packages/cache/src/cache/grammarStore.ts @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Grammar, matchGrammar } from "action-grammar"; +import { MatchOptions } from "../constructions/constructionCache.js"; +import { MatchResult, GrammarStore } from "./types.js"; +import { getSchemaNamespaceKey, splitSchemaNamespaceKey } from "./cache.js"; +import { SchemaInfoProvider } from "../explanation/schemaInfoProvider.js"; +import { + createExecutableAction, + RequestAction, +} from "../explanation/requestAction.js"; +import { sortMatches } from "./sortMatches.js"; + +export class GrammarStoreImpl implements GrammarStore { + private readonly grammars: Map = new Map(); + private enabled: boolean = true; + public constructor( + private readonly schemaInfoProvider: SchemaInfoProvider | undefined, + ) {} + public addGrammar(schemaName: string, grammar: Grammar): void { + const namespaceKey = getSchemaNamespaceKey( + schemaName, + undefined, + this.schemaInfoProvider, + ); + this.grammars.set(namespaceKey, grammar); + } + public removeGrammar(schemaName: string): void { + const namespaceKey = getSchemaNamespaceKey( + schemaName, + undefined, + this.schemaInfoProvider, + ); + this.grammars.delete(namespaceKey); + } + public isEnabled(): boolean { + return this.enabled; + } + public setEnabled(enabled: boolean): void { + this.enabled = enabled; + } + public match(request: string, options?: MatchOptions): MatchResult[] { + if (!this.enabled) { + throw new Error("GrammarStore is disabled"); + } + const namespaceKeys = options?.namespaceKeys; + if (namespaceKeys?.length === 0) { + return []; + } + const matches: MatchResult[] = []; + const filter = namespaceKeys ? new Set(namespaceKeys) : undefined; + for (const [name, grammar] of this.grammars) { + if (filter && !filter.has(name)) { + continue; + } + const grammarMatches = matchGrammar(grammar, request); + if (grammarMatches.length === 0) { + continue; + } + const { schemaName } = splitSchemaNamespaceKey(name); + for (const m of grammarMatches) { + const action: any = m.match; + matches.push({ + type: "grammar", + match: new RequestAction(request, [ + createExecutableAction( + schemaName, + action.actionName, + action.parameters, + ), + ]), + wildcardCharCount: m.wildcardCharCount, + implicitParameterCount: 0, + nonOptionalCount: 0, + matchedCount: m.matchedValueCount, + entityWildcardPropertyNames: m.entityWildcardPropertyNames, + }); + } + } + return sortMatches(matches); + } +} diff --git a/ts/packages/cache/src/cache/sortMatches.ts b/ts/packages/cache/src/cache/sortMatches.ts new file mode 100644 index 000000000..8d3eb87aa --- /dev/null +++ b/ts/packages/cache/src/cache/sortMatches.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { MatchResult } from "./types.js"; + +export function sortMatches(matches: T[]) { + return matches.sort((a, b) => { + // REVIEW: temporary heuristics to get better result with wildcards + + // Prefer non-wildcard matches + if (a.wildcardCharCount === 0) { + if (b.wildcardCharCount !== 0) { + return -1; + } + } else { + if (b.wildcardCharCount === 0) { + return 1; + } + } + + // Prefer less implicit parameters + if (a.implicitParameterCount !== b.implicitParameterCount) { + return a.implicitParameterCount - b.implicitParameterCount; + } + + // Prefer more non-optional parts + if (b.nonOptionalCount !== a.nonOptionalCount) { + return b.nonOptionalCount - a.nonOptionalCount; + } + + // Prefer more matched parts + if (b.matchedCount !== a.matchedCount) { + return b.matchedCount - a.matchedCount; + } + + // Prefer less wildcard characters + return a.wildcardCharCount - b.wildcardCharCount; + }); +} diff --git a/ts/packages/cache/src/cache/types.ts b/ts/packages/cache/src/cache/types.ts new file mode 100644 index 000000000..411aa94e0 --- /dev/null +++ b/ts/packages/cache/src/cache/types.ts @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { ParamValueType, RequestAction } from "../explanation/requestAction.js"; + +export type MatchResult = { + type: "grammar" | "construction"; + match: RequestAction; + matchedCount: number; + wildcardCharCount: number; + nonOptionalCount: number; + implicitParameterCount: number; + entityWildcardPropertyNames: string[]; + conflictValues?: [string, ParamValueType[]][] | undefined; + partialPartCount?: number | undefined; // Only used for partial match +}; + +export interface GrammarStore { + setEnabled(enabled: boolean): void; + addGrammar(namespace: string, grammar: any): void; + removeGrammar(namespace: string): void; +} diff --git a/ts/packages/cache/src/constructions/constructionCache.ts b/ts/packages/cache/src/constructions/constructionCache.ts index cb4e941ee..c96d833b0 100644 --- a/ts/packages/cache/src/constructions/constructionCache.ts +++ b/ts/packages/cache/src/constructions/constructionCache.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { HistoryContext } from "../explanation/requestAction.js"; -import { Construction, MatchResult } from "./constructions.js"; +import { Construction, ConstructionMatchResult } from "./constructions.js"; import { MatchPart, MatchSet, isMatchPart } from "./matchPart.js"; import { Transforms } from "./transforms.js"; @@ -233,7 +233,7 @@ export class ConstructionCache { request: string, matchConfig: MatchConfig, constructionNamespace: Constructions, - ): MatchResult[] { + ): ConstructionMatchResult[] { return constructionNamespace.constructions.flatMap((construction) => { return construction.match(request, matchConfig); }); @@ -287,7 +287,10 @@ export class ConstructionCache { } // For matching - public match(request: string, options?: MatchOptions): MatchResult[] { + public match( + request: string, + options?: MatchOptions, + ): ConstructionMatchResult[] { const namespaceKeys = options?.namespaceKeys; if (namespaceKeys?.length === 0) { return []; @@ -304,7 +307,7 @@ export class ConstructionCache { // If the useTranslators is undefined use all the translators // otherwise filter the translators based on the useTranslators - const matches: MatchResult[] = []; + const matches: ConstructionMatchResult[] = []; const filter = namespaceKeys ? new Set(namespaceKeys) : undefined; for (const [ name, @@ -320,44 +323,7 @@ export class ConstructionCache { ); } debugConstMatchStat(getMatchPartsCacheStats(config.matchPartsCache)); - return matches.sort((a, b) => { - // REVIEW: temporary heuristics to get better result with wildcards - - // Prefer non-wildcard matches - if (a.wildcardCharCount === 0) { - if (b.wildcardCharCount !== 0) { - return -1; - } - } else { - if (b.wildcardCharCount === 0) { - return 1; - } - } - - // Prefer less implicit parameters - if ( - a.construction.implicitParameterCount !== - b.construction.implicitParameterCount - ) { - return ( - a.construction.implicitParameterCount - - b.construction.implicitParameterCount - ); - } - - // Prefer more non-optional parts - if (b.nonOptionalCount !== a.nonOptionalCount) { - return b.nonOptionalCount - a.nonOptionalCount; - } - - // Prefer more matched parts - if (b.matchedCount !== a.matchedCount) { - return b.matchedCount - a.matchedCount; - } - - // Prefer less wildcard characters - return a.wildcardCharCount - b.wildcardCharCount; - }); + return matches; } public get matchSets(): IterableIterator { diff --git a/ts/packages/cache/src/constructions/constructions.ts b/ts/packages/cache/src/constructions/constructions.ts index 5ddfed4b6..eec692655 100644 --- a/ts/packages/cache/src/constructions/constructions.ts +++ b/ts/packages/cache/src/constructions/constructions.ts @@ -26,6 +26,7 @@ import { ConstructionPartJSON, ParsePartJSON, } from "./constructionJSONTypes.js"; +import { MatchResult } from "../cache/types.js"; export type ImplicitParameter = { paramName: string; @@ -122,7 +123,10 @@ export class Construction { return this.implicitParameters ? this.implicitParameters.length : 0; } - public match(request: string, config: MatchConfig): MatchResult[] { + public match( + request: string, + config: MatchConfig, + ): ConstructionMatchResult[] { const matchedValues = matchParts( request, this.parts, @@ -141,6 +145,7 @@ export class Construction { ); return [ { + type: "construction", construction: this, match: new RequestAction( request, @@ -153,6 +158,7 @@ export class Construction { matchedCount: matchedValues.matchedCount, wildcardCharCount: matchedValues.wildcardCharCount, nonOptionalCount: this.parts.filter((p) => !p.optional).length, + implicitParameterCount: this.implicitParameterCount, partialPartCount: matchedValues.partialPartCount, }, ]; @@ -329,13 +335,6 @@ function isParsePartJSON(part: ConstructionPartJSON): part is ParsePartJSON { return (part as any).parserName !== undefined; } -export type MatchResult = { +export type ConstructionMatchResult = MatchResult & { construction: Construction; - match: RequestAction; - matchedCount: number; - wildcardCharCount: number; - nonOptionalCount: number; - entityWildcardPropertyNames: string[]; - conflictValues?: [string, ParamValueType[]][] | undefined; - partialPartCount?: number | undefined; // Only used for partial match }; diff --git a/ts/packages/cache/src/grammar/exportGrammar.ts b/ts/packages/cache/src/grammar/exportGrammar.ts index 0b0c3f07c..673f281e2 100644 --- a/ts/packages/cache/src/grammar/exportGrammar.ts +++ b/ts/packages/cache/src/grammar/exportGrammar.ts @@ -1,7 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { Expr, Rule, RuleDefinition, writeGrammar } from "action-grammar/rules"; +import { + Expr, + Rule, + RuleDefinition, + writeGrammarRules, + ValueNode, +} from "action-grammar/rules"; import { Construction, getDefaultMatchValueTranslator, @@ -13,8 +19,7 @@ import { MatchSet, TransformInfo, } from "../constructions/matchPart.js"; -import { loadConstructionCacheFile } from "../cache/store.js"; -import { ValueNode } from "../../../actionGrammar/dist/grammarParser.js"; +import { loadConstructionCacheFile } from "../cache/constructionStore.js"; import { setObjectProperty } from "common-utils"; import { MatchedValueTranslator } from "../constructions/constructionValue.js"; @@ -47,7 +52,7 @@ export async function convertConstructionFileToGrammar(fileName: string) { } convertConstructions(state, constructions.constructions); } - return writeGrammar(Array.from(state.definitions)); + return writeGrammarRules(Array.from(state.definitions)); } export function convertConstructionsToGrammar(constructions: Construction[]) { @@ -57,7 +62,7 @@ export function convertConstructionsToGrammar(constructions: Construction[]) { ruleNameNextIds: new Map(), }; convertConstructions(state, constructions); - return writeGrammar(Array.from(state.definitions)); + return writeGrammarRules(Array.from(state.definitions)); } function convertConstructions(state: State, constructions: Construction[]) { diff --git a/ts/packages/cache/src/index.ts b/ts/packages/cache/src/index.ts index 168579e07..678225d68 100644 --- a/ts/packages/cache/src/index.ts +++ b/ts/packages/cache/src/index.ts @@ -11,7 +11,7 @@ export type { PromptEntity, } from "./explanation/requestAction.js"; -export type { ConstructionStore } from "./cache/store.js"; +export type { ConstructionStore } from "./cache/constructionStore.js"; export type { MatchOptions } from "./constructions/constructionCache.js"; export type { GenericExplanationResult, @@ -50,7 +50,8 @@ export { getPropertyInfo, } from "./explanation/requestAction.js"; export { AgentCacheFactory, getDefaultExplainerName } from "./cache/factory.js"; -export { MatchResult, WildcardMode } from "./constructions/constructions.js"; +export type { MatchResult, GrammarStore } from "./cache/types.js"; +export { WildcardMode } from "./constructions/constructions.js"; // Testing export { getNamespaceForCache } from "./explanation/schemaInfoProvider.js"; diff --git a/ts/packages/cli/src/commands/grammar/match.ts b/ts/packages/cli/src/commands/grammar/match.ts index 4322d37f6..e34b3211b 100644 --- a/ts/packages/cli/src/commands/grammar/match.ts +++ b/ts/packages/cli/src/commands/grammar/match.ts @@ -3,8 +3,19 @@ import fs from "node:fs"; import { Args, Command } from "@oclif/core"; -import { loadGrammar, matchGrammar } from "action-grammar"; +import { + grammarFromJson, + loadGrammarRules, + matchGrammar, +} from "action-grammar"; +async function load(fileName: string) { + const content = await fs.promises.readFile(fileName, "utf-8"); + + return content === "{" + ? grammarFromJson(JSON.parse(content)) + : loadGrammarRules(fileName, content); +} export default class MatchCommand extends Command { static description = "Match input against a grammar"; @@ -19,16 +30,17 @@ export default class MatchCommand extends Command { async run(): Promise { const { args } = await this.parse(MatchCommand); - const grammarContent = await fs.promises.readFile( - args.grammar, - "utf-8", - ); - const grammar = loadGrammar(args.grammar, grammarContent); - + const grammar = await load(args.grammar); const result = matchGrammar(grammar, args.input); - if (result) { + if (result.length > 0) { console.log("Matched:"); - console.log(result); + console.log( + JSON.stringify( + result.map((r) => r.match), + null, + 2, + ), + ); } else { console.log("No match"); } diff --git a/ts/packages/defaultAgentProvider/test/grammar.spec.ts b/ts/packages/defaultAgentProvider/test/grammar.spec.ts index 964afff76..6c1a6a557 100644 --- a/ts/packages/defaultAgentProvider/test/grammar.spec.ts +++ b/ts/packages/defaultAgentProvider/test/grammar.spec.ts @@ -13,7 +13,7 @@ import { import { fromJsonActions, RequestAction, toJsonActions } from "agent-cache"; import { getDefaultAppAgentProviders } from "../src/defaultAgentProviders.js"; import { glob } from "glob"; -import { loadGrammar, matchGrammar } from "action-grammar"; +import { loadGrammarRules, matchGrammar } from "action-grammar"; import { convertConstructionsToGrammar } from "agent-cache/grammar"; const dataFiles = ["test/data/explanations/**/v5/*.json"]; @@ -87,7 +87,7 @@ describe("Grammar", () => { continue; } it(testName, async () => { - const g = loadGrammar("test", grammar); + const g = loadGrammarRules("test", grammar); const matched = matchGrammar(g, requestAction.request); // TODO: once MatchPart allow matches ignoring diacritical marks, @@ -99,7 +99,7 @@ describe("Grammar", () => { // Able to match roundtrip expect(matched.length).not.toEqual(0); - expect(matched[0]).toEqual( + expect(matched[0].match).toEqual( toJsonActions(requestAction.actions), ); diff --git a/ts/packages/dispatcher/package.json b/ts/packages/dispatcher/package.json index 644a7e049..277245610 100644 --- a/ts/packages/dispatcher/package.json +++ b/ts/packages/dispatcher/package.json @@ -47,6 +47,7 @@ "@azure/identity": "^4.10.0", "@azure/msal-node-extensions": "^1.5.0", "@typeagent/agent-sdk": "workspace:*", + "action-grammar": "workspace:*", "action-schema": "workspace:*", "agent-cache": "workspace:*", "agent-rpc": "workspace:*", diff --git a/ts/packages/dispatcher/src/agentProvider/npmAgentProvider.ts b/ts/packages/dispatcher/src/agentProvider/npmAgentProvider.ts index 9384e45e4..2aa604a56 100644 --- a/ts/packages/dispatcher/src/agentProvider/npmAgentProvider.ts +++ b/ts/packages/dispatcher/src/agentProvider/npmAgentProvider.ts @@ -22,11 +22,20 @@ export type NpmAppAgentInfo = { }; function patchPaths(manifest: ActionManifest, dir: string) { - if (manifest.schema && typeof manifest.schema.schemaFile === "string") { - manifest.schema.schemaFile = path.resolve( - dir, - manifest.schema.schemaFile, - ); + if (manifest.schema) { + if (typeof manifest.schema.schemaFile === "string") { + manifest.schema.schemaFile = path.resolve( + dir, + manifest.schema.schemaFile, + ); + } + + if (typeof manifest.schema.grammarFile === "string") { + manifest.schema.grammarFile = path.resolve( + dir, + manifest.schema.grammarFile, + ); + } } if (manifest.subActionManifests) { for (const subManifest of Object.values(manifest.subActionManifests)) { diff --git a/ts/packages/dispatcher/src/context/appAgentManager.ts b/ts/packages/dispatcher/src/context/appAgentManager.ts index 27ef9629d..93b6ed64d 100644 --- a/ts/packages/dispatcher/src/context/appAgentManager.ts +++ b/ts/packages/dispatcher/src/context/appAgentManager.ts @@ -32,6 +32,10 @@ import { AppAgentStateConfig, appAgentStateKeys, } from "./appAgentStateConfig.js"; +import { GrammarStore } from "agent-cache"; +import { getPackageFilePath } from "../utils/getPackageFilePath.js"; +import fs from "node:fs"; +import { Grammar, grammarFromJson } from "action-grammar"; const debug = registerDebug("typeagent:dispatcher:agents"); const debugError = registerDebug("typeagent:dispatcher:agents:error"); @@ -91,6 +95,32 @@ export const alwaysEnabledAgents = { commands: ["system"], }; +function loadGrammar(actionConfig: ActionConfig): Grammar | undefined { + if (actionConfig.grammarFile === undefined) { + return undefined; + } + + let source: string; + if (typeof actionConfig.grammarFile === "string") { + const fullPath = getPackageFilePath(actionConfig.grammarFile); + source = fs.readFileSync(fullPath, "utf-8"); + const isActionGrammar = actionConfig.grammarFile.endsWith(".ag.json"); + if (!isActionGrammar) { + throw new Error( + `Unsupported grammar file extension: ${actionConfig.grammarFile}`, + ); + } + } else { + if (actionConfig.grammarFile.format !== "ag") { + throw new Error( + `Unsupported grammar file extension: ${actionConfig.grammarFile}`, + ); + } + source = actionConfig.grammarFile.content; + } + return grammarFromJson(JSON.parse(source)); +} + export class AppAgentManager implements ActionConfigProvider { private readonly agents = new Map(); private readonly actionConfigs = new Map(); @@ -237,6 +267,7 @@ export class AppAgentManager implements ActionConfigProvider { public async addProvider( provider: AppAgentProvider, + actionGrammarStore: GrammarStore | undefined, actionEmbeddingCache?: EmbeddingCache, ) { const semanticMapP: Promise[] = []; @@ -246,6 +277,8 @@ export class AppAgentManager implements ActionConfigProvider { name, manifest, semanticMapP, + + actionGrammarStore, provider, actionEmbeddingCache, ); @@ -259,6 +292,7 @@ export class AppAgentManager implements ActionConfigProvider { appAgentName: string, manifest: AppAgentManifest, semanticMapP: Promise[], + actionGrammarStore: GrammarStore | undefined, provider?: AppAgentProvider, actionEmbeddingCache?: EmbeddingCache, ) { @@ -289,6 +323,21 @@ export class AppAgentManager implements ActionConfigProvider { ), ); } + + if (actionGrammarStore) { + try { + const g = loadGrammar(config); + if (g) { + debug(`Adding grammar for schema: ${schemaName}`); + actionGrammarStore.addGrammar(schemaName, g); + } + } catch (e) { + // REVIEW: Ignore errors for now. + debugError( + `Failed to load grammar for schema: ${schemaName}\n${e}`, + ); + } + } } catch (e: any) { schemaErrors.set(schemaName, e); } @@ -321,6 +370,7 @@ export class AppAgentManager implements ActionConfigProvider { appAgentName: string, manifest: AppAgentManifest, appAgent: AppAgent, + actionGrammarStore?: GrammarStore, ) { if (this.agents.has(appAgentName)) { throw new Error(`App agent '${appAgentName}' already exists`); @@ -332,6 +382,7 @@ export class AppAgentManager implements ActionConfigProvider { appAgentName, manifest, semanticMapP, + actionGrammarStore, ); record.appAgent = appAgent; @@ -340,7 +391,7 @@ export class AppAgentManager implements ActionConfigProvider { debug("Finish action embeddings"); } - private cleanupAgent(appAgentName: string) { + private cleanupAgent(appAgentName: string, grammarStore?: GrammarStore) { for (const [schemaName, config] of this.actionConfigs) { if (getAppAgentName(schemaName) !== appAgentName) { continue; @@ -351,13 +402,19 @@ export class AppAgentManager implements ActionConfigProvider { if (config.transient) { delete this.transientAgents[schemaName]; } + if (grammarStore) { + grammarStore.removeGrammar(schemaName); + } } } - public async removeAgent(appAgentName: string) { + public async removeAgent( + appAgentName: string, + grammarStore?: GrammarStore, + ) { const record = this.getRecord(appAgentName); this.agents.delete(appAgentName); - this.cleanupAgent(appAgentName); + this.cleanupAgent(appAgentName, grammarStore); await this.closeSessionContext(record); if (record.appAgent !== undefined) { diff --git a/ts/packages/dispatcher/src/context/commandHandlerContext.ts b/ts/packages/dispatcher/src/context/commandHandlerContext.ts index 3a47f8812..36090f752 100644 --- a/ts/packages/dispatcher/src/context/commandHandlerContext.ts +++ b/ts/packages/dispatcher/src/context/commandHandlerContext.ts @@ -319,11 +319,19 @@ async function addAppAgentProviders( } const inlineAppProvider = createBuiltinAppAgentProvider(context); - await context.agents.addProvider(inlineAppProvider, embeddingCache); + await context.agents.addProvider( + inlineAppProvider, + context.agentCache.grammarStore, + embeddingCache, + ); if (appAgentProviders) { for (const provider of appAgentProviders) { - await context.agents.addProvider(provider, embeddingCache); + await context.agents.addProvider( + provider, + context.agentCache.grammarStore, + embeddingCache, + ); } } if (embeddingCachePath) { @@ -361,7 +369,7 @@ export async function installAppProvider( provider: AppAgentProvider, ) { // Don't use embedding cache for a new agent. - await context.agents.addProvider(provider); + await context.agents.addProvider(provider, context.agentCache.grammarStore); await setAppAgentStates(context); @@ -705,5 +713,9 @@ export async function changeContextConfig( } } + if (changed.cache?.grammar !== undefined) { + agentCache.grammarStore.setEnabled(changed.cache.grammar); + } + return changed; } diff --git a/ts/packages/dispatcher/src/context/interactiveIO.ts b/ts/packages/dispatcher/src/context/interactiveIO.ts index e62bcdf25..28c2c5dd9 100644 --- a/ts/packages/dispatcher/src/context/interactiveIO.ts +++ b/ts/packages/dispatcher/src/context/interactiveIO.ts @@ -32,7 +32,7 @@ export interface IAgentMessage { export type NotifyExplainedData = { error?: string | undefined; - fromCache: boolean; + fromCache: "construction" | "grammar" | false; fromUser: boolean; time: string; }; diff --git a/ts/packages/dispatcher/src/context/session.ts b/ts/packages/dispatcher/src/context/session.ts index 0b264ad93..cd94cc4d8 100644 --- a/ts/packages/dispatcher/src/context/session.ts +++ b/ts/packages/dispatcher/src/context/session.ts @@ -153,6 +153,7 @@ export type DispatcherConfig = { // Cache behaviors cache: CacheConfig & { enabled: boolean; + grammar: boolean; autoSave: boolean; builtInCache: boolean; matchWildcard: boolean; @@ -235,6 +236,7 @@ const defaultSessionConfig: SessionConfig = { }, cache: { enabled: true, + grammar: true, autoSave: true, mergeMatchSets: true, // the session default is different then the default in the cache cacheConflicts: true, // the session default is different then the default in the cache @@ -601,6 +603,8 @@ export async function setupAgentCache( ); } await agentCache.constructionStore.setAutoSave(config.cache.autoSave); + + agentCache.grammarStore.setEnabled(config.cache.grammar); } export async function setupBuiltInCache( diff --git a/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts b/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts index e9a232102..eee0d935a 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts @@ -1258,6 +1258,20 @@ export function getConfigCommandHandlers(): CommandHandlerTable { command: new AgentToggleCommandHandler(AgentToggle.Command), agent: new AgentToggleCommandHandler(AgentToggle.Agent), request: new ConfigRequestCommandHandler(), + match: { + description: "Configure match behavior", + commands: { + grammar: getToggleHandlerTable( + "grammar cache usage", + async (context, enable: boolean) => { + await changeContextConfig( + { cache: { grammar: enable } }, + context, + ); + }, + ), + }, + }, translation: configTranslationCommandHandlers, explainer: configExplainerCommandHandlers, execution: configExecutionCommandHandlers, diff --git a/ts/packages/dispatcher/src/context/system/handlers/installCommandHandlers.ts b/ts/packages/dispatcher/src/context/system/handlers/installCommandHandlers.ts index c83a826f0..d754364d1 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/installCommandHandlers.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/installCommandHandlers.ts @@ -82,7 +82,10 @@ export class UninstallCommandHandler implements CommandHandler { const name = params.args.name; installer.uninstall(name); - await systemContext.agents.removeAgent(name); + await systemContext.agents.removeAgent( + name, + systemContext.agentCache.grammarStore, + ); displayResult(`Agent '${name}' uninstalled.`, context); } diff --git a/ts/packages/dispatcher/src/execute/pendingActions.ts b/ts/packages/dispatcher/src/execute/pendingActions.ts index 7d16f7d1a..42f20c808 100644 --- a/ts/packages/dispatcher/src/execute/pendingActions.ts +++ b/ts/packages/dispatcher/src/execute/pendingActions.ts @@ -9,6 +9,7 @@ import { getPropertyType, resolveTypeReference, resolveUnionType, + ActionSchemaEntityTypeDefinition, } from "action-schema"; import { ExecutableAction, @@ -38,7 +39,6 @@ import { SearchSelectExpr } from "knowpro"; import { conversation as kp } from "knowledge-processor"; import { getObjectProperty } from "common-utils"; import { ActionSchemaFile } from "../translation/actionConfigProvider.js"; -import { ActionSchemaEntityTypeDefinition } from "../../../actionSchema/dist/type.js"; import { getActionParametersType } from "../translation/actionSchemaUtils.js"; import { isPendingRequestAction } from "../translation/pendingRequest.js"; diff --git a/ts/packages/dispatcher/src/execute/sessionContext.ts b/ts/packages/dispatcher/src/execute/sessionContext.ts index dc11bacc3..f04699ecf 100644 --- a/ts/packages/dispatcher/src/execute/sessionContext.ts +++ b/ts/packages/dispatcher/src/execute/sessionContext.ts @@ -61,7 +61,10 @@ export function createSessionContext( `Permission denied: dynamic agent '${agentName}' not added by this agent`, ); } - return context.agents.removeAgent(agentName); + return context.agents.removeAgent( + agentName, + context.agentCache.grammarStore, + ); }) : () => { throw new Error("Permission denied: cannot remove dynamic agent"); diff --git a/ts/packages/dispatcher/src/translation/interpretRequest.ts b/ts/packages/dispatcher/src/translation/interpretRequest.ts index 30e1ae6d8..6e59c7d9d 100644 --- a/ts/packages/dispatcher/src/translation/interpretRequest.ts +++ b/ts/packages/dispatcher/src/translation/interpretRequest.ts @@ -61,7 +61,7 @@ export type InterpretResult = { requestAction: RequestAction; elapsedMs: number; fromUser: boolean; - fromCache: boolean; + fromCache: "construction" | "grammar" | false; tokenUsage?: ai.CompletionUsageStats | undefined; }; @@ -256,7 +256,7 @@ export async function interpretRequest( const { requestAction, replacedAction } = await confirmTranslation( translateResult.elapsedMs, - translateResult.type === "match" + translateResult.type !== "translate" ? unicodeChar.constructionSign : DispatcherEmoji, translateResult.requestAction, @@ -286,7 +286,8 @@ export async function interpretRequest( elapsedMs: translateResult.elapsedMs, requestAction, fromUser: replacedAction !== undefined, - fromCache: translateResult.type === "match", + fromCache: + translateResult.type === "translate" ? false : translateResult.type, tokenUsage, }; } diff --git a/ts/packages/dispatcher/src/translation/matchRequest.ts b/ts/packages/dispatcher/src/translation/matchRequest.ts index e33c4cac7..9c6da9f5e 100644 --- a/ts/packages/dispatcher/src/translation/matchRequest.ts +++ b/ts/packages/dispatcher/src/translation/matchRequest.ts @@ -168,8 +168,8 @@ export async function matchRequest( activeSchemas?: string[], ): Promise { const systemContext = context.sessionContext.agentContext; - const constructionStore = systemContext.agentCache.constructionStore; - if (!constructionStore.isEnabled()) { + const agentCache = systemContext.agentCache; + if (!agentCache.isEnabled()) { return undefined; } const startTime = performance.now(); @@ -191,7 +191,7 @@ export async function matchRequest( entityWildcard: config.cache.matchEntityWildcard, rejectReferences: config.explainer.filter.reference.list, }; - const matches = constructionStore.match(request, { + const matches = agentCache.match(request, { ...matchConfig, namespaceKeys: systemContext.agentCache.getNamespaceKeys( activeSchemaNames, @@ -208,15 +208,16 @@ export async function matchRequest( } return { - type: "match", + type: match.type, requestAction: match.match, elapsedMs, config: { ...matchConfig, explainerName: systemContext.agentCache.explainerName, }, + // For logging allMatches: matches.map((m) => { - const { construction: _, match, ...rest } = m; + const { match, ...rest } = m; return { action: match.actions, ...rest }; }), }; diff --git a/ts/packages/dispatcher/src/translation/translateRequest.ts b/ts/packages/dispatcher/src/translation/translateRequest.ts index d5f6994b9..2d0c17de1 100644 --- a/ts/packages/dispatcher/src/translation/translateRequest.ts +++ b/ts/packages/dispatcher/src/translation/translateRequest.ts @@ -721,7 +721,7 @@ export type TranslationResult = { requestAction: RequestAction; elapsedMs: number; - type: "translate" | "match"; + type: "translate" | "construction" | "grammar"; config: any; allMatches?: any; }; diff --git a/ts/packages/shell/src/renderer/src/messageContainer.ts b/ts/packages/shell/src/renderer/src/messageContainer.ts index 62187d17d..bb9d8c943 100644 --- a/ts/packages/shell/src/renderer/src/messageContainer.ts +++ b/ts/packages/shell/src/renderer/src/messageContainer.ts @@ -617,7 +617,7 @@ export class MessageContainer { const fromCache = data.fromCache; const timestamp = data.time; const cachePart = fromCache - ? "Translated by cache match" + ? `Translated by ${fromCache}` : "Translated by model"; let message: string; let color: string; diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index b64e8b5b2..8cc0edb80 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -845,6 +845,25 @@ importers: specifier: ~5.4.5 version: 5.4.5 + packages/actionGrammarCompiler: + dependencies: + '@oclif/core': + specifier: ^4.2.10 + version: 4.5.2 + '@oclif/plugin-help': + specifier: ^6 + version: 6.2.26 + action-grammar: + specifier: workspace:* + version: link:../actionGrammar + devDependencies: + rimraf: + specifier: ^6.0.1 + version: 6.0.1 + typescript: + specifier: ~5.4.5 + version: 5.4.5 + packages/actionSchema: dependencies: debug: @@ -1866,6 +1885,9 @@ importers: '@types/spotify-api': specifier: ^0.0.25 version: 0.0.25 + action-grammar-compiler: + specifier: workspace:* + version: link:../../actionGrammarCompiler action-schema-compiler: specifier: workspace:* version: link:../../actionSchemaCompiler @@ -2690,6 +2712,9 @@ importers: '@typeagent/agent-sdk': specifier: workspace:* version: link:../agentSdk + action-grammar: + specifier: workspace:* + version: link:../actionGrammar action-schema: specifier: workspace:* version: link:../actionSchema