Merge pull request #55 from salesforcecli/sh/agent-generate-spec-v2

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2024, Salesforce.com, Inc.
+Copyright (c) 2025, Salesforce.com, Inc.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

command-snapshot.json

@@ -35,6 +35,29 @@
     ],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:generate:spec-v2",
+    "flagAliases": [],
+    "flagChars": ["o", "t"],
+    "flags": [
+      "api-version",
+      "company-description",
+      "company-name",
+      "company-website",
+      "flags-dir",
+      "grounding-context",
+      "json",
+      "max-topics",
+      "output-file",
+      "prompt-template",
+      "role",
+      "spec",
+      "target-org",
+      "type"
+    ],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:generate:testset",

messages/agent.generate.spec-v2.md

@@ -0,0 +1,73 @@
+# summary
+
+Generate an agent spec, which is the list of jobs that the agent performs.
+
+# description
+
+When using Salesforce CLI to create an agent in your org, you can choose to generate a YAML-formatted agent spec file with this command as a first step.
+
+An agent spec is a list of jobs and descriptions that capture what the agent can do. Use flags such as --role and --company-description to provide details about your company and the role that the agent plays in your company; you can also enter the information interactively if you prefer. When you then execute this command, the large language model (LLM) associated with your org uses the information to generate the list of jobs that the agent most likely performs. We recommend that you provide good details for --role, --company-description, etc, so that the LLM can generate the best and most relevant list of jobs and descriptions. Once generated, you can edit the spec file; for example, you can remove jobs that don't apply to your agent.
+
+When your agent spec is ready, you then create the agent in your org by specifying the agent spec file to the --job-spec flag of the "agent create" CLI command.
+
+# flags.type.summary
+
+Type of agent to create.
+
+# flags.role.summary
+
+Role of the agent.
+
+# flags.company-name.summary
+
+Name of your company.
+
+# flags.company-description.summary
+
+Description of your company.
+
+# flags.company-website.summary
+
+Website URL of your company.
+
+# flags.output-file.summary
+
+Path for the generated agent spec file (yaml); can be an absolute or relative path.
+
+# flags.max-topics.summary
+
+Maximum number of agent job topics to generate in the spec; default is 10.
+
+# flags.prompt-template.summary
+
+Developer name of a customized prompt template to use instead of the default.
+
+# flags.grounding-context.summary
+
+Context information to be used with the customized prompt template.
+
+# flags.spec.summary
+
+Spec file (yaml) to use as input to the command.
+
+# examples
+
+- Create an agent spec for your default org in the default location and use flags to specify the agent's role and your company details:
+
+  <%= config.bin %> <%= command.id %> --type customer --role "Assist users in navigating and managing bookings" --company-name "Coral Cloud" --company-description "Resort that manages guests and their reservations and experiences"
+
+- Create an agent spec by being prompted for role and company details interactively; write the generated file to the "specs" directory and use the org with alias "my-org":
+
+  <%= config.bin %> <%= command.id %> --output-dir specs --target-org my-org
+
+# error.invalidAgentType
+
+agentType must be either "customer" or "internal". Found: [%s]
+
+# error.invalidMaxTopics
+
+maxNumOfTopics must be a number greater than 0. Found: [%s]
+
+# error.missingRequiredFlags
+
+Missing required flags: %s

mocks/einstein_ai-evaluations_runs.json

@@ -0,0 +1,4 @@
+{
+  "aiEvaluationId": "4KBSM000000003F4AQ",
+  "status": "NEW"
+}

mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json

@@ -0,0 +1,4 @@
+{
+  "status": "IN_PROGRESS",
+  "startTime": "2024-11-13T15:00:00.000Z"
+}

mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json

@@ -0,0 +1,4 @@
+{
+  "status": "IN_PROGRESS",
+  "startTime": "2024-11-13T15:00:00.000Z"
+}

mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json

@@ -0,0 +1,4 @@
+{
+  "status": "COMPLETED",
+  "startTime": "2024-11-13T15:00:00.000Z"
+}

mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json

@@ -0,0 +1,96 @@
+{
+  "status": "COMPLETED",
+  "startTime": "2024-11-28T12:00:00Z",
+  "endTime": "2024-11-28T12:05:00Z",
+  "errorMessage": null,
+  "testCases": [
+    {
+      "status": "COMPLETED",
+      "number": 1,
+      "startTime": "2024-11-28T12:00:10Z",
+      "endTime": "2024-11-28T12:00:20Z",
+      "generatedData": {
+        "type": "AGENT",
+        "actionsSequence": ["Action1", "Action2"],
+        "outcome": "Success",
+        "topic": "Mathematics",
+        "inputTokensCount": 50,
+        "outputTokensCount": 55
+      },
+      "expectationResults": [
+        {
+          "name": "topic_sequence_match",
+          "actualValue": "Result A",
+          "expectedValue": "Result A",
+          "score": 1.0,
+          "result": "Passed",
+          "metricLabel": "Accuracy",
+          "metricExplainability": "Measures the correctness of the result.",
+          "status": "Completed",
+          "startTime": "2024-11-28T12:00:12Z",
+          "endTime": "2024-11-28T12:00:13Z",
+          "errorCode": null,
+          "errorMessage": null
+        },
+        {
+          "name": "action_sequence_match",
+          "actualValue": "Result B",
+          "expectedValue": "Result B",
+          "score": 0.9,
+          "result": "Passed",
+          "metricLabel": "Precision",
+          "metricExplainability": "Measures the precision of the result.",
+          "status": "Completed",
+          "startTime": "2024-11-28T12:00:14Z",
+          "endTime": "2024-11-28T12:00:15Z",
+          "errorCode": null,
+          "errorMessage": null
+        }
+      ]
+    },
+    {
+      "status": "ERROR",
+      "number": 2,
+      "startTime": "2024-11-28T12:00:30Z",
+      "endTime": "2024-11-28T12:00:40Z",
+      "generatedData": {
+        "type": "AGENT",
+        "actionsSequence": ["Action3", "Action4"],
+        "outcome": "Failure",
+        "topic": "Physics",
+        "inputTokensCount": 60,
+        "outputTokensCount": 50
+      },
+      "expectationResults": [
+        {
+          "name": "topic_sequence_match",
+          "actualValue": "Result C",
+          "expectedValue": "Result D",
+          "score": 0.5,
+          "result": "Failed",
+          "metricLabel": "Accuracy",
+          "metricExplainability": "Measures the correctness of the result.",
+          "status": "Completed",
+          "startTime": "2024-11-28T12:00:32Z",
+          "endTime": "2024-11-28T12:00:33Z",
+          "errorCode": null,
+          "errorMessage": "Expected \"Result D\" but got \"Result C\"."
+        },
+        {
+          "name": "topic_sequence_match",
+          "actualValue": "Result C",
+          "expectedValue": "Result D",
+          "score": 0.5,
+          "result": "Failed",
+          "metricLabel": "Accuracy",
+          "metricExplainability": "Measures the correctness of the result.",
+          "status": "Completed",
+          "startTime": "2024-11-28T12:00:32Z",
+          "endTime": "2024-11-28T12:00:33Z",
+          "errorCode": null,
+          "errorMessage": "Expected \"Result D\" but got \"Result C\"."
+        }
+      ]
+    }
+  ]
+}

package.json

@@ -11,14 +11,15 @@
     "@inquirer/select": "^4.0.1",
     "@oclif/core": "^4",
     "@oclif/multi-stage-output": "^0.7.12",
-    "@salesforce/agents": "^0.5.2",
+    "@salesforce/agents": "^0.6.0",
     "@salesforce/core": "^8.8.0",
     "@salesforce/kit": "^3.2.1",
     "@salesforce/sf-plugins-core": "^12.1.0",
     "ansis": "^3.3.2",
     "ink": "^5.0.1",
     "ink-text-input": "^6.0.0",
-    "react": "^18.3.1"
+    "react": "^18.3.1",
+    "yaml": "^2.7.0"
   },
   "devDependencies": {
     "@oclif/plugin-command-snapshot": "^5.2.19",

schemas/agent-generate-spec__v2.json

@@ -0,0 +1,72 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentCreateSpecResult",
+  "definitions": {
+    "AgentCreateSpecResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "topics": {
+          "$ref": "#/definitions/DraftAgentTopics"
+        },
+        "agentType": {
+          "type": "string",
+          "enum": ["customer", "internal"],
+          "description": "Internal type is copilots; used by customers' employees. Customer type is agents; used by customers' customers."
+        },
+        "role": {
+          "type": "string"
+        },
+        "companyName": {
+          "type": "string"
+        },
+        "companyDescription": {
+          "type": "string"
+        },
+        "companyWebsite": {
+          "type": "string"
+        },
+        "maxNumOfTopics": {
+          "type": "number",
+          "description": "The maximum number of topics to create in the spec. Default is 10."
+        },
+        "promptTemplateName": {
+          "type": "string",
+          "description": "Developer name of the prompt template."
+        },
+        "groundingContext": {
+          "type": "string",
+          "description": "Context info to be used in customized prompt template"
+        },
+        "isSuccess": {
+          "type": "boolean"
+        },
+        "errorMessage": {
+          "type": "string"
+        },
+        "specPath": {
+          "type": "string"
+        }
+      },
+      "required": ["agentType", "companyDescription", "companyName", "isSuccess", "role", "topics"]
+    },
+    "DraftAgentTopics": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          }
+        },
+        "required": ["name", "description"],
+        "additionalProperties": false
+      },
+      "minItems": 1,
+      "maxItems": 1
+    }
+  }
+}