Create validation plans from instanced criteria, send to editor for e…

…valuation, get results. Needs clean up.
microsoft · Jan 24, 2024 · 60e1072 · 60e1072
1 parent 2b6462c
commit 60e1072
Show file tree

Hide file tree

Showing 16 changed files with 302 additions and 69 deletions.
diff --git a/docs/teachertool/catalog-shared-test.json b/docs/teachertool/catalog-shared-test.json
@@ -14,6 +14,27 @@
                     "path": "checks[0].count"
                 }
             ]
+        },
+        {
+            "id": "59AAC5BA-B0B3-4389-AA90-1E767EFA8563",
+            "use": "block_used_n_times",
+            "template": "${block_id} used ${count} times",
+            "description": "This block was used the specified number of times in your project.",
+            "docPath": "/teachertool",
+            "params": [
+                {
+                    "name": "block_id",
+                    "type": "string",
+                    "picker": "blockSelector",
+                    "path": "checks[0].blocks[:clear&:append]"
+                },
+                {
+                    "name": "count",
+                    "type": "number",
+                    "default": 1,
+                    "path": "checks[0].count"
+                }
+            ]
         }
     ]
 }
diff --git a/docs/teachertool/catalog-shared.json b/docs/teachertool/catalog-shared.json
@@ -1,31 +1,16 @@
 {
     "criteria": [
         {
-            "id": "A7C78825-51B5-45B0-9E42-2AAC767D4E02",
-            "use": "processes_and_displays_array_in_loop",
-            "template": "One kind of loop to process/display the information in an array",
+            "id": "B3FD8B3D-61B4-42F4-B0EF-64BD7D62CDAB",
+            "use": "two_different_loops",
+            "template": "Two different kinds of loops used.",
             "docPath": "/teachertool"
         },
         {
-            "id": "59AAC5BA-B0B3-4389-AA90-1E767EFA8563",
-            "use": "block_used_n_times",
-            "template": "${block_id} used ${count} times",
-            "description": "This block was used the specified number of times in your project.",
-            "docPath": "/teachertool",
-            "params": [
-                {
-                    "name": "block_id",
-                    "type": "string",
-                    "picker": "blockSelector",
-                    "path": "checks[0].blocks[:clear&:append]"
-                },
-                {
-                    "name": "count",
-                    "type": "number",
-                    "default": 1,
-                    "path": "checks[0].count"
-                }
-            ]
+            "id": "49262A2B-C02A-43A2-BAD5-FCAC6AE9D464",
+            "use": "custom_function_called",
+            "template": "A custom function exists and gets called.",
+            "docPath": "/teachertool"
         }
     ]
 }
diff --git a/docs/teachertool/validator-plans-shared-test.json b/docs/teachertool/validator-plans-shared-test.json
@@ -0,0 +1,17 @@
+{
+    "validatorPlans": [
+        {
+            ".desc": "Block used n times (to be filled in by the user)",
+            "name": "block_used_n_times",
+            "threshold": 1,
+            "checks": [
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "": 0
+                    }
+                }
+            ]
+        }
+    ]
+}
diff --git a/docs/teachertool/validator-plans-shared.json b/docs/teachertool/validator-plans-shared.json
@@ -0,0 +1,35 @@
+{
+    "validatorPlans": [
+        {
+            ".desc": "Two different kinds of loops used",
+            "name": "two_different_loops",
+            "threshold": 2,
+            "checks": [
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "controls_repeat_ext": 1
+                    }
+                },
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "device_while": 1
+                    }
+                },
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "pxt_controls_for": 1
+                    }
+                },
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "pxt_controls_for_of": 1
+                    }
+                }
+            ]
+        }
+    ]
+}
diff --git a/pxtblocks/code-validation/evaluationResult.ts b/pxtblocks/code-validation/evaluationResult.ts
@@ -1,5 +1,5 @@
 namespace pxt.blocks {
   export interface EvaluationResult {
-   blockIdResults: pxt.Map<boolean>;
+   results: pxt.Map<boolean>;
   }
 }
diff --git a/pxtblocks/code-validation/rubricCriteria.ts b/pxtblocks/code-validation/rubricCriteria.ts
@@ -31,20 +31,41 @@ namespace pxt.blocks {
         return requiredBlockCounts;
     }
 
-    export function validateProject(usedBlocks: Blockly.Block[], rubric: string): EvaluationResult {
-        const rubricData = parseRubric(rubric);
-        const finalResult: pxt.Map<boolean> = {};
-        rubricData.criteria.forEach((criteria: RubricCriteria) => {
-            (criteria as BlockCheckCriteria).blockRequirements.forEach((blockSet) => {
-                const result = validateBlockSet(usedBlocks, blockSet);
-                Object.keys(result).forEach((blockId) => {
-                    finalResult[blockId] = result[blockId];
+    async function runValidatorPlan(usedBlocks: Blockly.Block[], plan: ValidatorPlanWithId): Promise<boolean> {
+      const checkRuns = plan.checks.map((check) => new Promise<boolean>((resolve) => {
+            switch (check.validator) {
+              case "blocksExist":
+                const blockExistsCheck = check as BlocksExistValidatorCheck;
+                const blockResults = validateBlocksExist({
+                  usedBlocks,
+                  requiredBlockCounts: blockExistsCheck.blockCounts,
                 });
-            });
-        });
-        return { blockIdResults: finalResult } as EvaluationResult;
+                const success = blockResults.disabledBlocks.length === 0 && blockResults.missingBlocks.length === 0 && blockResults.insufficientBlocks.length === 0;
+                resolve(success);
+                break;
+              default:
+                console.error(`Unrecognized validator: ${check.validator}`);
+                break;
+            }
+            resolve(false);
+      }));
+
+      const results = await Promise.all(checkRuns);
+      const successCount = results.filter((r) => r).length;
+      return successCount >= plan.threshold;
     }
 
+    export async function validateProject(usedBlocks: Blockly.Block[], rubric: string): Promise<EvaluationResult> {
+        const validatorPlans = JSON.parse(rubric) as ValidatorPlanWithId[];
+
+        const finalResult: pxt.Map<boolean> = {};
+        for (const plan of validatorPlans) {
+            const planResult = await runValidatorPlan(usedBlocks, plan);
+            finalResult[plan.id] = planResult;
+        }
+
+        return { results: finalResult } as EvaluationResult;
+    }
 
     function validateBlockSet(usedBlocks: Blockly.Block[], blockSet: BlockSet): pxt.Map<boolean> {
         const requiredBlockCounts = blockSetToRequiredBlockCounts(blockSet);

diff --git a/pxtblocks/code-validation/validatorPlans.ts b/pxtblocks/code-validation/validatorPlans.ts
@@ -0,0 +1,63 @@
+namespace pxt.blocks {
+    // A set of validation checks (with inputs) to run for a given criteria.
+    export interface ValidatorPlan {
+        name: string;
+        threshold: number;
+        checks: ValidatorCheckBase[];
+    }
+
+    // Used so a specific plan can be identified when sending results.
+    export interface ValidatorPlanWithId extends ValidatorPlan {
+        id: string;
+    }
+
+    // Base class to describes a single validation check to run (with inputs).
+    // Each type of validation will need to implement its own ValidatorCheck based on this.
+    export interface ValidatorCheckBase {
+        validator: string;
+    }
+
+    // Inputs for "Blocks Exist" validation.
+    export interface BlocksExistValidatorCheck extends ValidatorCheckBase {
+        validator: "blocksExist";
+        blockCounts: pxt.Map<number>;
+    }
+
+    function isBlocksExistValidatorCheck(check: ValidatorCheckBase): check is BlocksExistValidatorCheck {
+        return check.validator === "blocksExist";
+    }
+
+    // export function getValidatorCheck(check: ValidatorCheckBase): ValidatorCheckBase {
+    //     switch (check.validator) {
+    //         case "blocksExist":
+    //             return ;
+    //         default:
+    //             console.error(`Unrecognized validator: ${check.validator}`);
+    //             return undefined;
+    //     }
+    // }
+
+    // export function parseValidatorPlans(plans: string): ValidatorPlan[] {
+    //     let plansObj;
+    //     try {
+    //         plansObj = JSON.parse(plans);
+    //     } catch (e) {
+    //         console.error(`Error parsing validator plans. ${e}`);
+    //         return undefined;
+    //     }
+
+    //     const finalPlans = plansObj.map(plan => {
+    //         const checks: ValidatorCheckBase[] = plan.checks.map((c: ValidatorCheckBase) => {
+    //             return getValidatorCheck(c);
+    //         }).filter((r: ValidatorCheckBase) => !!r);
+
+    //         return {
+    //             name: plan.name,
+    //             threshold: plan.threshold,
+    //             checks: checks
+    //         } as ValidatorPlan;
+    //     });
+
+    //     return finalPlans;
+    // }
+}
diff --git a/teachertool/src/App.tsx b/teachertool/src/App.tsx
@@ -14,6 +14,7 @@ import EvalResultDisplay from "./components/EvalResultDisplay";
 import { loadCatalog } from "./transforms/loadCatalog";
 import ActiveRubricDisplay from "./components/ActiveRubricDisplay";
 import CatalogModal from "./components/CatalogModal";
+import { loadValidatorPlans } from "./transforms/loadValidatorPlans";
 
 
 function App() {
@@ -27,8 +28,9 @@ function App() {
             // Init subsystems.
             NotificationService.initialize();
 
-            // Load criteria catalog
+            // Load catalog and validator plans into state.
             loadCatalog();
+            loadValidatorPlans();
         }
     }, [ready]);
 

diff --git a/teachertool/src/components/DebugInput.tsx b/teachertool/src/components/DebugInput.tsx
@@ -11,11 +11,10 @@ interface IProps {}
 
 const DebugInput: React.FC<IProps> = ({}) => {
     const [shareLink, setShareLink] = useState("https://arcade.makecode.com/S70821-26848-68192-30094");
-    const [rubric, setRubric] = useState("");
 
     const evaluate = async () => {
         await loadProjectMetadataAsync(shareLink);
-        await runEvaluateAsync(rubric);
+        await runEvaluateAsync();
     }
 
     return (
@@ -29,14 +28,6 @@ const DebugInput: React.FC<IProps> = ({}) => {
                     initialValue={shareLink}
                     onChange={setShareLink} />
             </div>
-            <div className="rubric-json-input-container">
-                {lf("Rubric:")}
-                <Textarea
-                    id="rubricJsonInput"
-                    className="json-input"
-                    rows={20}
-                    onChange={setRubric} />
-            </div>
             <Button id="evaluateSingleProjectButton" className="primary" onClick={evaluate} title={"Evaluate"} label={lf("Evaluate")} />
         </div>
     )

diff --git a/teachertool/src/components/EvalResultDisplay.tsx b/teachertool/src/components/EvalResultDisplay.tsx
@@ -2,23 +2,30 @@
 
 import { useContext, useState } from "react";
 import { AppStateContext } from "../state/appStateContext";
+import { getCatalogCriteriaWithId } from "../utils";
 
 interface IProps {}
 
 const EvalResultDisplay: React.FC<IProps> = ({}) => {
     const { state: teacherTool } = useContext(AppStateContext);
 
+    function getTemplateFromCriteriaInstanceId(instanceId: string): string {
+        const catalogCriteriaId = teacherTool.selectedCriteria?.find(criteria => criteria.instanceId === instanceId)?.catalogCriteriaId;
+        if (!catalogCriteriaId) return "";
+        return getCatalogCriteriaWithId(catalogCriteriaId)?.template ?? "";
+    }
+
     return (
         <>
             {teacherTool.projectMetadata && (
                 <div className="eval-results-container">
                     <h3>{lf("Project: {0}", teacherTool.projectMetadata.name)}</h3>
                     {teacherTool.currentEvalResult === undefined && <div className="common-spinner" />}
-                    {Object.keys(teacherTool.currentEvalResult?.blockIdResults ?? {}).map((id) => {
-                        const result = teacherTool.currentEvalResult?.blockIdResults[id];
+                    {Object.keys(teacherTool.currentEvalResult?.results ?? {}).map((id) => {
+                        const result = teacherTool.currentEvalResult?.results[id];
                         return (
                             <div className="result-block-id" key={id}>
-                                <p className="block-id-label">{id}:</p>
+                                <p className="block-id-label">{getTemplateFromCriteriaInstanceId(id)}:</p>
                                 <p className={result ? "positive-text" : "negative-text"}>{result ? "passed" : "failed"}</p>
                             </div>
                         );

diff --git a/teachertool/src/state/actions.ts b/teachertool/src/state/actions.ts
@@ -52,6 +52,11 @@ type HideModal = ActionBase & {
     type: "HIDE_MODAL";
 };
 
+type SetValidatorPlans = ActionBase & {
+    type: "SET_VALIDATOR_PLANS";
+    plans: pxt.blocks.ValidatorPlan[] | undefined;
+};
+
 /**
  * Union of all actions
  */
@@ -65,7 +70,8 @@ export type Action =
     | AddCriteriaInstances
     | RemoveCriteriaInstance
     | ShowModal
-    | HideModal;
+    | HideModal
+    | SetValidatorPlans;
 
 /**
  * Action creators
@@ -116,6 +122,11 @@ const hideModal = (): HideModal => ({
     type: "HIDE_MODAL",
 });
 
+const setValidatorPlans = (plans: pxt.blocks.ValidatorPlan[] | undefined): SetValidatorPlans => ({
+    type: "SET_VALIDATOR_PLANS",
+    plans,
+});
+
 export {
     postNotification,
     removeNotification,
@@ -125,5 +136,6 @@ export {
     addCriteriaInstances,
     removeCriteriaInstance,
     showModal,
-    hideModal
+    hideModal,
+    setValidatorPlans
 };
diff --git a/teachertool/src/state/reducer.ts b/teachertool/src/state/reducer.ts
@@ -73,6 +73,12 @@ export default function reducer(state: AppState, action: Action): AppState {
                 modal: undefined,
             };
         }
+        case "SET_VALIDATOR_PLANS": {
+            return {
+                ...state,
+                validatorPlans: action.plans,
+            };
+        }
     }
 
     return state;

diff --git a/teachertool/src/state/state.ts b/teachertool/src/state/state.ts
@@ -8,6 +8,7 @@ export type AppState = {
     catalog: pxt.blocks.CatalogCriteria[] | undefined;
     selectedCriteria: pxt.blocks.CriteriaInstance[];
     modal: ModalType | undefined;
+    validatorPlans: pxt.blocks.ValidatorPlan[] | undefined;
 };
 
 export const initialAppState: AppState = {
@@ -16,5 +17,6 @@ export const initialAppState: AppState = {
     projectMetadata: undefined,
     catalog: undefined,
     selectedCriteria: [],
-    modal: undefined
+    modal: undefined,
+    validatorPlans: undefined,
 };