feat: add a batch assignment grader.

README.md

@@ -1,3 +1,95 @@
 # Canvas Automated Grader
 
-A Python automatic grader that evaluates JSON files.
+A Python automatic grader that evaluates JSON responses from the API end point https://api.openai.lawrencemcdaniel.com/examples/default-marv-sarcastic-chat. Verifies the structural integrity of the JSON response and implements a graduated Rubric based on how closely the response submitted matches this successful [test case](./grader/tests/events/correct.json).
+
+## Installation
+
+```console
+git clone https://github.com/lpm0073/automatic-grader.git
+cd automatic-grader
+make init
+make activate
+```
+
+## Usage
+
+```console
+python3 -m grader.batch 'path/to/homework/json/files/'
+```
+
+### Expected output
+
+```console
+% done! Graded 10 assignments. Output files are in path/to/homework/json/files/out
+```
+
+```json
+{
+  "grade": 100,
+  "message": "Great job!",
+  "message_type": "Success"
+}
+```
+
+````json
+{
+    "grade": 80,
+    "message": "The assignment's statusCode must be 200. received: 403",
+    "message_type": "ResponseFailedError"
+}```
+
+```json
+{
+    "grade": 90,
+    "message": "The assignment's statusCode must be an integer. received: <class 'str'>",
+    "message_type": "IncorrectResponseTypeError"
+}```
+
+```json
+{
+    "grade": 70,
+    "message": "The assignment is missing one or more required keys. missing: {'type', 'example', 'additional_kwargs'}",
+    "message_type": "InvalidResponseStructureError"
+}```
+
+```json
+{
+    "grade": 70,
+    "message": "The messages list must contain at least two elements. messages: [{'content': \"Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.\", 'additional_kwargs': {}, 'type': 'ai', 'example': False}]",
+    "message_type": "InvalidResponseStructureError"
+}
+````
+
+```json
+{
+  "grade": 70,
+  "message": "All elements in the messages list must be dictionaries. messages: ['bad', 'data']",
+  "message_type": "InvalidResponseStructureError"
+}
+```
+
+```json
+{
+  "grade": 70,
+  "message": "The request_meta_data key lambda_langchain must exist. request_meta_data: {}",
+  "message_type": "InvalidResponseStructureError"
+}
+```
+
+## Contributing
+
+This project uses a mostly automated pull request and unit testing process. See the resources in .github for additional details. You additionally should ensure that pre-commit is installed and working correctly on your dev machine by running the following command from the root of the repo.
+
+```console
+pre-commit run --all-files
+```
+
+### Developer setup
+
+```console
+git clone https://github.com/lpm0073/automatic-grader.git
+cd automatic-grader
+make init
+make activate
+make test
+```

grader/__version__.py

@@ -1,3 +1,2 @@
 # -*- coding: utf-8 -*-
-__version__ = '1.1.6'
-
+__version__ = "1.1.6"

grader/batch.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""Batch grader for course assignments."""
+import argparse
+import glob
+import json
+import os
+
+from .grader import AutomatedGrader
+
+
+def main(filepath: str = None, output_folder: str = "out"):
+    """Grade an assignment."""
+    graded = 0
+    if filepath is None:
+        print("""usage: grade_assignment.py [-h] filepath""")
+
+    OUTPUT_FILE_PATH = os.path.join(filepath, output_folder)
+    if not os.path.exists(OUTPUT_FILE_PATH):
+        os.makedirs(OUTPUT_FILE_PATH)
+
+    assignments = glob.glob(os.path.join(filepath, "*.json"))
+    for assignment_filename in assignments:
+        with open(assignment_filename, "r", encoding="utf-8") as f:
+            try:
+                assignment = json.load(f)
+            except json.JSONDecodeError:
+                print(f"warning: invalid JSON in assignment_filename: {assignment_filename}")
+                assignment = f.read()
+        grader = AutomatedGrader(assignment)
+        grade = grader.grade()
+        with open(
+            os.path.join(OUTPUT_FILE_PATH, f"{os.path.basename(assignment_filename)}"), "w", encoding="utf-8"
+        ) as f:
+            json.dump(grade, f, indent=4)
+            graded += 1
+
+    print(f"done! Graded {graded} assignments. Output files are in {OUTPUT_FILE_PATH}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Grade a set of homework assignments.")
+    parser.add_argument("filepath", type=str, help="The path to the homework files to grade.")
+    parser.add_argument(
+        "output_folder",
+        type=str,
+        nargs="?",  # optional
+        default="out",
+        help="The name of the subfolder where graded assignments will be saved.",
+    )
+
+    args = parser.parse_args()
+
+    main(args.filepath, args.output_folder)

grader/grader.py

@@ -2,6 +2,7 @@
 """Provide a class for grading a submission against an assignment.""" ""
 
 import json
+import os
 
 from .exceptions import (
     IncorrectResponseTypeError,
@@ -11,7 +12,10 @@
 )
 
 
+HERE = os.path.abspath(os.path.dirname(__file__))
 REQUIRED_KEYS_SPEC = "required-keys.json"
+REQUIRED_KEYS_PATH = os.path.join(HERE, "data", REQUIRED_KEYS_SPEC)
+
 HUMAN_PROMPT = {"content": "a prompt from a human", "additional_kwargs": {}, "type": "human", "example": False}
 AI_RESPONSE = {"content": "a response from the AI", "additional_kwargs": {}, "type": "ai", "example": False}
 
@@ -22,7 +26,7 @@ class AutomatedGrader:
 
     def __init__(self, assignment):
         self.assignment = assignment
-        with open("data/" + REQUIRED_KEYS_SPEC, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
+        with open(REQUIRED_KEYS_PATH, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
             self.required_keys = json.load(f)
 
     def validate_keys(self, subject, control):

grader/tests/test_responses.py

@@ -16,7 +16,7 @@ class TestGrader:
 
     def test_success(self):
         """Test a valid successful submission."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct.json")
+        assignment = get_event("tests/events/correct.json")
         automated_grader = AutomatedGrader(assignment=assignment)
         grade = automated_grader.grade()
 
@@ -32,7 +32,7 @@ def test_success(self):
 
     def test_success_verbose(self):
         """Test a valid successful submission."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct-verbose.json")
+        assignment = get_event("tests/events/correct-verbose.json")
         automated_grader = AutomatedGrader(assignment=assignment)
         grade = automated_grader.grade()
 
@@ -58,7 +58,7 @@ def test_bad_data(self):
 
     def test_incorrect_response_type(self):
         """Test an assignment with an incorrect response type."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt")
+        assignment = get_event("tests/events/incorrect-response-type.txt")
         automated_grader = AutomatedGrader(assignment=assignment)
         grade = automated_grader.grade()
         print(grade)
@@ -68,7 +68,7 @@ def test_incorrect_response_type(self):
 
     def test_incorrect_response_statuscode(self):
         """Test an assignment with an incorrect response status code."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json")
+        assignment = get_event("tests/events/incorrect-response-status.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -77,7 +77,7 @@ def test_incorrect_response_statuscode(self):
 
     def test_incorrect_messages(self):
         """Test an assignment with an incorrect message."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-wrong-messages.json")
+        assignment = get_event("tests/events/wrong-messages.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -86,7 +86,7 @@ def test_incorrect_messages(self):
 
     def test_incorrect_data_type(self):
         """Test an assignment with an incorrect data type."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-type-error.json")
+        assignment = get_event("tests/events/type-error.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -95,7 +95,7 @@ def test_incorrect_data_type(self):
 
     def test_bad_message_01(self):
         """Test an assignment with an incorrect message."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-1.json")
+        assignment = get_event("tests/events/bad-message-1.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -104,7 +104,7 @@ def test_bad_message_01(self):
 
     def test_bad_message_02(self):
         """Test an assignment with an incorrect message."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-2.json")
+        assignment = get_event("tests/events/bad-message-2.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -113,7 +113,7 @@ def test_bad_message_02(self):
 
     def test_bad_message_03(self):
         """Test an assignment with an incorrect message."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-3.json")
+        assignment = get_event("tests/events/bad-message-3.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()
@@ -122,7 +122,7 @@ def test_bad_message_03(self):
 
     def test_bad_message_04(self):
         """Test an assignment with an incorrect message."""
-        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-4.json")
+        assignment = get_event("tests/events/bad-message-4.json")
         automated_grader = AutomatedGrader(assignment=assignment)
 
         grade = automated_grader.grade()