From e7df088c842e37f622aa4670f483448cdeb3495b Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Tue, 28 Nov 2023 07:49:53 -0600 Subject: [PATCH] feat: add a batch assignment grader. --- README.md | 94 ++++++++++++++++++- grader/__version__.py | 3 +- grader/batch.py | 53 +++++++++++ grader/grader.py | 6 +- ...-bad-message-1.json => bad-message-1.json} | 0 ...-bad-message-2.json => bad-message-2.json} | 0 ...-bad-message-3.json => bad-message-3.json} | 0 ...-bad-message-4.json => bad-message-4.json} | 0 ...rect-verbose.json => correct-verbose.json} | 0 ...el-homework1-correct.json => correct.json} | 0 ...us.json => incorrect-response-status.json} | 0 ...e-type.txt => incorrect-response-type.txt} | 0 ...ework1-type-error.json => type-error.json} | 0 ...rong-messages.json => wrong-messages.json} | 0 grader/tests/test_responses.py | 20 ++-- 15 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 grader/batch.py rename grader/tests/events/{lawrence-mcdaniel-homework1-bad-message-1.json => bad-message-1.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-bad-message-2.json => bad-message-2.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-bad-message-3.json => bad-message-3.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-bad-message-4.json => bad-message-4.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-correct-verbose.json => correct-verbose.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-correct.json => correct.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-incorrect-response-status.json => incorrect-response-status.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-incorrect-response-type.txt => incorrect-response-type.txt} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-type-error.json => type-error.json} (100%) rename grader/tests/events/{lawrence-mcdaniel-homework1-wrong-messages.json => wrong-messages.json} (100%) diff --git a/README.md b/README.md index f2dc2e7..0366d6f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,95 @@ # Canvas Automated Grader -A Python automatic grader that evaluates JSON files. +A Python automatic grader that evaluates JSON responses from the API end point https://api.openai.lawrencemcdaniel.com/examples/default-marv-sarcastic-chat. Verifies the structural integrity of the JSON response and implements a graduated Rubric based on how closely the response submitted matches this successful [test case](./grader/tests/events/correct.json). + +## Installation + +```console +git clone https://github.com/lpm0073/automatic-grader.git +cd automatic-grader +make init +make activate +``` + +## Usage + +```console +python3 -m grader.batch 'path/to/homework/json/files/' +``` + +### Expected output + +```console +% done! Graded 10 assignments. Output files are in path/to/homework/json/files/out +``` + +```json +{ + "grade": 100, + "message": "Great job!", + "message_type": "Success" +} +``` + +````json +{ + "grade": 80, + "message": "The assignment's statusCode must be 200. received: 403", + "message_type": "ResponseFailedError" +}``` + +```json +{ + "grade": 90, + "message": "The assignment's statusCode must be an integer. received: ", + "message_type": "IncorrectResponseTypeError" +}``` + +```json +{ + "grade": 70, + "message": "The assignment is missing one or more required keys. missing: {'type', 'example', 'additional_kwargs'}", + "message_type": "InvalidResponseStructureError" +}``` + +```json +{ + "grade": 70, + "message": "The messages list must contain at least two elements. messages: [{'content': \"Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.\", 'additional_kwargs': {}, 'type': 'ai', 'example': False}]", + "message_type": "InvalidResponseStructureError" +} +```` + +```json +{ + "grade": 70, + "message": "All elements in the messages list must be dictionaries. messages: ['bad', 'data']", + "message_type": "InvalidResponseStructureError" +} +``` + +```json +{ + "grade": 70, + "message": "The request_meta_data key lambda_langchain must exist. request_meta_data: {}", + "message_type": "InvalidResponseStructureError" +} +``` + +## Contributing + +This project uses a mostly automated pull request and unit testing process. See the resources in .github for additional details. You additionally should ensure that pre-commit is installed and working correctly on your dev machine by running the following command from the root of the repo. + +```console +pre-commit run --all-files +``` + +### Developer setup + +```console +git clone https://github.com/lpm0073/automatic-grader.git +cd automatic-grader +make init +make activate +make test +``` diff --git a/grader/__version__.py b/grader/__version__.py index c43dd67..f547066 100644 --- a/grader/__version__.py +++ b/grader/__version__.py @@ -1,3 +1,2 @@ # -*- coding: utf-8 -*- -__version__ = '1.1.6' - +__version__ = "1.1.6" diff --git a/grader/batch.py b/grader/batch.py new file mode 100644 index 0000000..f47ee02 --- /dev/null +++ b/grader/batch.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +"""Batch grader for course assignments.""" +import argparse +import glob +import json +import os + +from .grader import AutomatedGrader + + +def main(filepath: str = None, output_folder: str = "out"): + """Grade an assignment.""" + graded = 0 + if filepath is None: + print("""usage: grade_assignment.py [-h] filepath""") + + OUTPUT_FILE_PATH = os.path.join(filepath, output_folder) + if not os.path.exists(OUTPUT_FILE_PATH): + os.makedirs(OUTPUT_FILE_PATH) + + assignments = glob.glob(os.path.join(filepath, "*.json")) + for assignment_filename in assignments: + with open(assignment_filename, "r", encoding="utf-8") as f: + try: + assignment = json.load(f) + except json.JSONDecodeError: + print(f"warning: invalid JSON in assignment_filename: {assignment_filename}") + assignment = f.read() + grader = AutomatedGrader(assignment) + grade = grader.grade() + with open( + os.path.join(OUTPUT_FILE_PATH, f"{os.path.basename(assignment_filename)}"), "w", encoding="utf-8" + ) as f: + json.dump(grade, f, indent=4) + graded += 1 + + print(f"done! Graded {graded} assignments. Output files are in {OUTPUT_FILE_PATH}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Grade a set of homework assignments.") + parser.add_argument("filepath", type=str, help="The path to the homework files to grade.") + parser.add_argument( + "output_folder", + type=str, + nargs="?", # optional + default="out", + help="The name of the subfolder where graded assignments will be saved.", + ) + + args = parser.parse_args() + + main(args.filepath, args.output_folder) diff --git a/grader/grader.py b/grader/grader.py index 90b48df..03c1853 100644 --- a/grader/grader.py +++ b/grader/grader.py @@ -2,6 +2,7 @@ """Provide a class for grading a submission against an assignment.""" "" import json +import os from .exceptions import ( IncorrectResponseTypeError, @@ -11,7 +12,10 @@ ) +HERE = os.path.abspath(os.path.dirname(__file__)) REQUIRED_KEYS_SPEC = "required-keys.json" +REQUIRED_KEYS_PATH = os.path.join(HERE, "data", REQUIRED_KEYS_SPEC) + HUMAN_PROMPT = {"content": "a prompt from a human", "additional_kwargs": {}, "type": "human", "example": False} AI_RESPONSE = {"content": "a response from the AI", "additional_kwargs": {}, "type": "ai", "example": False} @@ -22,7 +26,7 @@ class AutomatedGrader: def __init__(self, assignment): self.assignment = assignment - with open("data/" + REQUIRED_KEYS_SPEC, "r", encoding="utf-8") as f: # pylint: disable=invalid-name + with open(REQUIRED_KEYS_PATH, "r", encoding="utf-8") as f: # pylint: disable=invalid-name self.required_keys = json.load(f) def validate_keys(self, subject, control): diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-bad-message-1.json b/grader/tests/events/bad-message-1.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-bad-message-1.json rename to grader/tests/events/bad-message-1.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-bad-message-2.json b/grader/tests/events/bad-message-2.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-bad-message-2.json rename to grader/tests/events/bad-message-2.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-bad-message-3.json b/grader/tests/events/bad-message-3.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-bad-message-3.json rename to grader/tests/events/bad-message-3.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-bad-message-4.json b/grader/tests/events/bad-message-4.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-bad-message-4.json rename to grader/tests/events/bad-message-4.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json b/grader/tests/events/correct-verbose.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json rename to grader/tests/events/correct-verbose.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-correct.json b/grader/tests/events/correct.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-correct.json rename to grader/tests/events/correct.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json b/grader/tests/events/incorrect-response-status.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json rename to grader/tests/events/incorrect-response-status.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt b/grader/tests/events/incorrect-response-type.txt similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt rename to grader/tests/events/incorrect-response-type.txt diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-type-error.json b/grader/tests/events/type-error.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-type-error.json rename to grader/tests/events/type-error.json diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json b/grader/tests/events/wrong-messages.json similarity index 100% rename from grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json rename to grader/tests/events/wrong-messages.json diff --git a/grader/tests/test_responses.py b/grader/tests/test_responses.py index 0efa366..4822fb1 100644 --- a/grader/tests/test_responses.py +++ b/grader/tests/test_responses.py @@ -16,7 +16,7 @@ class TestGrader: def test_success(self): """Test a valid successful submission.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct.json") + assignment = get_event("tests/events/correct.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -32,7 +32,7 @@ def test_success(self): def test_success_verbose(self): """Test a valid successful submission.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct-verbose.json") + assignment = get_event("tests/events/correct-verbose.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -58,7 +58,7 @@ def test_bad_data(self): def test_incorrect_response_type(self): """Test an assignment with an incorrect response type.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt") + assignment = get_event("tests/events/incorrect-response-type.txt") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() print(grade) @@ -68,7 +68,7 @@ def test_incorrect_response_type(self): def test_incorrect_response_statuscode(self): """Test an assignment with an incorrect response status code.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json") + assignment = get_event("tests/events/incorrect-response-status.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -77,7 +77,7 @@ def test_incorrect_response_statuscode(self): def test_incorrect_messages(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-wrong-messages.json") + assignment = get_event("tests/events/wrong-messages.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -86,7 +86,7 @@ def test_incorrect_messages(self): def test_incorrect_data_type(self): """Test an assignment with an incorrect data type.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-type-error.json") + assignment = get_event("tests/events/type-error.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -95,7 +95,7 @@ def test_incorrect_data_type(self): def test_bad_message_01(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-1.json") + assignment = get_event("tests/events/bad-message-1.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -104,7 +104,7 @@ def test_bad_message_01(self): def test_bad_message_02(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-2.json") + assignment = get_event("tests/events/bad-message-2.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -113,7 +113,7 @@ def test_bad_message_02(self): def test_bad_message_03(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-3.json") + assignment = get_event("tests/events/bad-message-3.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade() @@ -122,7 +122,7 @@ def test_bad_message_03(self): def test_bad_message_04(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-4.json") + assignment = get_event("tests/events/bad-message-4.json") automated_grader = AutomatedGrader(assignment=assignment) grade = automated_grader.grade()