From fdc3da80d0dafd36c78f840a393a00bdca5b04b3 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Tue, 28 Nov 2023 10:59:00 -0600 Subject: [PATCH 1/2] feat: add configurable rubric --- README.md | 11 +++++++++++ grader/batch.py | 13 ++++++++++--- grader/config.py | 22 ++++++++++++++++++++++ grader/exceptions.py | 38 ++++++++++++++++++++++++++++++-------- grader/grader.py | 29 +++++++++++++++++++++-------- 5 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 grader/config.py diff --git a/README.md b/README.md index 442c3fe..988e7f4 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,17 @@ make activate python3 -m grader.batch 'path/to/homework/json/files/' ``` +## Rubric + +Rubric values are expressed as floats between 0 and 1.00, and can be overridden with environment variables. + +```console +AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT=0.10 +AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT=0.15 +AG_RESPONSE_FAILED_PENALTY_PCT=0.20 +AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT=0.30 +``` + ### Expected output ```console diff --git a/grader/batch.py b/grader/batch.py index f47ee02..c3015e6 100644 --- a/grader/batch.py +++ b/grader/batch.py @@ -8,7 +8,7 @@ from .grader import AutomatedGrader -def main(filepath: str = None, output_folder: str = "out"): +def main(filepath: str = None, output_folder: str = "out", potential_points: int = 100): """Grade an assignment.""" graded = 0 if filepath is None: @@ -26,7 +26,7 @@ def main(filepath: str = None, output_folder: str = "out"): except json.JSONDecodeError: print(f"warning: invalid JSON in assignment_filename: {assignment_filename}") assignment = f.read() - grader = AutomatedGrader(assignment) + grader = AutomatedGrader(assignment, potential_points=potential_points) grade = grader.grade() with open( os.path.join(OUTPUT_FILE_PATH, f"{os.path.basename(assignment_filename)}"), "w", encoding="utf-8" @@ -47,7 +47,14 @@ def main(filepath: str = None, output_folder: str = "out"): default="out", help="The name of the subfolder where graded assignments will be saved.", ) + parser.add_argument( + "potential_points", + type=int, + nargs="?", # optional + default=100, + help="The aggregate point potential for the assignment.", + ) args = parser.parse_args() - main(args.filepath, args.output_folder) + main(args.filepath, args.output_folder, args.potential_points) diff --git a/grader/config.py b/grader/config.py new file mode 100644 index 0000000..fa0f569 --- /dev/null +++ b/grader/config.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +"""Setup for automated_grader package.""" + +import os + +from dotenv import find_dotenv, load_dotenv + + +# for local development and unit testing +dotenv_path = find_dotenv() +if os.path.exists(dotenv_path): + load_dotenv(dotenv_path=dotenv_path, verbose=True) + + +# pylint: disable=too-few-public-methods +class AGRubric: + """Constants for the AutomatedGrader class.""" + + INCORRECT_RESPONSE_TYPE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT", "0.10")) + INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15")) + RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20")) + INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float(os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30")) diff --git a/grader/exceptions.py b/grader/exceptions.py index f85b3c0..08d008c 100644 --- a/grader/exceptions.py +++ b/grader/exceptions.py @@ -1,34 +1,56 @@ # -*- coding: utf-8 -*- """Custom exceptions for the AutomatedGrader class.""" +from .config import AGRubric -class InvalidResponseStructureError(Exception): + +class AGException(Exception): + """A custom base exception for the AutomatedGrader class.""" + + def __init__(self, message: str, penalty_pct: float): + self.message = message + + if not isinstance(penalty_pct, float): + raise TypeError(f"penalty_pct must be a float. penalty_pct: {penalty_pct}") + if not 0 <= penalty_pct <= 1: + raise ValueError(f"penalty_pct must be between 0 and 1. penalty_pct: {penalty_pct}") + + self._penalty_pct = penalty_pct + super().__init__(self.message) + + @property + def penalty_pct(self): + """Return the penalty percentage for this error.""" + return self._penalty_pct + + +class InvalidResponseStructureError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): self.message = message - super().__init__(self.message) + super().__init__(self.message, penalty_pct=AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT) -class IncorrectResponseValueError(Exception): +class IncorrectResponseValueError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): self.message = message - super().__init__(self.message) + super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT) -class IncorrectResponseTypeError(Exception): +class IncorrectResponseTypeError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): self.message = message - super().__init__(self.message) + super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT) -class ResponseFailedError(Exception): +class ResponseFailedError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): self.message = message - super().__init__(self.message) + super().__init__(self.message, penalty_pct=AGRubric.RESPONSE_FAILED_PENALTY_PCT) diff --git a/grader/grader.py b/grader/grader.py index 03c1853..96e42ad 100644 --- a/grader/grader.py +++ b/grader/grader.py @@ -5,6 +5,7 @@ import os from .exceptions import ( + AGException, IncorrectResponseTypeError, IncorrectResponseValueError, InvalidResponseStructureError, @@ -24,11 +25,22 @@ class AutomatedGrader: """Grade a submission against an assignment.""" - def __init__(self, assignment): - self.assignment = assignment + def __init__(self, assignment, potential_points=100): + self._assignment = assignment + self._potential_points = potential_points with open(REQUIRED_KEYS_PATH, "r", encoding="utf-8") as f: # pylint: disable=invalid-name self.required_keys = json.load(f) + @property + def assignment(self): + """Return the assignment.""" + return self._assignment + + @property + def potential_points(self): + """Return the potential points for the assignment.""" + return self._potential_points + def validate_keys(self, subject, control): """Validate that the subject has all the keys in the control dict.""" assignment_keys = set(subject.keys()) @@ -154,8 +166,9 @@ def validate(self): self.validate_body() self.validate_metadata() - def grade_response(self, grade, message=None): + def grade_response(self, message: AGException = None): """Create a grade dict from the assignment.""" + grade = self.potential_points * (1 - (message.penalty_pct if message else 0)) message_type = message.__class__.__name__ if message else "Success" message = str(message) if message else "Great job!" return { @@ -169,11 +182,11 @@ def grade(self): try: self.validate() except InvalidResponseStructureError as e: - return self.grade_response(70, e) + return self.grade_response(e) except ResponseFailedError as e: - return self.grade_response(80, e) + return self.grade_response(e) except IncorrectResponseValueError as e: - return self.grade_response(85, e) + return self.grade_response(e) except IncorrectResponseTypeError as e: - return self.grade_response(90, e) - return self.grade_response(100) + return self.grade_response(e) + return self.grade_response() From 5356a3730416850fa125caab085facd3643a4bb7 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Tue, 28 Nov 2023 11:03:51 -0600 Subject: [PATCH 2/2] docs: fix up command line examples. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 988e7f4..7e25495 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ make activate ## Usage ```console +# command-line help +python3 -m grader.batch -h + +# example usage python3 -m grader.batch 'path/to/homework/json/files/' ```