feat: add configurable rubric

README.md

@@ -14,9 +14,24 @@ make activate
 ## Usage
 
 ```console
+# command-line help
+python3 -m grader.batch -h
+
+# example usage
 python3 -m grader.batch 'path/to/homework/json/files/'
 ```
 
+## Rubric
+
+Rubric values are expressed as floats between 0 and 1.00, and can be overridden with environment variables.
+
+```console
+AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT=0.10
+AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT=0.15
+AG_RESPONSE_FAILED_PENALTY_PCT=0.20
+AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT=0.30
+```
+
 ### Expected output
 
 ```console

grader/batch.py

@@ -8,7 +8,7 @@
 from .grader import AutomatedGrader
 
 
-def main(filepath: str = None, output_folder: str = "out"):
+def main(filepath: str = None, output_folder: str = "out", potential_points: int = 100):
     """Grade an assignment."""
     graded = 0
     if filepath is None:
@@ -26,7 +26,7 @@ def main(filepath: str = None, output_folder: str = "out"):
             except json.JSONDecodeError:
                 print(f"warning: invalid JSON in assignment_filename: {assignment_filename}")
                 assignment = f.read()
-        grader = AutomatedGrader(assignment)
+        grader = AutomatedGrader(assignment, potential_points=potential_points)
         grade = grader.grade()
         with open(
             os.path.join(OUTPUT_FILE_PATH, f"{os.path.basename(assignment_filename)}"), "w", encoding="utf-8"
@@ -47,7 +47,14 @@ def main(filepath: str = None, output_folder: str = "out"):
         default="out",
         help="The name of the subfolder where graded assignments will be saved.",
     )
+    parser.add_argument(
+        "potential_points",
+        type=int,
+        nargs="?",  # optional
+        default=100,
+        help="The aggregate point potential for the assignment.",
+    )
 
     args = parser.parse_args()
 
-    main(args.filepath, args.output_folder)
+    main(args.filepath, args.output_folder, args.potential_points)

grader/config.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""Setup for automated_grader package."""
+
+import os
+
+from dotenv import find_dotenv, load_dotenv
+
+
+# for local development and unit testing
+dotenv_path = find_dotenv()
+if os.path.exists(dotenv_path):
+    load_dotenv(dotenv_path=dotenv_path, verbose=True)
+
+
+# pylint: disable=too-few-public-methods
+class AGRubric:
+    """Constants for the AutomatedGrader class."""
+
+    INCORRECT_RESPONSE_TYPE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT", "0.10"))
+    INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15"))
+    RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20"))
+    INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float(os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30"))

grader/exceptions.py

@@ -1,34 +1,56 @@
 # -*- coding: utf-8 -*-
 """Custom exceptions for the AutomatedGrader class."""
 
+from .config import AGRubric
 
-class InvalidResponseStructureError(Exception):
+
+class AGException(Exception):
+    """A custom base exception for the AutomatedGrader class."""
+
+    def __init__(self, message: str, penalty_pct: float):
+        self.message = message
+
+        if not isinstance(penalty_pct, float):
+            raise TypeError(f"penalty_pct must be a float. penalty_pct: {penalty_pct}")
+        if not 0 <= penalty_pct <= 1:
+            raise ValueError(f"penalty_pct must be between 0 and 1. penalty_pct: {penalty_pct}")
+
+        self._penalty_pct = penalty_pct
+        super().__init__(self.message)
+
+    @property
+    def penalty_pct(self):
+        """Return the penalty percentage for this error."""
+        return self._penalty_pct
+
+
+class InvalidResponseStructureError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT)
 
 
-class IncorrectResponseValueError(Exception):
+class IncorrectResponseValueError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT)
 
 
-class IncorrectResponseTypeError(Exception):
+class IncorrectResponseTypeError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT)
 
 
-class ResponseFailedError(Exception):
+class ResponseFailedError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.RESPONSE_FAILED_PENALTY_PCT)

grader/grader.py

@@ -5,6 +5,7 @@
 import os
 
 from .exceptions import (
+    AGException,
     IncorrectResponseTypeError,
     IncorrectResponseValueError,
     InvalidResponseStructureError,
@@ -24,11 +25,22 @@
 class AutomatedGrader:
     """Grade a submission against an assignment."""
 
-    def __init__(self, assignment):
-        self.assignment = assignment
+    def __init__(self, assignment, potential_points=100):
+        self._assignment = assignment
+        self._potential_points = potential_points
         with open(REQUIRED_KEYS_PATH, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
             self.required_keys = json.load(f)
 
+    @property
+    def assignment(self):
+        """Return the assignment."""
+        return self._assignment
+
+    @property
+    def potential_points(self):
+        """Return the potential points for the assignment."""
+        return self._potential_points
+
     def validate_keys(self, subject, control):
         """Validate that the subject has all the keys in the control dict."""
         assignment_keys = set(subject.keys())
@@ -154,8 +166,9 @@ def validate(self):
         self.validate_body()
         self.validate_metadata()
 
-    def grade_response(self, grade, message=None):
+    def grade_response(self, message: AGException = None):
         """Create a grade dict from the assignment."""
+        grade = self.potential_points * (1 - (message.penalty_pct if message else 0))
         message_type = message.__class__.__name__ if message else "Success"
         message = str(message) if message else "Great job!"
         return {
@@ -169,11 +182,11 @@ def grade(self):
         try:
             self.validate()
         except InvalidResponseStructureError as e:
-            return self.grade_response(70, e)
+            return self.grade_response(e)
         except ResponseFailedError as e:
-            return self.grade_response(80, e)
+            return self.grade_response(e)
         except IncorrectResponseValueError as e:
-            return self.grade_response(85, e)
+            return self.grade_response(e)
         except IncorrectResponseTypeError as e:
-            return self.grade_response(90, e)
-        return self.grade_response(100)
+            return self.grade_response(e)
+        return self.grade_response()