From cbaedad67c356bfbeaae27765629ed629b48f166 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Wed, 29 Nov 2023 08:12:55 -0600 Subject: [PATCH 1/5] refactor: add def grade_structure with common structural assertions --- grader/tests/test_responses.py | 59 +++++++++++++++------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/grader/tests/test_responses.py b/grader/tests/test_responses.py index 924be8b..f7f4131 100644 --- a/grader/tests/test_responses.py +++ b/grader/tests/test_responses.py @@ -17,47 +17,47 @@ class TestGrader: """Test the OpenAI API via Langchain using the Lambda Layer, 'genai'.""" + def grade_structure(self, grade): + """Test the structure of the grade dict.""" + print(grade) + assert isinstance(grade, dict), "The grade is not a dictionary" + assert "grade" in grade, "The dictionary does not contain the key 'grade'" + assert "message" in grade, "The dictionary does not contain the key 'message'" + assert "message_type" in grade, "The dictionary does not contain the key 'message_type'" + assert isinstance(grade["grade"], float), "The grade is not an float" + assert isinstance(grade["message"], str), "The message is not a string" + assert isinstance(grade["message_type"], str), "The message_type is not a string" + assert grade["grade"] >= 0, "The grade is less than 0" + assert grade["grade"] <= POTENTIAL_PONTS, "The grade exceeds the potential points" + def test_success(self): """Test a valid successful submission.""" assignment = get_event("tests/events/correct.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) - assert isinstance(grade, dict), "The grade is not a dictionary" assert grade["message_type"] == "Success" - assert "grade" in grade, "The dictionary does not contain the key 'grade'" - assert isinstance(grade["grade"], float), "The grade is not an float" - assert grade["grade"] == 100, "The grade is not 100" - - assert "message" in grade, "The dictionary does not contain the key 'message'" - assert isinstance(grade["message"], str), "The message is not a string" assert grade["message"] == "Great job!", "The message is not 'Great job!'" + assert grade["grade"] == 100, "The grade is not 100" def test_success_verbose(self): """Test a valid successful submission.""" assignment = get_event("tests/events/correct-verbose.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) - assert isinstance(grade, dict), "The grade is not a dictionary" assert grade["message_type"] == "Success" - assert "grade" in grade, "The dictionary does not contain the key 'grade'" - assert isinstance(grade["grade"], float), "The grade is not an float" - assert grade["grade"] == 100, "The grade is not 100" - - assert "message" in grade, "The dictionary does not contain the key 'message'" - assert isinstance(grade["message"], str), "The message is not a string" assert grade["message"] == "Great job!", "The message is not 'Great job!'" + assert grade["grade"] == 100, "The grade is not 100" def test_bad_data(self): """Test an assignment with bad data.""" assignment = get_event("tests/events/bad-data.txt") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["grade"] == 70, "The grade is not 70" assert grade["message_type"] == "InvalidResponseStructureError" @@ -67,7 +67,7 @@ def test_incorrect_response_type(self): assignment = get_event("tests/events/incorrect-response-type.txt") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["grade"] == 70, "The grade is not 70" assert grade["message_type"] == "InvalidResponseStructureError" @@ -76,9 +76,8 @@ def test_incorrect_response_statuscode(self): """Test an assignment with an incorrect response status code.""" assignment = get_event("tests/events/incorrect-response-status.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "ResponseFailedError" assert grade["grade"] == 80, "The grade is not 80" @@ -87,9 +86,8 @@ def test_incorrect_messages(self): """Test an assignment with an incorrect message.""" assignment = get_event("tests/events/wrong-messages.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -98,9 +96,8 @@ def test_incorrect_data_type(self): """Test an assignment with an incorrect data type.""" assignment = get_event("tests/events/type-error.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -109,9 +106,8 @@ def test_bad_message_01(self): """Test an assignment with an incorrect message.""" assignment = get_event("tests/events/bad-message-1.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -120,9 +116,8 @@ def test_bad_message_02(self): """Test an assignment with an incorrect message.""" assignment = get_event("tests/events/bad-message-2.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -131,9 +126,8 @@ def test_bad_message_03(self): """Test an assignment with an incorrect message.""" assignment = get_event("tests/events/bad-message-3.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -142,9 +136,8 @@ def test_bad_message_04(self): """Test an assignment with an incorrect message.""" assignment = get_event("tests/events/bad-message-4.json") automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - print(grade) + self.grade_structure(grade) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" From 3825aeff25cc5580b479868414a646c86a14ec65 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Wed, 29 Nov 2023 08:23:16 -0600 Subject: [PATCH 2/5] refactor: add InvalidJSONResponseError --- grader/config.py | 1 + grader/exceptions.py | 9 +++++++++ grader/grader.py | 5 +++-- grader/tests/test_responses.py | 17 +++++++++++++---- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/grader/config.py b/grader/config.py index fa0f569..3d8d5f2 100644 --- a/grader/config.py +++ b/grader/config.py @@ -20,3 +20,4 @@ class AGRubric: INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15")) RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20")) INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float(os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30")) + AG_INVALID_JSON_RESPONSE_PENALTY_PCT = float(os.getenv("AG_INVALID_JSON_RESPONSE_PENALTY_PCT", "0.50")) diff --git a/grader/exceptions.py b/grader/exceptions.py index ae44eb9..2dfc751 100644 --- a/grader/exceptions.py +++ b/grader/exceptions.py @@ -24,6 +24,14 @@ def penalty_pct(self): return self._penalty_pct +class InvalidJSONResponseError(AGException): + """A custom exception for the AutomatedGrader class.""" + + def __init__(self, message): + self.message = message + super().__init__(self.message, penalty_pct=AGRubric.AG_INVALID_JSON_RESPONSE_PENALTY_PCT) + + class InvalidResponseStructureError(AGException): """A custom exception for the AutomatedGrader class.""" @@ -58,6 +66,7 @@ def __init__(self, message): VALID_MESSAGE_TYPES = [ "Success", + InvalidJSONResponseError.__name__, IncorrectResponseTypeError.__name__, IncorrectResponseValueError.__name__, InvalidResponseStructureError.__name__, diff --git a/grader/grader.py b/grader/grader.py index 88965ca..287ae50 100644 --- a/grader/grader.py +++ b/grader/grader.py @@ -10,6 +10,7 @@ AGException, IncorrectResponseTypeError, IncorrectResponseValueError, + InvalidJSONResponseError, InvalidResponseStructureError, ResponseFailedError, ) @@ -72,8 +73,8 @@ def grade(self): assignment_json = json.loads(self.assignment) except json.JSONDecodeError as e: try: - raise InvalidResponseStructureError("The assignment is not valid JSON") from e - except InvalidResponseStructureError as reraised_e: + raise InvalidJSONResponseError("The assignment is not valid JSON") from e + except InvalidJSONResponseError as reraised_e: return self.grade_response(reraised_e) # 2.) attempt to validate the assignment using Pydantic diff --git a/grader/tests/test_responses.py b/grader/tests/test_responses.py index f7f4131..3189ba0 100644 --- a/grader/tests/test_responses.py +++ b/grader/tests/test_responses.py @@ -59,8 +59,9 @@ def test_bad_data(self): grade = automated_grader.grade() self.grade_structure(grade) - assert grade["grade"] == 70, "The grade is not 70" - assert grade["message_type"] == "InvalidResponseStructureError" + assert grade["grade"] == 50, "The grade is not 50" + assert grade["message_type"] == "InvalidJSONResponseError" + assert grade["message"] == "The assignment is not valid JSON" def test_incorrect_response_type(self): """Test an assignment with an incorrect response type.""" @@ -69,8 +70,9 @@ def test_incorrect_response_type(self): grade = automated_grader.grade() self.grade_structure(grade) - assert grade["grade"] == 70, "The grade is not 70" - assert grade["message_type"] == "InvalidResponseStructureError" + assert grade["grade"] == 50, "The grade is not 50" + assert grade["message_type"] == "InvalidJSONResponseError" + assert grade["message"] == "The assignment is not valid JSON" def test_incorrect_response_statuscode(self): """Test an assignment with an incorrect response status code.""" @@ -81,6 +83,7 @@ def test_incorrect_response_statuscode(self): assert grade["message_type"] == "ResponseFailedError" assert grade["grade"] == 80, "The grade is not 80" + assert grade["message"] == "status_code must be 200. received: 403" def test_incorrect_messages(self): """Test an assignment with an incorrect message.""" @@ -91,6 +94,7 @@ def test_incorrect_messages(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "The assignment failed pydantic validation." def test_incorrect_data_type(self): """Test an assignment with an incorrect data type.""" @@ -101,6 +105,7 @@ def test_incorrect_data_type(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "The assignment failed pydantic validation." def test_bad_message_01(self): """Test an assignment with an incorrect message.""" @@ -111,6 +116,7 @@ def test_bad_message_01(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "The assignment failed pydantic validation." def test_bad_message_02(self): """Test an assignment with an incorrect message.""" @@ -121,6 +127,7 @@ def test_bad_message_02(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "messages must contain at least 2 objects" def test_bad_message_03(self): """Test an assignment with an incorrect message.""" @@ -131,6 +138,7 @@ def test_bad_message_03(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "The assignment failed pydantic validation." def test_bad_message_04(self): """Test an assignment with an incorrect message.""" @@ -141,3 +149,4 @@ def test_bad_message_04(self): assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" + assert grade["message"] == "The assignment failed pydantic validation." From 1eaa31be04876409b219fbc10839a82af3fc74e9 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Wed, 29 Nov 2023 09:32:40 -0600 Subject: [PATCH 3/5] chore: add config and exception unit tests --- grader/config.py | 36 ++++++++++-- grader/exceptions.py | 2 +- grader/tests/test_config.py | 60 ++++++++++++++++++++ grader/tests/test_exceptions.py | 54 ++++++++++++++++++ grader/tests/test_responses.py | 98 ++++++++++++++++++--------------- 5 files changed, 198 insertions(+), 52 deletions(-) create mode 100644 grader/tests/test_config.py create mode 100644 grader/tests/test_exceptions.py diff --git a/grader/config.py b/grader/config.py index 3d8d5f2..9bcbe04 100644 --- a/grader/config.py +++ b/grader/config.py @@ -4,6 +4,7 @@ import os from dotenv import find_dotenv, load_dotenv +from pydantic import BaseModel, Field # for local development and unit testing @@ -13,11 +14,34 @@ # pylint: disable=too-few-public-methods +class _AGRubric(BaseModel): + """Private class to strongly type and bound package configuration params.""" + + INCORRECT_RESPONSE_TYPE_PENALTY_PCT: float = Field(0.10, gt=0.00, le=1.00) + INCORRECT_RESPONSE_VALUE_PENALTY_PCT: float = Field(0.15, gt=0.00, le=1.00) + RESPONSE_FAILED_PENALTY_PCT: float = Field(0.20, gt=0.00, le=1.00) + INVALID_RESPONSE_STRUCTURE_PENALTY_PCT: float = Field(0.30, gt=0.00, le=1.00) + INVALID_JSON_RESPONSE_PENALTY_PCT: float = Field(0.50, gt=0.00, le=1.00) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.INCORRECT_RESPONSE_TYPE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT", "0.10")) + self.INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15")) + self.RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20")) + self.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float( + os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30") + ) + self.INVALID_JSON_RESPONSE_PENALTY_PCT = float(os.getenv("AG_INVALID_JSON_RESPONSE_PENALTY_PCT", "0.50")) + + +ag_rubric = _AGRubric() + + class AGRubric: - """Constants for the AutomatedGrader class.""" + """Public class to access package configuration params.""" - INCORRECT_RESPONSE_TYPE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT", "0.10")) - INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15")) - RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20")) - INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float(os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30")) - AG_INVALID_JSON_RESPONSE_PENALTY_PCT = float(os.getenv("AG_INVALID_JSON_RESPONSE_PENALTY_PCT", "0.50")) + INCORRECT_RESPONSE_TYPE_PENALTY_PCT = ag_rubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT + INCORRECT_RESPONSE_VALUE_PENALTY_PCT = ag_rubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT + RESPONSE_FAILED_PENALTY_PCT = ag_rubric.RESPONSE_FAILED_PENALTY_PCT + INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = ag_rubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT + INVALID_JSON_RESPONSE_PENALTY_PCT = ag_rubric.INVALID_JSON_RESPONSE_PENALTY_PCT diff --git a/grader/exceptions.py b/grader/exceptions.py index 2dfc751..92c0dd1 100644 --- a/grader/exceptions.py +++ b/grader/exceptions.py @@ -29,7 +29,7 @@ class InvalidJSONResponseError(AGException): def __init__(self, message): self.message = message - super().__init__(self.message, penalty_pct=AGRubric.AG_INVALID_JSON_RESPONSE_PENALTY_PCT) + super().__init__(self.message, penalty_pct=AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT) class InvalidResponseStructureError(AGException): diff --git a/grader/tests/test_config.py b/grader/tests/test_config.py new file mode 100644 index 0000000..86e7e9a --- /dev/null +++ b/grader/tests/test_config.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# flake8: noqa: F401 +# pylint: disable=duplicate-code +""" +Test integrity of configuration variables. +""" +import pytest # pylint: disable=unused-import + +from ..config import AGRubric + + +class TestConfig: + """Test integrity of configuration variables.""" + + def test_configuration_data_types(self): + """Test the data types of the configuration variables.""" + assert isinstance( + AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT, float + ), "INCORRECT_RESPONSE_TYPE_PENALTY_PCT is not a float" + assert isinstance( + AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT, float + ), "INCORRECT_RESPONSE_VALUE_PENALTY_PCT is not a float" + assert isinstance(AGRubric.RESPONSE_FAILED_PENALTY_PCT, float), "RESPONSE_FAILED_PENALTY_PCT is not a float" + assert isinstance( + AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT, float + ), "INVALID_RESPONSE_STRUCTURE_PENALTY_PCT is not a float" + assert isinstance( + AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT, float + ), "INVALID_JSON_RESPONSE_PENALTY_PCT is not a float" + + def test_configuration_values(self): + """Test the values of the configuration variables.""" + assert ( + AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT <= 1.00 + ), "INCORRECT_RESPONSE_TYPE_PENALTY_PCT is greater than 1.00" + assert ( + AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT >= 0.00 + ), "INCORRECT_RESPONSE_TYPE_PENALTY_PCT is less than 0.00" + + assert ( + AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT <= 1.00 + ), "INCORRECT_RESPONSE_VALUE_PENALTY_PCT is greater than 1.00" + assert ( + AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT >= 0.00 + ), "INCORRECT_RESPONSE_VALUE_PENALTY_PCT is less than 0.00" + + assert AGRubric.RESPONSE_FAILED_PENALTY_PCT <= 1.00, "RESPONSE_FAILED_PENALTY_PCT is greater than 1.00" + assert AGRubric.RESPONSE_FAILED_PENALTY_PCT >= 0.00, "RESPONSE_FAILED_PENALTY_PCT is less than 0.00" + + assert ( + AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT <= 1.00 + ), "INVALID_RESPONSE_STRUCTURE_PENALTY_PCT is greater than 1.00" + assert ( + AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT >= 0.00 + ), "INVALID_RESPONSE_STRUCTURE_PENALTY_PCT is less than 0.00" + + assert ( + AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT <= 1.00 + ), "INVALID_JSON_RESPONSE_PENALTY_PCT is greater than 1.00" + assert AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT >= 0.00, "INVALID_JSON_RESPONSE_PENALTY_PCT is less than 0.00" diff --git a/grader/tests/test_exceptions.py b/grader/tests/test_exceptions.py new file mode 100644 index 0000000..8cd3543 --- /dev/null +++ b/grader/tests/test_exceptions.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# flake8: noqa: F401 +# pylint: disable=duplicate-code +""" +Test integrity of exception classes. +""" +import pytest # pylint: disable=unused-import + +from ..exceptions import ( + VALID_MESSAGE_TYPES, + AGException, + IncorrectResponseTypeError, + IncorrectResponseValueError, + InvalidJSONResponseError, + InvalidResponseStructureError, + ResponseFailedError, +) + + +class TestExceptions: + """Test integrity of exception classes.""" + + def eval_exception_class(self, exception_class): + """Evaluate an exception class.""" + try: + raise exception_class("This is an exception") + except exception_class as err: + assert issubclass( + exception_class, AGException + ), "IncorrectResponseTypeError is not a subclass of AGException" + assert err.penalty_pct <= 1.00, "Penalty percentage is greater than 1.00" + assert err.penalty_pct >= 0.00, "Penalty percentage is less than 0.00" + assert err.message == "This is an exception", "Incorrect message" + assert exception_class.__name__ in VALID_MESSAGE_TYPES, "Invalid message type" + + def test_incorrect_response_type(self): + """Test an incorrect response type.""" + self.eval_exception_class(IncorrectResponseTypeError) + + def test_incorrect_response_value(self): + """Test an incorrect response value.""" + self.eval_exception_class(IncorrectResponseValueError) + + def test_invalid_json_response(self): + """Test an invalid JSON response.""" + self.eval_exception_class(InvalidJSONResponseError) + + def test_invalid_response_structure(self): + """Test an invalid response structure.""" + self.eval_exception_class(InvalidResponseStructureError) + + def test_response_failed(self): + """Test a failed response.""" + self.eval_exception_class(ResponseFailedError) diff --git a/grader/tests/test_responses.py b/grader/tests/test_responses.py index 3189ba0..e829b7c 100644 --- a/grader/tests/test_responses.py +++ b/grader/tests/test_responses.py @@ -5,8 +5,9 @@ Test integrity of automated grader class methods. """ import pytest # pylint: disable=unused-import +from pydantic import ValidationError -from ..grader import AutomatedGrader +from ..grader import AutomatedGrader, Grade from .init import get_event @@ -17,6 +18,14 @@ class TestGrader: """Test the OpenAI API via Langchain using the Lambda Layer, 'genai'.""" + def grade(self, assignment_filespec: str, potential_points: int = POTENTIAL_PONTS): + """Grade an assignment, test its structure and return.""" + assignment = get_event(assignment_filespec) + automated_grader = AutomatedGrader(assignment=assignment, potential_points=potential_points) + grade = automated_grader.grade() + self.grade_structure(grade) + return grade + def grade_structure(self, grade): """Test the structure of the grade dict.""" print(grade) @@ -30,23 +39,49 @@ def grade_structure(self, grade): assert grade["grade"] >= 0, "The grade is less than 0" assert grade["grade"] <= POTENTIAL_PONTS, "The grade exceeds the potential points" + def test_grade_structure(self): + """Test the structure of the grade dict.""" + g = Grade(grade=100, message="Great job!", message_type="Success") + self.grade_structure(g.model_dump()) + + def test_low_grade(self): + """Test a low grade.""" + try: + Grade(grade=-1, message="Great job!", message_type="Success") + except ValidationError: + pass + else: + raise AssertionError("Grade must be greater than 0") + + def test_grade_invalid_message_type(self): + """Test an invalid message type.""" + try: + Grade(grade=-1, message="Great job!", message_type="Not a valid message type") + except ValidationError: + pass + else: + raise AssertionError("message_type out of range") + def test_success(self): """Test a valid successful submission.""" - assignment = get_event("tests/events/correct.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/correct.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "Success" assert grade["message"] == "Great job!", "The message is not 'Great job!'" assert grade["grade"] == 100, "The grade is not 100" + def test_low_potential_points(self): + """Test a valid successful submission.""" + try: + self.grade("tests/events/correct.json", potential_points=-1) + except ValidationError: + pass + else: + raise AssertionError("Potential points must be greater than 0") + def test_success_verbose(self): """Test a valid successful submission.""" - assignment = get_event("tests/events/correct-verbose.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/correct-verbose.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "Success" assert grade["message"] == "Great job!", "The message is not 'Great job!'" @@ -54,10 +89,7 @@ def test_success_verbose(self): def test_bad_data(self): """Test an assignment with bad data.""" - assignment = get_event("tests/events/bad-data.txt") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/bad-data.txt", potential_points=POTENTIAL_PONTS) assert grade["grade"] == 50, "The grade is not 50" assert grade["message_type"] == "InvalidJSONResponseError" @@ -65,10 +97,7 @@ def test_bad_data(self): def test_incorrect_response_type(self): """Test an assignment with an incorrect response type.""" - assignment = get_event("tests/events/incorrect-response-type.txt") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/incorrect-response-type.txt", potential_points=POTENTIAL_PONTS) assert grade["grade"] == 50, "The grade is not 50" assert grade["message_type"] == "InvalidJSONResponseError" @@ -76,10 +105,7 @@ def test_incorrect_response_type(self): def test_incorrect_response_statuscode(self): """Test an assignment with an incorrect response status code.""" - assignment = get_event("tests/events/incorrect-response-status.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/incorrect-response-status.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "ResponseFailedError" assert grade["grade"] == 80, "The grade is not 80" @@ -87,10 +113,7 @@ def test_incorrect_response_statuscode(self): def test_incorrect_messages(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/wrong-messages.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/wrong-messages.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -98,10 +121,7 @@ def test_incorrect_messages(self): def test_incorrect_data_type(self): """Test an assignment with an incorrect data type.""" - assignment = get_event("tests/events/type-error.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/type-error.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -109,10 +129,7 @@ def test_incorrect_data_type(self): def test_bad_message_01(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/bad-message-1.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/bad-message-1.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -120,10 +137,7 @@ def test_bad_message_01(self): def test_bad_message_02(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/bad-message-2.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/bad-message-2.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -131,10 +145,7 @@ def test_bad_message_02(self): def test_bad_message_03(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/bad-message-3.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/bad-message-3.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" @@ -142,10 +153,7 @@ def test_bad_message_03(self): def test_bad_message_04(self): """Test an assignment with an incorrect message.""" - assignment = get_event("tests/events/bad-message-4.json") - automated_grader = AutomatedGrader(assignment=assignment, potential_points=POTENTIAL_PONTS) - grade = automated_grader.grade() - self.grade_structure(grade) + grade = self.grade("tests/events/bad-message-4.json", potential_points=POTENTIAL_PONTS) assert grade["message_type"] == "InvalidResponseStructureError" assert grade["grade"] == 70, "The grade is not 70" From f354a8565a5b0e7ad98ac2b925c8b0163063e774 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Wed, 29 Nov 2023 09:37:34 -0600 Subject: [PATCH 4/5] chore: add data type and bound tests --- grader/tests/test_config.py | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/grader/tests/test_config.py b/grader/tests/test_config.py index 86e7e9a..a8ed7be 100644 --- a/grader/tests/test_config.py +++ b/grader/tests/test_config.py @@ -6,7 +6,7 @@ """ import pytest # pylint: disable=unused-import -from ..config import AGRubric +from ..config import AGRubric, _AGRubric class TestConfig: @@ -58,3 +58,42 @@ def test_configuration_values(self): AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT <= 1.00 ), "INVALID_JSON_RESPONSE_PENALTY_PCT is greater than 1.00" assert AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT >= 0.00, "INVALID_JSON_RESPONSE_PENALTY_PCT is less than 0.00" + + def test_illegal_config_values_low(self): + """Test illegal configuration values (low).""" + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_TYPE_PENALTY_PCT=-0.01) + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_VALUE_PENALTY_PCT=-0.01) + with pytest.raises(ValueError): + _AGRubric(RESPONSE_FAILED_PENALTY_PCT=-0.01) + with pytest.raises(ValueError): + _AGRubric(INVALID_RESPONSE_STRUCTURE_PENALTY_PCT=-0.01) + with pytest.raises(ValueError): + _AGRubric(INVALID_JSON_RESPONSE_PENALTY_PCT=-0.01) + + def test_illegal_config_values_high(self): + """Test illegal configuration values (high).""" + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_TYPE_PENALTY_PCT=1.01) + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_VALUE_PENALTY_PCT=1.01) + with pytest.raises(ValueError): + _AGRubric(RESPONSE_FAILED_PENALTY_PCT=1.01) + with pytest.raises(ValueError): + _AGRubric(INVALID_RESPONSE_STRUCTURE_PENALTY_PCT=1.01) + with pytest.raises(ValueError): + _AGRubric(INVALID_JSON_RESPONSE_PENALTY_PCT=1.01) + + def test_illegal_config_values_bad_data_type(self): + """Test illegal configuration values (bad data type).""" + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_TYPE_PENALTY_PCT="X0.01") + with pytest.raises(ValueError): + _AGRubric(INCORRECT_RESPONSE_VALUE_PENALTY_PCT="BAD DATA") + with pytest.raises(ValueError): + _AGRubric(RESPONSE_FAILED_PENALTY_PCT="TRUE") + with pytest.raises(ValueError): + _AGRubric(INVALID_RESPONSE_STRUCTURE_PENALTY_PCT="_0.01_") + with pytest.raises(ValueError): + _AGRubric(INVALID_JSON_RESPONSE_PENALTY_PCT="#0.01") From f9de1938eb203c72cb69ebcb98a52729bc2bffb6 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Wed, 29 Nov 2023 10:02:26 -0600 Subject: [PATCH 5/5] refactor: create AGExceptionModel with bounds and type checking for penalty_pct --- grader/exceptions.py | 55 +++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/grader/exceptions.py b/grader/exceptions.py index 92c0dd1..99be52e 100644 --- a/grader/exceptions.py +++ b/grader/exceptions.py @@ -1,67 +1,74 @@ # -*- coding: utf-8 -*- """Custom exceptions for the AutomatedGrader class.""" +from pydantic import BaseModel, Field + from .config import AGRubric -class AGException(Exception): - """A custom base exception for the AutomatedGrader class.""" +class AGExceptionModel(BaseModel): + """A Pydantic model for the custom exceptions for the AutomatedGrader class.""" - def __init__(self, message: str, penalty_pct: float): - self.message = message + message: str = Field(..., description="The message to display to the user.") + penalty_pct: float = Field( + ..., description="The percentage of the total grade to deduct for this error.", ge=0, le=1 + ) - if not isinstance(penalty_pct, float): - raise TypeError(f"penalty_pct must be a float. penalty_pct: {penalty_pct}") - if not 0 <= penalty_pct <= 1: - raise ValueError(f"penalty_pct must be between 0 and 1. penalty_pct: {penalty_pct}") - self._penalty_pct = penalty_pct - super().__init__(self.message) +class AGException(Exception): + """A custom base exception for the AutomatedGrader class.""" + + message: str + penalty_pct: float - @property - def penalty_pct(self): - """Return the penalty percentage for this error.""" - return self._penalty_pct + def __init__(self, ag_exception_model: AGExceptionModel): + super().__init__(ag_exception_model.message) + self.message = ag_exception_model.message + self.penalty_pct = ag_exception_model.penalty_pct class InvalidJSONResponseError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): - self.message = message - super().__init__(self.message, penalty_pct=AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT) + ag_exception_model = AGExceptionModel(message=message, penalty_pct=AGRubric.INVALID_JSON_RESPONSE_PENALTY_PCT) + super().__init__(ag_exception_model) class InvalidResponseStructureError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): - self.message = message - super().__init__(self.message, penalty_pct=AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT) + ag_exception_model = AGExceptionModel( + message=message, penalty_pct=AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT + ) + super().__init__(ag_exception_model) class IncorrectResponseValueError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): - self.message = message - super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT) + ag_exception_model = AGExceptionModel( + message=message, penalty_pct=AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT + ) + super().__init__(ag_exception_model) class IncorrectResponseTypeError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): - self.message = message - super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT) + ag_exception_model = AGExceptionModel(message=message, penalty_pct=AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT) + super().__init__(ag_exception_model) class ResponseFailedError(AGException): """A custom exception for the AutomatedGrader class.""" def __init__(self, message): - self.message = message - super().__init__(self.message, penalty_pct=AGRubric.RESPONSE_FAILED_PENALTY_PCT) + ag_exception_model = AGExceptionModel(message=message, penalty_pct=AGRubric.RESPONSE_FAILED_PENALTY_PCT) + super().__init__(ag_exception_model) VALID_MESSAGE_TYPES = [