From 3e38a5a4f6cd3bc686be5e234d5f77c3325aa062 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Tue, 14 May 2024 09:35:16 -0400 Subject: [PATCH 1/5] First version of the submit task code --- moocfi_cses.py | 80 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/moocfi_cses.py b/moocfi_cses.py index 2316afb..b13458f 100644 --- a/moocfi_cses.py +++ b/moocfi_cses.py @@ -6,6 +6,7 @@ import json from urllib.parse import urljoin from pathlib import Path +from time import sleep from typing import AnyStr, Optional from xml.etree.ElementTree import Element, tostring @@ -145,6 +146,12 @@ def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: parser_show.set_defaults(cmd="show") parser_show.add_argument("task_id", help="Numerical task identifier") + # submit exercise solution subparser + parser_submit = subparsers.add_parser("submit", help="Submit an exercise solution") + parser_submit.set_defaults(cmd="submit") + parser_submit.add_argument("--filename", help="Filename of the solution to submit") + parser_submit.add_argument("task_id", help="Numerical task identifier") + return parser.parse_args(args) @@ -221,7 +228,7 @@ def find_link(html: AnyStr, xpath: str) -> dict[str, str | None]: return link_data -def parse_form(html: AnyStr, xpath: str = ".//form") -> dict[str, str | None]: +def parse_form(html: AnyStr, xpath: str = ".//form") -> dict: """Search for the first form in html and return dict with action and all other found inputs""" form_element = htmlement.fromstring(html).find(xpath) form_data = dict() @@ -251,9 +258,10 @@ class Task: id: str name: str state: TaskState - description: str = "N/A" - code: str = "N/A" - submit_file: str = "N/A" + description: Optional[str] = None + code: Optional[str] = None + submit_file: Optional[str] = None + submit_link: Optional[str] = None def parse_task_list(html: AnyStr) -> list[Task]: @@ -313,6 +321,13 @@ def parse_task(html: AnyStr) -> Task: desc_div = desc_div_element if desc_div_element is not None else Element("div") description = html2text(tostring(desc_div).decode("utf8")) code = root.findtext(".//pre", "N/A") + submit_link_element = root.find('.//a[.="Submit"]') + submit_link = ( + submit_link_element.get("href", "N/A") + if submit_link_element is not None + else "N/A" + ) + submit_file = next( iter( [ @@ -330,6 +345,7 @@ def parse_task(html: AnyStr) -> Task: description=description.strip(), code=code, submit_file=submit_file, + submit_link=submit_link, ) return task @@ -341,10 +357,13 @@ def print_task(task: Task) -> None: print(f"\nSubmission file name: {task.submit_file}") -def submit_task(task_id: str, filename: str) -> None: - """submit file to the submit form or task_id""" - # NOTE: use parse_form - ... +# def submit_task(task_id: str, filename: str) -> None: +# """submit file to the submit form or task_id""" +# html = session.http_request(urljoin(base_url, f"task/{task_id}")) +# task = parse_task(html) +# answer = input("Do you want to submit this task? (y/n): ") +# if answer in ('y', 'Y'): +# with open(filename, 'r') as f: def main() -> None: @@ -399,6 +418,51 @@ def main() -> None: task = parse_task(html) print_task(task) + if args.cmd == "submit": + html = session.http_request(urljoin(base_url, f"task/{args.task_id}")) + task = parse_task(html) + if not task.submit_file or task.submit_file == "N/A": + raise ValueError("No submission filename found") + if not task.submit_link or task.submit_link == "N/A": + raise ValueError("No submission link found") + + submit_form_html = session.http_request(urljoin(base_url, task.submit_link)) + submit_form_data = parse_form(submit_form_html) + action = submit_form_data.pop("_action") + + for key, value in submit_form_data.items(): + submit_form_data[key] = (None, value) + submit_form_data["file"] = (task.submit_file, open(task.submit_file, "rb")) + submit_form_data["lang"] = (None, "Python3") + submit_form_data["option"] = (None, "CPython3") + + res = session.http_session.post( + urljoin(base_url, action), files=submit_form_data + ) + html = res.text + result_url = res.url + print("Waiting for test results.", end="") + while "Test report" not in html: + print(".", end="") + sleep(1) + html = session.http_request(result_url) + + print() + root = htmlement.fromstring(html) + submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") + submit_status_span_element = submit_status_element.find("td/span") or Element( + "span" + ) + submit_status = submit_status_span_element.text or "" + submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") + submit_result_span_element = submit_result_element.find("td/span") or Element( + "span" + ) + submit_result = submit_result_span_element.text or "" + + print(f"Submission status: {submit_status.lower()}") + print(f"Submission result: {submit_result.lower()}") + if __name__ == "__main__": main() From a595f44a0316fe1582201095e7e228dbd703b621 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Tue, 14 May 2024 09:35:16 -0400 Subject: [PATCH 2/5] First version of the submit task code --- tyora.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/tyora.py b/tyora.py index 4f0aa74..451be63 100644 --- a/tyora.py +++ b/tyora.py @@ -6,6 +6,7 @@ import json from urllib.parse import urljoin from pathlib import Path +from time import sleep from typing import AnyStr, Optional from xml.etree.ElementTree import Element, tostring @@ -145,6 +146,12 @@ def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: parser_show.set_defaults(cmd="show") parser_show.add_argument("task_id", help="Numerical task identifier") + # submit exercise solution subparser + parser_submit = subparsers.add_parser("submit", help="Submit an exercise solution") + parser_submit.set_defaults(cmd="submit") + parser_submit.add_argument("--filename", help="Filename of the solution to submit") + parser_submit.add_argument("task_id", help="Numerical task identifier") + return parser.parse_args(args) @@ -221,7 +228,7 @@ def find_link(html: AnyStr, xpath: str) -> dict[str, str | None]: return link_data -def parse_form(html: AnyStr, xpath: str = ".//form") -> dict[str, str | None]: +def parse_form(html: AnyStr, xpath: str = ".//form") -> dict: """Search for the first form in html and return dict with action and all other found inputs""" form_element = htmlement.fromstring(html).find(xpath) form_data = dict() @@ -251,9 +258,10 @@ class Task: id: str name: str state: TaskState - description: str = "N/A" - code: str = "N/A" - submit_file: str = "N/A" + description: Optional[str] = None + code: Optional[str] = None + submit_file: Optional[str] = None + submit_link: Optional[str] = None def parse_task_list(html: AnyStr) -> list[Task]: @@ -313,6 +321,13 @@ def parse_task(html: AnyStr) -> Task: desc_div = desc_div_element if desc_div_element is not None else Element("div") description = html2text(tostring(desc_div).decode("utf8")) code = root.findtext(".//pre", "N/A") + submit_link_element = root.find('.//a[.="Submit"]') + submit_link = ( + submit_link_element.get("href", "N/A") + if submit_link_element is not None + else "N/A" + ) + submit_file = next( iter( [ @@ -330,6 +345,7 @@ def parse_task(html: AnyStr) -> Task: description=description.strip(), code=code, submit_file=submit_file, + submit_link=submit_link, ) return task @@ -341,10 +357,13 @@ def print_task(task: Task) -> None: print(f"\nSubmission file name: {task.submit_file}") -def submit_task(task_id: str, filename: str) -> None: - """submit file to the submit form or task_id""" - # NOTE: use parse_form - ... +# def submit_task(task_id: str, filename: str) -> None: +# """submit file to the submit form or task_id""" +# html = session.http_request(urljoin(base_url, f"task/{task_id}")) +# task = parse_task(html) +# answer = input("Do you want to submit this task? (y/n): ") +# if answer in ('y', 'Y'): +# with open(filename, 'r') as f: def main() -> None: @@ -399,6 +418,51 @@ def main() -> None: task = parse_task(html) print_task(task) + if args.cmd == "submit": + html = session.http_request(urljoin(base_url, f"task/{args.task_id}")) + task = parse_task(html) + if not task.submit_file or task.submit_file == "N/A": + raise ValueError("No submission filename found") + if not task.submit_link or task.submit_link == "N/A": + raise ValueError("No submission link found") + + submit_form_html = session.http_request(urljoin(base_url, task.submit_link)) + submit_form_data = parse_form(submit_form_html) + action = submit_form_data.pop("_action") + + for key, value in submit_form_data.items(): + submit_form_data[key] = (None, value) + submit_form_data["file"] = (task.submit_file, open(task.submit_file, "rb")) + submit_form_data["lang"] = (None, "Python3") + submit_form_data["option"] = (None, "CPython3") + + res = session.http_session.post( + urljoin(base_url, action), files=submit_form_data + ) + html = res.text + result_url = res.url + print("Waiting for test results.", end="") + while "Test report" not in html: + print(".", end="") + sleep(1) + html = session.http_request(result_url) + + print() + root = htmlement.fromstring(html) + submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") + submit_status_span_element = submit_status_element.find("td/span") or Element( + "span" + ) + submit_status = submit_status_span_element.text or "" + submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") + submit_result_span_element = submit_result_element.find("td/span") or Element( + "span" + ) + submit_result = submit_result_span_element.text or "" + + print(f"Submission status: {submit_status.lower()}") + print(f"Submission result: {submit_result.lower()}") + if __name__ == "__main__": main() From e555cc0bebfccf2420c361d61a39503a694ec79f Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Thu, 16 May 2024 12:13:14 -0400 Subject: [PATCH 3/5] Run ruff and black on code --- tests/test_tyora.py | 17 +++++------------ tyora.py | 8 +++++--- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/test_tyora.py b/tests/test_tyora.py index cd5f527..8bf2982 100644 --- a/tests/test_tyora.py +++ b/tests/test_tyora.py @@ -25,10 +25,7 @@ class TestFindLink: valid_return = {"href": "somelink", "text": "sometext"} def test_find_link_success(self) -> None: - assert ( - tyora.find_link(self.valid_html, self.valid_xpath) - == self.valid_return - ) + assert tyora.find_link(self.valid_html, self.valid_xpath) == self.valid_return def test_find_link_bad_xpath(self) -> None: assert tyora.find_link(self.valid_html, self.invalid_xpath) == {} @@ -84,17 +81,13 @@ def test_login_failed(mock_session: tyora.Session) -> None: # TODO: functions that use user input or read or write files -def test_create_config() -> None: - ... +def test_create_config() -> None: ... -def test_write_config() -> None: - ... +def test_write_config() -> None: ... -def test_read_config() -> None: - ... +def test_read_config() -> None: ... -def test_get_cookiejar() -> None: - ... +def test_get_cookiejar() -> None: ... diff --git a/tyora.py b/tyora.py index 451be63..c82ab2d 100644 --- a/tyora.py +++ b/tyora.py @@ -287,9 +287,11 @@ def parse_task_list(html: AnyStr) -> list[Task]: task = Task( id=item_id, name=item_name, - state=TaskState.COMPLETE - if "full" in item_class - else TaskState.INCOMPLETE, + state=( + TaskState.COMPLETE + if "full" in item_class + else TaskState.INCOMPLETE + ), ) task_list.append(task) From b2af18f97a8291b86c56442ef23b3be9165b115d Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 19 May 2024 20:54:13 -0400 Subject: [PATCH 4/5] Fix code and clean up --- tyora.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tyora.py b/tyora.py index c82ab2d..afd00f3 100644 --- a/tyora.py +++ b/tyora.py @@ -149,7 +149,10 @@ def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: # submit exercise solution subparser parser_submit = subparsers.add_parser("submit", help="Submit an exercise solution") parser_submit.set_defaults(cmd="submit") - parser_submit.add_argument("--filename", help="Filename of the solution to submit") + parser_submit.add_argument( + "--filename", + help="Filename of the solution to submit (if not given will be guessed from task description)", + ) parser_submit.add_argument("task_id", help="Numerical task identifier") return parser.parse_args(args) @@ -315,19 +318,23 @@ def parse_task(html: AnyStr) -> Task: task_link_element = root.find('.//div[@class="nav sidebar"]/a') task_link = task_link_element if task_link_element is not None else Element("a") task_id = task_link.get("href", "").split("/")[-1] - task_name = task_link.text or "N/A" + if not task_id: + raise ValueError("Failed to find task id") + task_name = task_link.text or None + if not task_name: + raise ValueError("Failed to find task name") task_span_element = task_link.find("span") task_span = task_span_element if task_span_element is not None else Element("span") task_span_class = task_span.get("class", "") desc_div_element = root.find('.//div[@class="md"]') desc_div = desc_div_element if desc_div_element is not None else Element("div") description = html2text(tostring(desc_div).decode("utf8")) - code = root.findtext(".//pre", "N/A") + code = root.findtext(".//pre", None) submit_link_element = root.find('.//a[.="Submit"]') submit_link = ( - submit_link_element.get("href", "N/A") + submit_link_element.get("href", None) if submit_link_element is not None - else "N/A" + else None ) submit_file = next( @@ -338,7 +345,7 @@ def parse_task(html: AnyStr) -> Task: if code_element.text is not None and ".py" in code_element.text ] ), - "N/A", + None, ) task = Task( id=task_id, @@ -417,16 +424,21 @@ def main() -> None: if args.cmd == "show": html = session.http_request(urljoin(base_url, f"task/{args.task_id}")) - task = parse_task(html) + try: + task = parse_task(html) + except ValueError as e: + logger.debug(f"Error parsing task: {e}") + raise print_task(task) if args.cmd == "submit": html = session.http_request(urljoin(base_url, f"task/{args.task_id}")) task = parse_task(html) - if not task.submit_file or task.submit_file == "N/A": + if not task.submit_file and not args.filename: raise ValueError("No submission filename found") - if not task.submit_link or task.submit_link == "N/A": + if not task.submit_link: raise ValueError("No submission link found") + submit_file = args.filename or task.submit_file or "" submit_form_html = session.http_request(urljoin(base_url, task.submit_link)) submit_form_data = parse_form(submit_form_html) @@ -434,7 +446,7 @@ def main() -> None: for key, value in submit_form_data.items(): submit_form_data[key] = (None, value) - submit_form_data["file"] = (task.submit_file, open(task.submit_file, "rb")) + submit_form_data["file"] = (submit_file, open(submit_file, "rb")) submit_form_data["lang"] = (None, "Python3") submit_form_data["option"] = (None, "CPython3") From bd132011c4c1328fed36b418b55bb87143946f65 Mon Sep 17 00:00:00 2001 From: Edwin Hermans Date: Sun, 19 May 2024 21:25:34 -0400 Subject: [PATCH 5/5] Split off some functionality into its own function --- tyora.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tyora.py b/tyora.py index afd00f3..4a37e57 100644 --- a/tyora.py +++ b/tyora.py @@ -375,6 +375,25 @@ def print_task(task: Task) -> None: # with open(filename, 'r') as f: +def parse_submit_result(html: AnyStr) -> dict[str, str]: + root = htmlement.fromstring(html) + submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") + submit_status_span_element = submit_status_element.find("td/span") or Element( + "span" + ) + submit_status = submit_status_span_element.text or "" + submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") + submit_result_span_element = submit_result_element.find("td/span") or Element( + "span" + ) + submit_result = submit_result_span_element.text or "" + + return { + "status": submit_status.lower(), + "result": submit_result.lower(), + } + + def main() -> None: args = parse_args() @@ -462,20 +481,10 @@ def main() -> None: html = session.http_request(result_url) print() - root = htmlement.fromstring(html) - submit_status_element = root.find('.//td[.="Status:"]/..') or Element("td") - submit_status_span_element = submit_status_element.find("td/span") or Element( - "span" - ) - submit_status = submit_status_span_element.text or "" - submit_result_element = root.find('.//td[.="Result:"]/..') or Element("td") - submit_result_span_element = submit_result_element.find("td/span") or Element( - "span" - ) - submit_result = submit_result_span_element.text or "" + results = parse_submit_result(html) - print(f"Submission status: {submit_status.lower()}") - print(f"Submission result: {submit_result.lower()}") + print(f"Submission status: {results['status']}") + print(f"Submission result: {results['result']}") if __name__ == "__main__":